feature_engineering - Anomalies Detection.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<h3><center>Feature Engineering and Anomalies - Outliers Detection</h3>\n",
    "This notebook is devoted in feature exploration. The results of this notebook coming from experimentation and they are not used explicitly. In a later stage the findings of this notebook are implemented on both training and testing data, before fitting and testing a model."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "import matplotlib.pyplot as plt \n",
    "from matplotlib import style  \n",
    "style.use(\"ggplot\")\n",
    "\n",
    "import pandas as pd\n",
    "\n",
    "from commons import load_data, get_dummies"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "# load data\n",
    "filename = \"data/traindata.csv\"\n",
    "data = load_data(filename)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## feature engineering \"day\" feature"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>purpose_validated</th>\n",
       "      <th>dur_in_sec</th>\n",
       "      <th>hour_start</th>\n",
       "      <th>hour_end</th>\n",
       "      <th>day_of_month</th>\n",
       "      <th>week</th>\n",
       "      <th>dist_gtfsstops_all</th>\n",
       "      <th>dist_gtfsstops_train</th>\n",
       "      <th>dist_gtfsstops_tram</th>\n",
       "      <th>dist_gtfsstops_bus</th>\n",
       "      <th>day</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>home</td>\n",
       "      <td>1017.087000</td>\n",
       "      <td>8</td>\n",
       "      <td>9</td>\n",
       "      <td>2</td>\n",
       "      <td>5</td>\n",
       "      <td>126.188532</td>\n",
       "      <td>574.259045</td>\n",
       "      <td>10496.593564</td>\n",
       "      <td>126.188532</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>leisure</td>\n",
       "      <td>310.000000</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>14</td>\n",
       "      <td>2</td>\n",
       "      <td>411.478950</td>\n",
       "      <td>3388.489009</td>\n",
       "      <td>24408.713071</td>\n",
       "      <td>411.478950</td>\n",
       "      <td>6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>leisure</td>\n",
       "      <td>4993.099000</td>\n",
       "      <td>16</td>\n",
       "      <td>17</td>\n",
       "      <td>23</td>\n",
       "      <td>51</td>\n",
       "      <td>242.799218</td>\n",
       "      <td>420.061325</td>\n",
       "      <td>242.799218</td>\n",
       "      <td>242.799218</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3</td>\n",
       "      <td>errand</td>\n",
       "      <td>422.515001</td>\n",
       "      <td>11</td>\n",
       "      <td>11</td>\n",
       "      <td>29</td>\n",
       "      <td>26</td>\n",
       "      <td>73.118524</td>\n",
       "      <td>1757.238956</td>\n",
       "      <td>16999.384112</td>\n",
       "      <td>73.118524</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4</td>\n",
       "      <td>leisure</td>\n",
       "      <td>9673.579000</td>\n",
       "      <td>18</td>\n",
       "      <td>21</td>\n",
       "      <td>28</td>\n",
       "      <td>4</td>\n",
       "      <td>644.638357</td>\n",
       "      <td>644.638357</td>\n",
       "      <td>45432.181046</td>\n",
       "      <td>2333.563174</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   id purpose_validated   dur_in_sec  hour_start  hour_end  day_of_month  \\\n",
       "0   0              home  1017.087000           8         9             2   \n",
       "1   1           leisure   310.000000           0         0            14   \n",
       "2   2           leisure  4993.099000          16        17            23   \n",
       "3   3            errand   422.515001          11        11            29   \n",
       "4   4           leisure  9673.579000          18        21            28   \n",
       "\n",
       "   week  dist_gtfsstops_all  dist_gtfsstops_train  dist_gtfsstops_tram  \\\n",
       "0     5          126.188532            574.259045         10496.593564   \n",
       "1     2          411.478950           3388.489009         24408.713071   \n",
       "2    51          242.799218            420.061325           242.799218   \n",
       "3    26           73.118524           1757.238956         16999.384112   \n",
       "4     4          644.638357            644.638357         45432.181046   \n",
       "\n",
       "   dist_gtfsstops_bus  day  \n",
       "0          126.188532    3  \n",
       "1          411.478950    6  \n",
       "2          242.799218    4  \n",
       "3           73.118524    3  \n",
       "4         2333.563174    5  "
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "# weekdays vs weekends feature.\n",
    "weekend = [5,6]\n",
    "data['weekend'] = 0\n",
    "data.loc[data['day'].isin(weekend),'weekend'] = 1\n",
    "\n",
    "# After this commands we have creted a new feature called \"weekend\" which has only two values, \n",
    "#either 1 or 0. The value 1 corresponds to the days 5 and 6 of the feature 'day', whilst the \n",
    "#value 0 corresponds to every other day of the week."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>purpose_validated</th>\n",
       "      <th>dur_in_sec</th>\n",
       "      <th>hour_start</th>\n",
       "      <th>hour_end</th>\n",
       "      <th>day_of_month</th>\n",
       "      <th>week</th>\n",
       "      <th>dist_gtfsstops_all</th>\n",
       "      <th>dist_gtfsstops_train</th>\n",
       "      <th>dist_gtfsstops_tram</th>\n",
       "      <th>dist_gtfsstops_bus</th>\n",
       "      <th>day</th>\n",
       "      <th>weekend</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>home</td>\n",
       "      <td>1017.087000</td>\n",
       "      <td>8</td>\n",
       "      <td>9</td>\n",
       "      <td>2</td>\n",
       "      <td>5</td>\n",
       "      <td>126.188532</td>\n",
       "      <td>574.259045</td>\n",
       "      <td>10496.593564</td>\n",
       "      <td>126.188532</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>leisure</td>\n",
       "      <td>310.000000</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>14</td>\n",
       "      <td>2</td>\n",
       "      <td>411.478950</td>\n",
       "      <td>3388.489009</td>\n",
       "      <td>24408.713071</td>\n",
       "      <td>411.478950</td>\n",
       "      <td>6</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>leisure</td>\n",
       "      <td>4993.099000</td>\n",
       "      <td>16</td>\n",
       "      <td>17</td>\n",
       "      <td>23</td>\n",
       "      <td>51</td>\n",
       "      <td>242.799218</td>\n",
       "      <td>420.061325</td>\n",
       "      <td>242.799218</td>\n",
       "      <td>242.799218</td>\n",
       "      <td>4</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3</td>\n",
       "      <td>errand</td>\n",
       "      <td>422.515001</td>\n",
       "      <td>11</td>\n",
       "      <td>11</td>\n",
       "      <td>29</td>\n",
       "      <td>26</td>\n",
       "      <td>73.118524</td>\n",
       "      <td>1757.238956</td>\n",
       "      <td>16999.384112</td>\n",
       "      <td>73.118524</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4</td>\n",
       "      <td>leisure</td>\n",
       "      <td>9673.579000</td>\n",
       "      <td>18</td>\n",
       "      <td>21</td>\n",
       "      <td>28</td>\n",
       "      <td>4</td>\n",
       "      <td>644.638357</td>\n",
       "      <td>644.638357</td>\n",
       "      <td>45432.181046</td>\n",
       "      <td>2333.563174</td>\n",
       "      <td>5</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>5</td>\n",
       "      <td>leisure</td>\n",
       "      <td>331.453000</td>\n",
       "      <td>6</td>\n",
       "      <td>6</td>\n",
       "      <td>1</td>\n",
       "      <td>13</td>\n",
       "      <td>20.641240</td>\n",
       "      <td>20.641240</td>\n",
       "      <td>1549.041373</td>\n",
       "      <td>69.659482</td>\n",
       "      <td>5</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   id purpose_validated   dur_in_sec  hour_start  hour_end  day_of_month  \\\n",
       "0   0              home  1017.087000           8         9             2   \n",
       "1   1           leisure   310.000000           0         0            14   \n",
       "2   2           leisure  4993.099000          16        17            23   \n",
       "3   3            errand   422.515001          11        11            29   \n",
       "4   4           leisure  9673.579000          18        21            28   \n",
       "5   5           leisure   331.453000           6         6             1   \n",
       "\n",
       "   week  dist_gtfsstops_all  dist_gtfsstops_train  dist_gtfsstops_tram  \\\n",
       "0     5          126.188532            574.259045         10496.593564   \n",
       "1     2          411.478950           3388.489009         24408.713071   \n",
       "2    51          242.799218            420.061325           242.799218   \n",
       "3    26           73.118524           1757.238956         16999.384112   \n",
       "4     4          644.638357            644.638357         45432.181046   \n",
       "5    13           20.641240             20.641240          1549.041373   \n",
       "\n",
       "   dist_gtfsstops_bus  day  weekend  \n",
       "0          126.188532    3        0  \n",
       "1          411.478950    6        1  \n",
       "2          242.799218    4        0  \n",
       "3           73.118524    3        0  \n",
       "4         2333.563174    5        1  \n",
       "5           69.659482    5        1  "
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.head(n=6)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Since the feature 'Day' contains categorical values, it is neccesary to transform the values into dummy variables, using the one-hotting technique. This will prevent the model from seeing the higher values (like Friday = 4) as more important than others."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "# transform to dummy\n",
    "data = get_dummies(data, columns = [\"day\"])  #Convert categorical variable into dummy/indicator variables"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "After this command the feature 'days' will be splited into five features, one for each day of the week (from Monday to Friday)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>purpose_validated</th>\n",
       "      <th>dur_in_sec</th>\n",
       "      <th>hour_start</th>\n",
       "      <th>hour_end</th>\n",
       "      <th>day_of_month</th>\n",
       "      <th>week</th>\n",
       "      <th>dist_gtfsstops_all</th>\n",
       "      <th>dist_gtfsstops_train</th>\n",
       "      <th>dist_gtfsstops_tram</th>\n",
       "      <th>dist_gtfsstops_bus</th>\n",
       "      <th>weekend</th>\n",
       "      <th>day_0</th>\n",
       "      <th>day_1</th>\n",
       "      <th>day_2</th>\n",
       "      <th>day_3</th>\n",
       "      <th>day_4</th>\n",
       "      <th>day_5</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>home</td>\n",
       "      <td>1017.087000</td>\n",
       "      <td>8</td>\n",
       "      <td>9</td>\n",
       "      <td>2</td>\n",
       "      <td>5</td>\n",
       "      <td>126.188532</td>\n",
       "      <td>574.259045</td>\n",
       "      <td>10496.593564</td>\n",
       "      <td>126.188532</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>leisure</td>\n",
       "      <td>310.000000</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>14</td>\n",
       "      <td>2</td>\n",
       "      <td>411.478950</td>\n",
       "      <td>3388.489009</td>\n",
       "      <td>24408.713071</td>\n",
       "      <td>411.478950</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>leisure</td>\n",
       "      <td>4993.099000</td>\n",
       "      <td>16</td>\n",
       "      <td>17</td>\n",
       "      <td>23</td>\n",
       "      <td>51</td>\n",
       "      <td>242.799218</td>\n",
       "      <td>420.061325</td>\n",
       "      <td>242.799218</td>\n",
       "      <td>242.799218</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3</td>\n",
       "      <td>errand</td>\n",
       "      <td>422.515001</td>\n",
       "      <td>11</td>\n",
       "      <td>11</td>\n",
       "      <td>29</td>\n",
       "      <td>26</td>\n",
       "      <td>73.118524</td>\n",
       "      <td>1757.238956</td>\n",
       "      <td>16999.384112</td>\n",
       "      <td>73.118524</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4</td>\n",
       "      <td>leisure</td>\n",
       "      <td>9673.579000</td>\n",
       "      <td>18</td>\n",
       "      <td>21</td>\n",
       "      <td>28</td>\n",
       "      <td>4</td>\n",
       "      <td>644.638357</td>\n",
       "      <td>644.638357</td>\n",
       "      <td>45432.181046</td>\n",
       "      <td>2333.563174</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>5</td>\n",
       "      <td>leisure</td>\n",
       "      <td>331.453000</td>\n",
       "      <td>6</td>\n",
       "      <td>6</td>\n",
       "      <td>1</td>\n",
       "      <td>13</td>\n",
       "      <td>20.641240</td>\n",
       "      <td>20.641240</td>\n",
       "      <td>1549.041373</td>\n",
       "      <td>69.659482</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   id purpose_validated   dur_in_sec  hour_start  hour_end  day_of_month  \\\n",
       "0   0              home  1017.087000           8         9             2   \n",
       "1   1           leisure   310.000000           0         0            14   \n",
       "2   2           leisure  4993.099000          16        17            23   \n",
       "3   3            errand   422.515001          11        11            29   \n",
       "4   4           leisure  9673.579000          18        21            28   \n",
       "5   5           leisure   331.453000           6         6             1   \n",
       "\n",
       "   week  dist_gtfsstops_all  dist_gtfsstops_train  dist_gtfsstops_tram  \\\n",
       "0     5          126.188532            574.259045         10496.593564   \n",
       "1     2          411.478950           3388.489009         24408.713071   \n",
       "2    51          242.799218            420.061325           242.799218   \n",
       "3    26           73.118524           1757.238956         16999.384112   \n",
       "4     4          644.638357            644.638357         45432.181046   \n",
       "5    13           20.641240             20.641240          1549.041373   \n",
       "\n",
       "   dist_gtfsstops_bus  weekend  day_0  day_1  day_2  day_3  day_4  day_5  \n",
       "0          126.188532        0      0      0      0      1      0      0  \n",
       "1          411.478950        1      0      0      0      0      0      0  \n",
       "2          242.799218        0      0      0      0      0      1      0  \n",
       "3           73.118524        0      0      0      0      1      0      0  \n",
       "4         2333.563174        1      0      0      0      0      0      1  \n",
       "5           69.659482        1      0      0      0      0      0      1  "
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.head(n=6)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## feature engineering \"hour_start\" feature"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The commands that follow will create eight extra features. More specificaly, the feature 'hour_start', will be split into eight groups('start hour bins'). Thus, for a certaine example of the data that belongs to a certain bin, will be assigned to it the value 1, and for every other bin with the value 0. Additionally the feature \"hour_start\" will be converted to dummie variables like the previous example."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "8\n"
     ]
    }
   ],
   "source": [
    "hour_start_bins = pd.cut(data.hour_start, bins=8)\n",
    "data['start_hour_bins'] = hour_start_bins\n",
    "data = get_dummies(data, ['start_hour_bins', 'hour_start'])\n",
    "print(hour_start_bins.nunique())\n",
    "#print(data.hour_start.nunique())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index(['id', 'purpose_validated', 'dur_in_sec', 'hour_end', 'day_of_month',\n",
       "       'week', 'dist_gtfsstops_all', 'dist_gtfsstops_train',\n",
       "       'dist_gtfsstops_tram', 'dist_gtfsstops_bus', 'weekend', 'day_0',\n",
       "       'day_1', 'day_2', 'day_3', 'day_4', 'day_5',\n",
       "       'start_hour_bins_(-0.023, 2.875]', 'start_hour_bins_(2.875, 5.75]',\n",
       "       'start_hour_bins_(5.75, 8.625]', 'start_hour_bins_(8.625, 11.5]',\n",
       "       'start_hour_bins_(11.5, 14.375]', 'start_hour_bins_(14.375, 17.25]',\n",
       "       'start_hour_bins_(17.25, 20.125]', 'hour_start_0', 'hour_start_1',\n",
       "       'hour_start_2', 'hour_start_3', 'hour_start_4', 'hour_start_5',\n",
       "       'hour_start_6', 'hour_start_7', 'hour_start_8', 'hour_start_9',\n",
       "       'hour_start_10', 'hour_start_11', 'hour_start_12', 'hour_start_13',\n",
       "       'hour_start_14', 'hour_start_15', 'hour_start_16', 'hour_start_17',\n",
       "       'hour_start_18', 'hour_start_19', 'hour_start_20', 'hour_start_21',\n",
       "       'hour_start_22'],\n",
       "      dtype='object')"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>purpose_validated</th>\n",
       "      <th>dur_in_sec</th>\n",
       "      <th>hour_end</th>\n",
       "      <th>day_of_month</th>\n",
       "      <th>week</th>\n",
       "      <th>dist_gtfsstops_all</th>\n",
       "      <th>dist_gtfsstops_train</th>\n",
       "      <th>dist_gtfsstops_tram</th>\n",
       "      <th>dist_gtfsstops_bus</th>\n",
       "      <th>...</th>\n",
       "      <th>hour_start_13</th>\n",
       "      <th>hour_start_14</th>\n",
       "      <th>hour_start_15</th>\n",
       "      <th>hour_start_16</th>\n",
       "      <th>hour_start_17</th>\n",
       "      <th>hour_start_18</th>\n",
       "      <th>hour_start_19</th>\n",
       "      <th>hour_start_20</th>\n",
       "      <th>hour_start_21</th>\n",
       "      <th>hour_start_22</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>home</td>\n",
       "      <td>1017.087000</td>\n",
       "      <td>9</td>\n",
       "      <td>2</td>\n",
       "      <td>5</td>\n",
       "      <td>126.188532</td>\n",
       "      <td>574.259045</td>\n",
       "      <td>10496.593564</td>\n",
       "      <td>126.188532</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>leisure</td>\n",
       "      <td>310.000000</td>\n",
       "      <td>0</td>\n",
       "      <td>14</td>\n",
       "      <td>2</td>\n",
       "      <td>411.478950</td>\n",
       "      <td>3388.489009</td>\n",
       "      <td>24408.713071</td>\n",
       "      <td>411.478950</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>leisure</td>\n",
       "      <td>4993.099000</td>\n",
       "      <td>17</td>\n",
       "      <td>23</td>\n",
       "      <td>51</td>\n",
       "      <td>242.799218</td>\n",
       "      <td>420.061325</td>\n",
       "      <td>242.799218</td>\n",
       "      <td>242.799218</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3</td>\n",
       "      <td>errand</td>\n",
       "      <td>422.515001</td>\n",
       "      <td>11</td>\n",
       "      <td>29</td>\n",
       "      <td>26</td>\n",
       "      <td>73.118524</td>\n",
       "      <td>1757.238956</td>\n",
       "      <td>16999.384112</td>\n",
       "      <td>73.118524</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4</td>\n",
       "      <td>leisure</td>\n",
       "      <td>9673.579000</td>\n",
       "      <td>21</td>\n",
       "      <td>28</td>\n",
       "      <td>4</td>\n",
       "      <td>644.638357</td>\n",
       "      <td>644.638357</td>\n",
       "      <td>45432.181046</td>\n",
       "      <td>2333.563174</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 47 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   id purpose_validated   dur_in_sec  hour_end  day_of_month  week  \\\n",
       "0   0              home  1017.087000         9             2     5   \n",
       "1   1           leisure   310.000000         0            14     2   \n",
       "2   2           leisure  4993.099000        17            23    51   \n",
       "3   3            errand   422.515001        11            29    26   \n",
       "4   4           leisure  9673.579000        21            28     4   \n",
       "\n",
       "   dist_gtfsstops_all  dist_gtfsstops_train  dist_gtfsstops_tram  \\\n",
       "0          126.188532            574.259045         10496.593564   \n",
       "1          411.478950           3388.489009         24408.713071   \n",
       "2          242.799218            420.061325           242.799218   \n",
       "3           73.118524           1757.238956         16999.384112   \n",
       "4          644.638357            644.638357         45432.181046   \n",
       "\n",
       "   dist_gtfsstops_bus      ...        hour_start_13  hour_start_14  \\\n",
       "0          126.188532      ...                    0              0   \n",
       "1          411.478950      ...                    0              0   \n",
       "2          242.799218      ...                    0              0   \n",
       "3           73.118524      ...                    0              0   \n",
       "4         2333.563174      ...                    0              0   \n",
       "\n",
       "   hour_start_15  hour_start_16  hour_start_17  hour_start_18  hour_start_19  \\\n",
       "0              0              0              0              0              0   \n",
       "1              0              0              0              0              0   \n",
       "2              0              1              0              0              0   \n",
       "3              0              0              0              0              0   \n",
       "4              0              0              0              1              0   \n",
       "\n",
       "   hour_start_20  hour_start_21  hour_start_22  \n",
       "0              0              0              0  \n",
       "1              0              0              0  \n",
       "2              0              0              0  \n",
       "3              0              0              0  \n",
       "4              0              0              0  \n",
       "\n",
       "[5 rows x 47 columns]"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.head(n=5)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## day_of_month feature"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "# just transform to dummy\n",
    "# maybe bin start,middle,end of month (maybe go out more when they have money at the beginning of the month)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Week Feature"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# TODO: week to month and then month to season"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## dist_* features"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "I already have the min distance. I can also add the max and mean and sum distance. \n",
    "This will add two more features."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [],
   "source": [
    "data[\"dist_max\"] = data[[\"dist_gtfsstops_train\",\"dist_gtfsstops_tram\",\"dist_gtfsstops_bus\"]].max(axis=1)\n",
    "data[\"dist_mean\"] = data[[\"dist_gtfsstops_train\",\"dist_gtfsstops_tram\",\"dist_gtfsstops_bus\"]].mean(axis=1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Anomaly detection"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<h5>At this point an effort is given with respect of finding incosistencies in our data</h5>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 118,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.neighbors import LocalOutlierFactor\n",
    "clf = LocalOutlierFactor(n_neighbors=20, contamination=0.002)\n",
    "y_pred = clf.fit_predict(data[[\"dur_in_sec\",\"dist_gtfsstops_train\",\"dist_gtfsstops_tram\",\"dist_gtfsstops_bus\"]])\n",
    "data['is_anomaly'] = y_pred"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 120,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>purpose_validated</th>\n",
       "      <th>dur_in_sec</th>\n",
       "      <th>hour_end</th>\n",
       "      <th>day_of_month</th>\n",
       "      <th>week</th>\n",
       "      <th>dist_gtfsstops_all</th>\n",
       "      <th>dist_gtfsstops_train</th>\n",
       "      <th>dist_gtfsstops_tram</th>\n",
       "      <th>dist_gtfsstops_bus</th>\n",
       "      <th>...</th>\n",
       "      <th>hour_start_16</th>\n",
       "      <th>hour_start_17</th>\n",
       "      <th>hour_start_18</th>\n",
       "      <th>hour_start_19</th>\n",
       "      <th>hour_start_20</th>\n",
       "      <th>hour_start_21</th>\n",
       "      <th>hour_start_22</th>\n",
       "      <th>dist_max</th>\n",
       "      <th>dist_sum</th>\n",
       "      <th>is_anomaly</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>168</th>\n",
       "      <td>168</td>\n",
       "      <td>wait</td>\n",
       "      <td>2020.306999</td>\n",
       "      <td>11</td>\n",
       "      <td>5</td>\n",
       "      <td>1</td>\n",
       "      <td>1.450138e+07</td>\n",
       "      <td>1.450138e+07</td>\n",
       "      <td>1.477359e+07</td>\n",
       "      <td>1.464657e+07</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1.477359e+07</td>\n",
       "      <td>4.392153e+07</td>\n",
       "      <td>-1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>171</th>\n",
       "      <td>171</td>\n",
       "      <td>leisure</td>\n",
       "      <td>764.063999</td>\n",
       "      <td>12</td>\n",
       "      <td>11</td>\n",
       "      <td>49</td>\n",
       "      <td>4.295447e+01</td>\n",
       "      <td>8.604259e+01</td>\n",
       "      <td>1.480002e+05</td>\n",
       "      <td>8.198159e+01</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1.480002e+05</td>\n",
       "      <td>1.481682e+05</td>\n",
       "      <td>-1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>404</th>\n",
       "      <td>404</td>\n",
       "      <td>leisure</td>\n",
       "      <td>872.068000</td>\n",
       "      <td>12</td>\n",
       "      <td>5</td>\n",
       "      <td>1</td>\n",
       "      <td>1.450215e+07</td>\n",
       "      <td>1.450215e+07</td>\n",
       "      <td>1.477431e+07</td>\n",
       "      <td>1.464724e+07</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1.477431e+07</td>\n",
       "      <td>4.392369e+07</td>\n",
       "      <td>-1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1685</th>\n",
       "      <td>1685</td>\n",
       "      <td>leisure</td>\n",
       "      <td>353.017000</td>\n",
       "      <td>19</td>\n",
       "      <td>18</td>\n",
       "      <td>11</td>\n",
       "      <td>5.131761e+01</td>\n",
       "      <td>7.224220e+01</td>\n",
       "      <td>1.479223e+05</td>\n",
       "      <td>5.131761e+01</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1.479223e+05</td>\n",
       "      <td>1.480459e+05</td>\n",
       "      <td>-1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1848</th>\n",
       "      <td>1848</td>\n",
       "      <td>errand</td>\n",
       "      <td>1159.506000</td>\n",
       "      <td>9</td>\n",
       "      <td>18</td>\n",
       "      <td>7</td>\n",
       "      <td>1.454995e+07</td>\n",
       "      <td>1.454995e+07</td>\n",
       "      <td>1.482290e+07</td>\n",
       "      <td>1.469650e+07</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1.482290e+07</td>\n",
       "      <td>4.406935e+07</td>\n",
       "      <td>-1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2374</th>\n",
       "      <td>2374</td>\n",
       "      <td>home</td>\n",
       "      <td>-171448.670000</td>\n",
       "      <td>8</td>\n",
       "      <td>24</td>\n",
       "      <td>25</td>\n",
       "      <td>3.075106e+02</td>\n",
       "      <td>2.645471e+03</td>\n",
       "      <td>3.310238e+04</td>\n",
       "      <td>3.075106e+02</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>3.310238e+04</td>\n",
       "      <td>3.605536e+04</td>\n",
       "      <td>-1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2726</th>\n",
       "      <td>2726</td>\n",
       "      <td>wait</td>\n",
       "      <td>546.000000</td>\n",
       "      <td>8</td>\n",
       "      <td>27</td>\n",
       "      <td>9</td>\n",
       "      <td>4.407361e+01</td>\n",
       "      <td>7.527494e+03</td>\n",
       "      <td>1.392099e+05</td>\n",
       "      <td>2.669291e+03</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1.392099e+05</td>\n",
       "      <td>1.494066e+05</td>\n",
       "      <td>-1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3405</th>\n",
       "      <td>3405</td>\n",
       "      <td>leisure</td>\n",
       "      <td>414.962000</td>\n",
       "      <td>13</td>\n",
       "      <td>8</td>\n",
       "      <td>32</td>\n",
       "      <td>3.672308e+01</td>\n",
       "      <td>9.930756e+03</td>\n",
       "      <td>1.454029e+05</td>\n",
       "      <td>3.672308e+01</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1.454029e+05</td>\n",
       "      <td>1.553704e+05</td>\n",
       "      <td>-1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3686</th>\n",
       "      <td>3686</td>\n",
       "      <td>wait</td>\n",
       "      <td>725.000000</td>\n",
       "      <td>8</td>\n",
       "      <td>29</td>\n",
       "      <td>52</td>\n",
       "      <td>1.162068e+01</td>\n",
       "      <td>1.193461e+02</td>\n",
       "      <td>1.470435e+05</td>\n",
       "      <td>7.225561e+01</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1.470435e+05</td>\n",
       "      <td>1.472351e+05</td>\n",
       "      <td>-1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3734</th>\n",
       "      <td>3734</td>\n",
       "      <td>wait</td>\n",
       "      <td>918.999000</td>\n",
       "      <td>9</td>\n",
       "      <td>23</td>\n",
       "      <td>16</td>\n",
       "      <td>3.893122e+01</td>\n",
       "      <td>4.738335e+01</td>\n",
       "      <td>1.479104e+05</td>\n",
       "      <td>3.893122e+01</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1.479104e+05</td>\n",
       "      <td>1.479967e+05</td>\n",
       "      <td>-1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>10 rows × 50 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "        id purpose_validated     dur_in_sec  hour_end  day_of_month  week  \\\n",
       "168    168              wait    2020.306999        11             5     1   \n",
       "171    171           leisure     764.063999        12            11    49   \n",
       "404    404           leisure     872.068000        12             5     1   \n",
       "1685  1685           leisure     353.017000        19            18    11   \n",
       "1848  1848            errand    1159.506000         9            18     7   \n",
       "2374  2374              home -171448.670000         8            24    25   \n",
       "2726  2726              wait     546.000000         8            27     9   \n",
       "3405  3405           leisure     414.962000        13             8    32   \n",
       "3686  3686              wait     725.000000         8            29    52   \n",
       "3734  3734              wait     918.999000         9            23    16   \n",
       "\n",
       "      dist_gtfsstops_all  dist_gtfsstops_train  dist_gtfsstops_tram  \\\n",
       "168         1.450138e+07          1.450138e+07         1.477359e+07   \n",
       "171         4.295447e+01          8.604259e+01         1.480002e+05   \n",
       "404         1.450215e+07          1.450215e+07         1.477431e+07   \n",
       "1685        5.131761e+01          7.224220e+01         1.479223e+05   \n",
       "1848        1.454995e+07          1.454995e+07         1.482290e+07   \n",
       "2374        3.075106e+02          2.645471e+03         3.310238e+04   \n",
       "2726        4.407361e+01          7.527494e+03         1.392099e+05   \n",
       "3405        3.672308e+01          9.930756e+03         1.454029e+05   \n",
       "3686        1.162068e+01          1.193461e+02         1.470435e+05   \n",
       "3734        3.893122e+01          4.738335e+01         1.479104e+05   \n",
       "\n",
       "      dist_gtfsstops_bus     ...      hour_start_16  hour_start_17  \\\n",
       "168         1.464657e+07     ...                  0              0   \n",
       "171         8.198159e+01     ...                  0              0   \n",
       "404         1.464724e+07     ...                  0              0   \n",
       "1685        5.131761e+01     ...                  0              0   \n",
       "1848        1.469650e+07     ...                  0              0   \n",
       "2374        3.075106e+02     ...                  0              0   \n",
       "2726        2.669291e+03     ...                  0              0   \n",
       "3405        3.672308e+01     ...                  0              0   \n",
       "3686        7.225561e+01     ...                  0              0   \n",
       "3734        3.893122e+01     ...                  0              0   \n",
       "\n",
       "      hour_start_18  hour_start_19  hour_start_20  hour_start_21  \\\n",
       "168               0              0              0              0   \n",
       "171               0              0              0              0   \n",
       "404               0              0              0              0   \n",
       "1685              0              1              0              0   \n",
       "1848              0              0              0              0   \n",
       "2374              0              0              0              0   \n",
       "2726              0              0              0              0   \n",
       "3405              0              0              0              0   \n",
       "3686              0              0              0              0   \n",
       "3734              0              0              0              0   \n",
       "\n",
       "      hour_start_22      dist_max      dist_sum  is_anomaly  \n",
       "168               0  1.477359e+07  4.392153e+07          -1  \n",
       "171               0  1.480002e+05  1.481682e+05          -1  \n",
       "404               0  1.477431e+07  4.392369e+07          -1  \n",
       "1685              0  1.479223e+05  1.480459e+05          -1  \n",
       "1848              0  1.482290e+07  4.406935e+07          -1  \n",
       "2374              0  3.310238e+04  3.605536e+04          -1  \n",
       "2726              0  1.392099e+05  1.494066e+05          -1  \n",
       "3405              0  1.454029e+05  1.553704e+05          -1  \n",
       "3686              0  1.470435e+05  1.472351e+05          -1  \n",
       "3734              0  1.479104e+05  1.479967e+05          -1  \n",
       "\n",
       "[10 rows x 50 columns]"
      ]
     },
     "execution_count": 120,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.loc[data['is_anomaly'] == -1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 121,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[168, 171, 404, 1685, 1848, 2374, 2726, 3405, 3686, 3734]"
      ]
     },
     "execution_count": 121,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# just remove by id \n",
    "list(data.loc[data['is_anomaly'] == -1, 'id']) # this is a list containing the id's \n",
    "                                               # of the incosistent examples."
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}