Main.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "iNYkuahidXY3"
   },
   "outputs": [],
   "source": [
    "# Import all necessary libraries\n",
    "\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "from datetime import datetime, timedelta\n",
    "from math import radians, cos, sin, asin, sqrt\n",
    "from pandas.tseries.holiday import USFederalHolidayCalendar\n",
    "\n",
    "import seaborn as sns\n",
    "import plotly.graph_objects as go\n",
    "%matplotlib inline\n",
    "import matplotlib\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "import xgboost as xgb\n",
    "from sklearn.model_selection import KFold\n",
    "from sklearn.cluster import MiniBatchKMeans"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "Sk317dg5dXcD"
   },
   "outputs": [],
   "source": [
    "# Read the train and test datasets\n",
    "train_data = pd.read_csv('https://firebasestorage.googleapis.com/v0/b/test-bbb2d.appspot.com/o/train.csv?alt=media&token=505676d4-dad3-43ac-bc96-643776d68b06')\n",
    "test_data = pd.read_csv('https://firebasestorage.googleapis.com/v0/b/test-bbb2d.appspot.com/o/test.csv?alt=media&token=4525c442-18d9-4d0a-afba-3b6fabf8db4b')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "0-T0r-fcF7Rm"
   },
   "source": [
    "# INITIAL DATA PRE-PROCESSING "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 197
    },
    "id": "3R8aZYVjdXe4",
    "outputId": "4b375440-4a62-48ad-80e2-acc364fdc0fd"
   },
   "outputs": [],
   "source": [
    "train_data.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 287
    },
    "id": "Rh27toMXDDnH",
    "outputId": "8271995c-7f91-47e1-aa24-968c31e7acb4"
   },
   "outputs": [],
   "source": [
    "train_data.describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "u9YeeYRdDDsH",
    "outputId": "de2975b1-83fc-4f1e-a55f-71215af28886"
   },
   "outputs": [],
   "source": [
    "train_data.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 47
    },
    "id": "iWS0YQqrHQzQ",
    "outputId": "145c9026-edcc-4cec-b55b-d446bd43b816"
   },
   "outputs": [],
   "source": [
    "train_data[train_data.isna().any(axis=1)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "x_CWK4Mqjuyz",
    "outputId": "7194403e-17dc-4cf2-f690-9cb02f38eca6"
   },
   "outputs": [],
   "source": [
    "train_data.dtypes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "RmbJGtfnk8r2"
   },
   "outputs": [],
   "source": [
    "# remove outliers for passenger_count 7,8,9,0\n",
    "index = train_data[ (train_data['passenger_count'] == 7) | (train_data['passenger_count'] == 8) | (train_data['passenger_count'] == 9) | (train_data['passenger_count'] == 0)].index\n",
    "train_data.drop(index , inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 323
    },
    "id": "2EA89Ce9DK0W",
    "outputId": "7f839487-edec-4645-f1b4-00d5e4813c83"
   },
   "outputs": [],
   "source": [
    "# Remove outliers\n",
    "mean = np.mean(train_data['trip_duration'])\n",
    "print(\"Mean for Trip Duration is:\", mean)\n",
    "\n",
    "standard_deviation = np.std(train_data['trip_duration'])\n",
    "print(\"Standard Deviation for Trip Duration is:\", standard_deviation)\n",
    "\n",
    "train_data = train_data[train_data['trip_duration'].between(mean - 2*standard_deviation, mean + 2*standard_deviation, inclusive = True)]\n",
    "train_data.describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "0QsXLgOdVRKj"
   },
   "outputs": [],
   "source": [
    "# Data Formatting\n",
    "\n",
    "train_data['store_and_fwd_flag'] = train_data['store_and_fwd_flag'].map(lambda x: 0 if x == 'N' else 1)\n",
    "test_data['store_and_fwd_flag'] = test_data['store_and_fwd_flag'].map(lambda x: 0 if x == 'N' else 1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "BAy2GLx3DK23",
    "outputId": "73abde9c-b425-4bf9-f69b-be42e7a1b5c2"
   },
   "outputs": [],
   "source": [
    "# Feature Extraction\n",
    "# Decomposing timestamp for train data into date and time\n",
    "\n",
    "train_data['pickup_datetime'] = pd.to_datetime(train_data.pickup_datetime)\n",
    "train_data.loc[:, 'pickup_date'] = train_data['pickup_datetime'].dt.date\n",
    "train_data.loc[:, 'pickup_time'] = train_data['pickup_datetime'].dt.time\n",
    "train_data['dropoff_datetime'] = pd.to_datetime(train_data.dropoff_datetime)\n",
    "train_data.loc[:, 'dropoff_date'] = train_data['dropoff_datetime'].dt.date\n",
    "train_data.loc[:, 'dropoff_time'] = train_data['dropoff_datetime'].dt.time\n",
    "train_data.loc[:, 'dropoff_hour'] = train_data['dropoff_datetime'].dt.hour\n",
    "train_data.loc[:, 'dropoff_weekday'] = train_data['dropoff_datetime'].dt.weekday\n",
    "train_data.loc[:, 'dropoff_month'] = train_data['dropoff_datetime'].dt.month\n",
    "train_data.loc[:, 'pickup_weekday'] = train_data['pickup_datetime'].dt.weekday\n",
    "train_data.loc[:, 'pickup_weekofyear'] = train_data['pickup_datetime'].dt.weekofyear\n",
    "train_data.loc[:, 'pickup_hour'] = train_data['pickup_datetime'].dt.hour\n",
    "train_data.loc[:, 'pickup_minute'] = train_data['pickup_datetime'].dt.minute\n",
    "train_data.loc[:, 'pickup_dt'] = (train_data['pickup_datetime'] - train_data['pickup_datetime'].min()).dt.total_seconds()\n",
    "train_data.loc[:, 'pickup_week_hour'] = train_data['pickup_weekday'] * 24 + train_data['pickup_hour']\n",
    "train_data.loc[:, 'pickup_dayofyear'] = train_data['pickup_datetime'].dt.dayofyear\n",
    "train_data.loc[:, 'pickup_month'] = train_data['pickup_datetime'].dt.month"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "BkYmPyL11myY",
    "outputId": "098d0831-3039-48fe-802c-26abc8d4c379"
   },
   "outputs": [],
   "source": [
    "# Feature Extraction\n",
    "# Decomposing timestamp for test data into date and time\n",
    "\n",
    "test_data['pickup_datetime'] = pd.to_datetime(test_data.pickup_datetime)\n",
    "test_data.loc[:, 'pickup_date'] = test_data['pickup_datetime'].dt.date\n",
    "test_data.loc[:, 'pickup_time'] = test_data['pickup_datetime'].dt.time\n",
    "test_data.loc[:, 'pickup_weekday'] = test_data['pickup_datetime'].dt.weekday\n",
    "test_data.loc[:, 'pickup_weekofyear'] = test_data['pickup_datetime'].dt.weekofyear\n",
    "test_data.loc[:, 'pickup_hour'] = test_data['pickup_datetime'].dt.hour\n",
    "test_data.loc[:, 'pickup_minute'] = test_data['pickup_datetime'].dt.minute\n",
    "test_data.loc[:, 'pickup_dt'] = (test_data['pickup_datetime'] - test_data['pickup_datetime'].min()).dt.total_seconds()\n",
    "test_data.loc[:, 'pickup_week_hour'] = test_data['pickup_weekday'] * 24 + test_data['pickup_hour']\n",
    "test_data.loc[:, 'pickup_dayofyear'] = test_data['pickup_datetime'].dt.dayofyear\n",
    "test_data.loc[:, 'pickup_month'] = test_data['pickup_datetime'].dt.month"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "xcRfHI5iUtSV"
   },
   "outputs": [],
   "source": [
    "# Feature Extraction\n",
    "# Identifying holidays for train data from USFederalHolidayCalendar()\n",
    "\n",
    "calendar = USFederalHolidayCalendar()\n",
    "holidays = calendar.holidays()\n",
    "\n",
    "train_data['pickup_holiday'] = pd.to_datetime(train_data.pickup_datetime.dt.date).isin(holidays)\n",
    "train_data['pickup_holiday'] = train_data.pickup_holiday.map(lambda x: 1 if x == True else 0)\n",
    "train_data['pickup_near_holiday'] = (pd.to_datetime(train_data.pickup_datetime.dt.date).isin(holidays + timedelta(days=1)) | pd.to_datetime(train_data.pickup_datetime.dt.date).isin(holidays - timedelta(days=1)))\n",
    "train_data['pickup_near_holiday'] = train_data.pickup_near_holiday.map(lambda x: 1 if x == True else 0)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "F98i6M560_Dv"
   },
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "F9shqPDF0s0h"
   },
   "outputs": [],
   "source": [
    "# Identifying holidays for test data from USFederalHolidayCalendar()\n",
    "\n",
    "test_data['pickup_holiday'] = pd.to_datetime(test_data.pickup_datetime.dt.date).isin(holidays)\n",
    "test_data['pickup_holiday'] = test_data.pickup_holiday.map(lambda x: 1 if x == True else 0)\n",
    "test_data['pickup_near_holiday'] = (pd.to_datetime(test_data.pickup_datetime.dt.date).isin(holidays + timedelta(days=1)) | pd.to_datetime(test_data.pickup_datetime.dt.date).isin(holidays - timedelta(days=1)))\n",
    "test_data['pickup_near_holiday'] = test_data.pickup_near_holiday.map(lambda x: 1 if x == True else 0)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "GoIXOqfVNpiK"
   },
   "source": [
    "### COMPUTE DISTANCES (Using Longitudes & Latitudes)\n",
    "\n",
    "In order of increasing importance/accuracy:\n",
    "(Haversine Distance = Bearing Distance) < Manhattan Distance < OSRM Dataset Distance < Google Distance API"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "QKFnD9t4LyYd"
   },
   "outputs": [],
   "source": [
    "# Compute Distances using Longitudes & Latitudes \n",
    "\n",
    "# 1 Haversine Distance\n",
    "def haversine_distance(lat1, lng1, lat2, lng2):\n",
    "    lat1, lng1, lat2, lng2 = map(np.radians, (lat1, lng1, lat2, lng2))\n",
    "    AVG_EARTH_RADIUS = 6371 \n",
    "    lat = lat2 - lat1\n",
    "    lng = lng2 - lng1\n",
    "    d = np.sin(lat * 0.5) ** 2 + np.cos(lat1) * np.cos(lat2) * np.sin(lng * 0.5) ** 2\n",
    "    h = 2 * AVG_EARTH_RADIUS * np.arcsin(np.sqrt(d))\n",
    "    return h\n",
    "\n",
    "# 2 Bearing Distance \n",
    "def bearing_direction(lat1, lng1, lat2, lng2):\n",
    "    AVG_EARTH_RADIUS = 6371\n",
    "    lng_delta_rad = np.radians(lng2 - lng1)\n",
    "    lat1, lng1, lat2, lng2 = map(np.radians, (lat1, lng1, lat2, lng2))\n",
    "    y = np.sin(lng_delta_rad) * np.cos(lat2)\n",
    "    x = np.cos(lat1) * np.sin(lat2) - np.sin(lat1) * np.cos(lat2) * np.cos(lng_delta_rad)\n",
    "    return np.degrees(np.arctan2(y, x))\n",
    "\n",
    "# 3 Manhattan Distance\n",
    "def manhattan_distance(lat1, lng1, lat2, lng2):\n",
    "    a = haversine_distance(lat1, lng1, lat1, lng2)\n",
    "    b = haversine_distance(lat1, lng1, lat2, lng1)\n",
    "    return a + b\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "l6SpIPvvRbAM"
   },
   "outputs": [],
   "source": [
    "train_data.loc[:, 'direction'] = bearing_direction(train_data['pickup_latitude'].values, train_data['pickup_longitude'].values, train_data['dropoff_latitude'].values, train_data['dropoff_longitude'].values)\n",
    "train_data.loc[:, 'distance_haversine'] = haversine_distance(train_data['pickup_latitude'].values, train_data['pickup_longitude'].values, train_data['dropoff_latitude'].values, train_data['dropoff_longitude'].values)\n",
    "train_data.loc[:, 'distance_manhattan'] = manhattan_distance(train_data['pickup_latitude'].values, train_data['pickup_longitude'].values, train_data['dropoff_latitude'].values, train_data['dropoff_longitude'].values)\n",
    "train_data.loc[:, 'center_latitude'] = (train_data['pickup_latitude'].values + train_data['dropoff_latitude'].values) / 2\n",
    "train_data.loc[:, 'center_longitude'] = (train_data['pickup_longitude'].values + train_data['dropoff_longitude'].values) / 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "SlkxIHAF2CYz"
   },
   "outputs": [],
   "source": [
    "test_data.loc[:, 'direction'] = bearing_direction(test_data['pickup_latitude'].values, test_data['pickup_longitude'].values, test_data['dropoff_latitude'].values, test_data['dropoff_longitude'].values)\n",
    "test_data.loc[:, 'distance_haversine'] = haversine_distance(test_data['pickup_latitude'].values, test_data['pickup_longitude'].values, test_data['dropoff_latitude'].values, test_data['dropoff_longitude'].values)\n",
    "test_data.loc[:, 'distance_manhattan'] = manhattan_distance(test_data['pickup_latitude'].values, test_data['pickup_longitude'].values, test_data['dropoff_latitude'].values, test_data['dropoff_longitude'].values)\n",
    "test_data.loc[:, 'center_latitude'] = (test_data['pickup_latitude'].values + test_data['dropoff_latitude'].values) / 2\n",
    "test_data.loc[:, 'center_longitude'] = (test_data['pickup_longitude'].values + test_data['dropoff_longitude'].values) / 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 234
    },
    "id": "qLMHZKkz5U65",
    "outputId": "0bf3aac5-b08e-47a7-d82d-be01d33e47ae"
   },
   "outputs": [],
   "source": [
    "google_distance_dataset = pd.read_csv('https://firebasestorage.googleapis.com/v0/b/pizzarush-40b2f.appspot.com/o/train_with_distance848k.csv?alt=media&token=48501761-2b33-4508-9589-4ceb2eabe33d')\n",
    "google_distance_dataset.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "uiLNlbm5uZej"
   },
   "source": [
    "**Google Distance API**\n",
    "\n",
    "The distance between each longitude and latitude of every record is calculated here using the Google Maps Distance Matrix API.\n",
    "The code has been commented here to avoid running the code snippet again due to request restrictions of the *free tier API Key*, after scraping the distance for all of the 1.5 million datapoints."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "mVDtN8QcO7x_"
   },
   "outputs": [],
   "source": [
    "# # 4 Google Distance API\n",
    "\n",
    "# import requests\n",
    "\n",
    "# MAPS_KEY = 'AIzaSyB-f9LXy03l1HgMOG9LAXuI1ZXd0eP1pxw'\n",
    "\n",
    "# pickup_longitude = train_data['pickup_longitude'][0]\n",
    "# pickup_latitude = train_data['pickup_latitude'][0]\n",
    "\n",
    "# dropoff_longitude = train_data['dropoff_longitude'][0]\n",
    "# dropoff_latitude = train_data['dropoff_latitude'][0]\n",
    "\n",
    "# origin_str = f'{pickup_latitude},{pickup_longitude}'\n",
    "# destination_str = f'{dropoff_latitude},{dropoff_longitude}'\n",
    "\n",
    "# url ='https://maps.googleapis.com/maps/api/distancematrix/json?'\n",
    "\n",
    "# response=requests.get(url + 'origins=' + origin_str +\n",
    "#                    '&destinations=' + destination_str +\n",
    "#                    '&key=' + MAPS_KEY)\n",
    "\n",
    "# print(response.json())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "5kpuGeVMPM4K"
   },
   "outputs": [],
   "source": [
    "# # Make API calls in batches of 1000 train instance records \n",
    "\n",
    "# start = 1 \n",
    "# end = 1000\n",
    "\n",
    "# for i in range(start, end+1):        \n",
    "#     pickup_longitude = df['pickup_longitude'][i]\n",
    "#     pickup_latitude = df['pickup_latitude'][i]\n",
    "\n",
    "#     dropoff_longitude = df['dropoff_longitude'][i]\n",
    "#     dropoff_latitude = df['dropoff_latitude'][i]\n",
    "\n",
    "#     origin_str = f'{pickup_latitude},{pickup_longitude}'\n",
    "#     destination_str = f'{dropoff_latitude},{dropoff_longitude}'\n",
    "\n",
    "#     url ='https://maps.googleapis.com/maps/api/distancematrix/json?'\n",
    "    \n",
    "#     try:\n",
    "#         response=requests.get(url + 'origins=' + origin_str +\n",
    "#                        '&destinations=' + destination_str +\n",
    "#                        '&key=' + MAPS_KEY)\n",
    "        \n",
    "#         distance = response.json()['rows'][0]['elements'][0]['distance']['value']\n",
    "#     except:\n",
    "#         distance = None\n",
    "    \n",
    "#     print(f'distance... {i} = {distance}')\n",
    "#     df['google_distance'][i] = distance"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "JmYQLp6e6ak3",
    "outputId": "8b13b24d-70f1-41e8-d859-9b51b6151fe9"
   },
   "outputs": [],
   "source": [
    "google_distance = google_distance_dataset[:848000]['google_distance']\n",
    "print(google_distance)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 283
    },
    "id": "quEvYaZt6xg1",
    "outputId": "00fb2ee4-0a69-4923-9acc-313acc9da287"
   },
   "outputs": [],
   "source": [
    "plt.figure()\n",
    "\n",
    "x = list(range(848000))\n",
    "y1 = train_data[:848000]['direction']\n",
    "y2 = train_data[:848000]['distance_haversine']\n",
    "y3 = train_data[:848000]['distance_manhattan']\n",
    "y4 = google_distance\n",
    "\n",
    "plt.plot(x,y1)\n",
    "plt.plot(x,y2)\n",
    "plt.plot(x, y3)\n",
    "plt.plot(x, y4)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 295
    },
    "id": "O1vxV_Ej9aaw",
    "outputId": "83c7a76d-bdf3-4088-abe4-ae2fbeec3f10"
   },
   "outputs": [],
   "source": [
    "train_data[['direction', 'distance_haversine', 'distance_manhattan']].plot()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 301
    },
    "id": "Yfr-7ZKBU9bA",
    "outputId": "0cddd4c7-4dad-4f22-8ef4-c307f0c2fc49"
   },
   "outputs": [],
   "source": [
    "train_data.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "MDixSw8Fdlvi"
   },
   "source": [
    "# UNI-VARIATE ANALYSIS"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "ZgjxUf1XYA3l",
    "outputId": "83011914-245e-48a9-b062-2461ed1e96d4"
   },
   "outputs": [],
   "source": [
    "# Finding number of trips with 0 travelled distance which come out to be 5894.\n",
    "\n",
    "train_data[(train_data['distance_haversine']==0)].count()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 1000
    },
    "id": "FP4QHZXId2yQ",
    "outputId": "53793f43-79e6-4ceb-a92f-c35036d26a1e"
   },
   "outputs": [],
   "source": [
    "# Numeric Variables: Visualising using Box Plot, Dist Plot, Violin Plot \n",
    "# pickup_longitude, pickup_latitude, dropoff_longitude, dropoff_latitude, trip_duration\n",
    "\n",
    "numeric_data = pd.DataFrame(train_data[[\n",
    "                                        'pickup_longitude',\t\n",
    "                                        'pickup_latitude',\t\n",
    "                                        'dropoff_longitude',\t\n",
    "                                        'dropoff_latitude',\t\n",
    "                                        'trip_duration',\t\n",
    "                                        ]])\n",
    "\n",
    "f, axes = plt.subplots(5, 3, figsize=(18, 30))\n",
    "colors = [\"r\", \"g\", \"b\", \"m\", \"c\"]\n",
    "\n",
    "count = 0\n",
    "for var in numeric_data:\n",
    "    sns.boxplot(numeric_data[var], orient = \"h\", color = colors[count], ax = axes[count,0])\n",
    "    sns.distplot(numeric_data[var], color = colors[count], ax = axes[count,1])\n",
    "    sns.violinplot(numeric_data[var], color = colors[count], ax = axes[count,2])\n",
    "    count += 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 515
    },
    "id": "lcqgIn4Zdn38",
    "outputId": "a3f8687d-9d0c-49a6-97a9-f247c568d586"
   },
   "outputs": [],
   "source": [
    "# Categorical Variables: Visualising using Violin Plots\n",
    "# vendor_id, passenger_count, store_and_fwd_flag\n",
    "\n",
    "f, axes = plt.subplots(3, 1, figsize=(16,8))\n",
    "\n",
    "sns.violinplot(x = 'vendor_id', y = 'trip_duration', data = train_data, ax= axes[0])\n",
    "sns.violinplot(x = 'passenger_count', y = 'trip_duration', data = train_data, ax= axes[1])\n",
    "sns.violinplot(x = 'store_and_fwd_flag', y = 'trip_duration', data = train_data, ax= axes[2])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 420
    },
    "id": "o97dP8EbjWUT",
    "outputId": "52ba2866-9699-474f-847a-00acd0369947"
   },
   "outputs": [],
   "source": [
    "# Store and Forward Flag Distribution\n",
    "\n",
    "sns.distplot(train_data['store_and_fwd_flag'],kde=False)\n",
    "plt.title('Distribution of Store and Forward flag Count')\n",
    "plt.show()\n",
    "\n",
    "train_data['store_and_fwd_flag'].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 366
    },
    "id": "B1LUkxVAdn6f",
    "outputId": "17760c0e-a6bf-4e3f-8671-4ddd06e70ea8"
   },
   "outputs": [],
   "source": [
    "# Passenger Count Distribution\n",
    "\n",
    "sns.distplot(train_data['passenger_count'],kde=False)\n",
    "plt.title('Distribution of Passenger Count')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 441
    },
    "id": "QKz4mwuNY6M7",
    "outputId": "9eef8e54-9060-4059-8c38-0e83de4342de"
   },
   "outputs": [],
   "source": [
    "# Count of occurences of each pickup day, pickup hour and pickup month\n",
    "\n",
    "fig, sub = plt.subplots(1,3,figsize=(25,6))\n",
    "counter = 0\n",
    "\n",
    "for feat, subplot in zip([\"pickup_weekday\",\"pickup_hour\", \"pickup_month\"], sub.flatten()):\n",
    "    \n",
    "    if counter < 3:\n",
    "        sns.barplot(x=train_data[feat].value_counts().index, y = train_data[feat].value_counts().values, ax= subplot, palette=\"CMRmap\")\n",
    "        subplot.grid()\n",
    "        subplot.set_title(\"Train set {}\".format(feat))\n",
    "    \n",
    "    counter+=1\n",
    "    \n",
    "fig.tight_layout()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 441
    },
    "id": "rjiDzWlmLUOI",
    "outputId": "cc249c33-27f3-4437-e1e8-c0e52e9f540d"
   },
   "outputs": [],
   "source": [
    "# Count of occurences of each dropoff weekday and dropoff hour \n",
    "# dropoff month has been excluded since the pickup month and dropoff month would be in the same month\n",
    "\n",
    "fig, sub = plt.subplots(1,2,figsize=(25,6))\n",
    "counter = 0\n",
    "\n",
    "for feat, subplot in zip([\"dropoff_weekday\",\"dropoff_hour\"], sub.flatten()):\n",
    "    \n",
    "    if counter < 2:\n",
    "        sns.barplot(x=train_data[feat].value_counts().index, y = train_data[feat].value_counts().values, ax= subplot, palette=\"CMRmap\")\n",
    "        subplot.grid()\n",
    "        subplot.set_title(\"Train set {}\".format(feat))\n",
    "    \n",
    "    counter+=1\n",
    "    \n",
    "fig.tight_layout()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 296
    },
    "id": "rLmZkbskFMIH",
    "outputId": "8a1addbd-15b0-47ea-d765-9fe9ff3dfc06"
   },
   "outputs": [],
   "source": [
    "# Histogram to visualise counts for each Trip Duration \n",
    "\n",
    "graph = sns.histplot(train_data[\"trip_duration\"], bins = 200)\n",
    "graph.set(xlabel='Trip duration in seconds', ylabel='Trip Count')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 314
    },
    "id": "n3Pp3XDoFMKy",
    "outputId": "7795f9ee-810f-4f90-e9cf-84c5e4d63816"
   },
   "outputs": [],
   "source": [
    "# Histogram to visualise counts for each Trip Duration, with logarithmic normalisation/scaling\n",
    "\n",
    "train_data['trip_duration_normalised'] = np.log(train_data['trip_duration'].values + 1)\n",
    "graph = sns.histplot(train_data[\"trip_duration_normalised\"], bins = 200)\n",
    "graph.set(xlabel='Trip duration in seconds (Normalised)', ylabel='Trip Count')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 542
    },
    "id": "ZcRQispNFMNa",
    "outputId": "8fcd6b17-a293-47a2-c346-55f79208493d"
   },
   "outputs": [],
   "source": [
    "# Visualise Trips Count by Pick Up date\n",
    "\n",
    "fig = go.Figure()\n",
    "pickup_date_count = pd.DataFrame(train_data.groupby('pickup_date').count()[['id']])\n",
    "pickup_date_count.index.name = 'Pickup Dates'\n",
    "pickup_date_count.reset_index(inplace=True)\n",
    "\n",
    "\n",
    "fig.add_trace(go.Scatter(x=pickup_date_count['Pickup Dates'], y=pickup_date_count['id'],\n",
    "                    mode='lines+markers',\n",
    "                    name='lines+markers'))\n",
    "\n",
    "fig.update_xaxes(title_text=\"Pickup Months\")\n",
    "fig.update_yaxes(title_text=\"Count of Trips\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "5Qc6zmCvF0JE"
   },
   "source": [
    "# BI-VARIATE ANALYSIS"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 1000
    },
    "id": "5N1w8RtNGG7_",
    "outputId": "ed37c69d-e348-4b46-a7c3-1ac577fa3b64"
   },
   "outputs": [],
   "source": [
    "# Heap Map to visualise Feature Correlation\n",
    "\n",
    "f, axes = plt.subplots(1, 1, figsize=(20, 20))\n",
    "sns.heatmap(train_data.corr(), vmin = -1, vmax = 1, linewidths = 1,\n",
    "           annot = True, fmt = \".2f\", annot_kws = {\"size\": 18}, cmap = \"RdBu\")\n",
    "f.set_figwidth(30)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 440
    },
    "id": "Ffpi3P4W2jGP",
    "outputId": "f60afede-6869-44a8-f7ce-f352311f96a5"
   },
   "outputs": [],
   "source": [
    "# Average Trip Duration VS Store-and-Forward Flag and Average Trip Duration VS Vendor ID\n",
    "\n",
    "fig, ax = plt.subplots(ncols=2, figsize=(12,4))\n",
    "\n",
    "# Average Trip Duration VS Store-and-Forward Flag\n",
    "\n",
    "store_and_fwd_flag_df = train_data.groupby('store_and_fwd_flag')['trip_duration'].mean()\n",
    "plt.ylabel('Time in Seconds')\n",
    "sns.barplot(store_and_fwd_flag_df.index, store_and_fwd_flag_df.values, ax=ax[0])\n",
    "plt.setp(ax[0], ylabel='Time in Seconds')\n",
    "\n",
    "# Average Trip Duration VS Vendor ID\n",
    "\n",
    "vendor_id_df = train_data.groupby('vendor_id')['trip_duration'].mean()\n",
    "plt.ylabel('Time in Seconds')\n",
    "sns.barplot(vendor_id_df.index, vendor_id_df.values, ax=ax[1])\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 368
    },
    "id": "GjJhcLdmIsE8",
    "outputId": "9725b9fe-cfb6-4fad-984f-99cffa1fd4e7"
   },
   "outputs": [],
   "source": [
    "# Passenger Count VS Trip Duration time in seconds\n",
    "\n",
    "passenger_count_df = train_data.groupby('passenger_count')['trip_duration'].mean()\n",
    "plt.ylabel('Time in Seconds')\n",
    "sns.barplot(passenger_count_df.index, passenger_count_df.values)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 368
    },
    "id": "JiZ4QSryMVNw",
    "outputId": "f16700d0-e544-443c-85b0-236e8765131c"
   },
   "outputs": [],
   "source": [
    "# Passenger Count VS Trip Distance\n",
    "\n",
    "passenger_count_distance_df = train_data.groupby('passenger_count')['distance_manhattan'].mean()\n",
    "plt.ylabel('Trip Distance in KM')\n",
    "sns.barplot(passenger_count_distance_df.index, passenger_count_distance_df.values)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 402
    },
    "id": "CGYe418mhMh9",
    "outputId": "1f0020f7-e397-47da-a983-cc227eabd0b6"
   },
   "outputs": [],
   "source": [
    "# Relationship b/w Vendor ID and Trip Duration\n",
    "\n",
    "sns.catplot(x=\"vendor_id\", y=\"trip_duration\",kind=\"strip\",data=train_data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 400
    },
    "id": "TYUNlJtah1KI",
    "outputId": "3eea6505-9692-4cef-beab-ad5228738159"
   },
   "outputs": [],
   "source": [
    "# Relationship b/w Trip Duration and Time of Day\n",
    "\n",
    "ax=sns.catplot(x=\"pickup_hour\", y=\"trip_duration\",kind=\"bar\",data=train_data)\n",
    "plt.title('Distribution of pickup hours')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 490
    },
    "id": "ftWHCT8JhVRY",
    "outputId": "511606d1-53c0-4cbe-d2fc-e96fab733793"
   },
   "outputs": [],
   "source": [
    "# Relationship b/w Trip Duration and Day of Week\n",
    "\n",
    "sns.catplot(x=\"pickup_weekday\",y=\"trip_duration\",kind=\"bar\",data=train_data,height=6,aspect=1)\n",
    "plt.title('The Average Trip Duration per PickUp Day of the Week')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 294
    },
    "id": "CPOuYMk1IsHq",
    "outputId": "51bc835b-1610-4653-97e6-83591084696b"
   },
   "outputs": [],
   "source": [
    "# Visualise PickUp Locations Density by plotting corresponding Longitudes/Latitudes\n",
    "\n",
    "fig, ax = plt.subplots(ncols=1, sharex=True, sharey=True)\n",
    "ax.scatter(train_data['pickup_longitude'].values[:625134], train_data['pickup_latitude'].values[:625134],\n",
    "              color='red', s=1, label='train', alpha=0.5)\n",
    "\n",
    "ax.title.set_text('Train coordinates')\n",
    "ax.set_xlabel('longitude')\n",
    "ax.set_ylabel('latitude')\n",
    "plt.xlim(-74.05, -73.76)\n",
    "plt.ylim(40.60, 40.90)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 606
    },
    "id": "YGyyJ7TJIsKl",
    "outputId": "3167eb46-bf46-46f1-b035-a77fcf797e5b"
   },
   "outputs": [],
   "source": [
    "# Visualise main Neighbourhoods of New York City \n",
    "\n",
    "from sklearn.cluster import KMeans\n",
    "\n",
    "coordinates_df = pd.DataFrame()\n",
    "coordinates_df['all_longitude'] = list(train_data.pickup_longitude) + list(train_data.dropoff_longitude)\n",
    "coordinates_df['all_latitude'] = list(train_data.pickup_latitude) + list(train_data.dropoff_latitude)\n",
    "\n",
    "kmeans = KMeans(n_clusters=20, random_state=2, n_init = 10).fit(coordinates_df)\n",
    "coordinates_df['kmeans_label'] = kmeans.labels_\n",
    "\n",
    "coordinates_df = coordinates_df.sample(200000)\n",
    "plt.figure(figsize = (10,10))\n",
    "for label in coordinates_df.kmeans_label.unique():\n",
    "    plt.plot(coordinates_df.all_longitude[coordinates_df.kmeans_label == label],coordinates_df.all_latitude[coordinates_df.kmeans_label == label],'.', alpha = 0.5, markersize = 0.5)\n",
    "\n",
    "plt.title('Neighborhoods of New York City')\n",
    "plt.xlim(-74.05, -73.76)\n",
    "plt.ylim(40.60, 40.90)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 521
    },
    "id": "Pz-HNO87Ycur",
    "outputId": "7226921d-3aa6-4757-db06-70defa386045"
   },
   "outputs": [],
   "source": [
    "# Use folium to visualize pickup and dropoff points in New York\n",
    "\n",
    "import folium\n",
    "f = folium.Figure(width=1500, height=500)\n",
    "mapa = folium.Map(location = (40.7679, -73.9822), zoom_start=11).add_to(f)\n",
    "\n",
    "for index, row in train_data.sample(1000).iterrows():\n",
    "    folium.Marker([row[\"pickup_latitude\"], row[\"pickup_longitude\"]], icon=folium.Icon(color=\"blue\")).add_to(mapa)\n",
    "    folium.Marker([row[\"dropoff_latitude\"], row[\"dropoff_longitude\"]], icon=folium.Icon(color=\"red\")).add_to(mapa)\n",
    "\n",
    "\n",
    "display(mapa)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "Bz0-4kI_X91z"
   },
   "source": [
    "## Addtional Dataset: OSRM"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "Kzbc9I1iFoah"
   },
   "outputs": [],
   "source": [
    "# OSRM Dataset\n",
    "fr1 = pd.read_csv('https://firebasestorage.googleapis.com/v0/b/craftrip-594f5.appspot.com/o/fastest_routes_train_part_1.csv?alt=media&token=9b1832f8-2654-4835-9621-265e4f1f4c58', usecols=['id', 'total_distance', 'total_travel_time',  'number_of_steps', ])\n",
    "fr2 = pd.read_csv('https://firebasestorage.googleapis.com/v0/b/craftrip-594f5.appspot.com/o/fastest_routes_train_part_2.csv?alt=media&token=26cc43e2-a68f-447c-9c9b-80255bb620d1', usecols=['id', 'total_distance', 'total_travel_time', 'number_of_steps'])\n",
    "test_street_info = pd.read_csv('https://firebasestorage.googleapis.com/v0/b/craftrip-594f5.appspot.com/o/fastest_routes_test.csv?alt=media&token=dd7ff2fe-ea90-46d9-8189-e0defbdf8955',\n",
    "                               usecols=['id', 'total_distance', 'total_travel_time', 'number_of_steps'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 197
    },
    "id": "Z2tbBOxLFt5_",
    "outputId": "0ff44b75-d6d8-4afb-bc73-4f0c93e70f80"
   },
   "outputs": [],
   "source": [
    "fr1.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 287
    },
    "id": "MRq92xG_GW_V",
    "outputId": "a07ebd78-2c0d-47e9-ee29-2a0b54a64829"
   },
   "outputs": [],
   "source": [
    "fr2.describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 197
    },
    "id": "Q6Xn4dLhGcpF",
    "outputId": "01d7b3c9-6386-4674-8c63-b9ca22c92e26"
   },
   "outputs": [],
   "source": [
    "test_street_info.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "PQIr1EeLYO4C"
   },
   "source": [
    "OSRM: BIVARIATE ANALYSIS"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "on-Zzh_iMm-B"
   },
   "outputs": [],
   "source": [
    "# Concatenate both the OSRM datasets\n",
    "\n",
    "train_street_info = pd.concat((fr1, fr2))\n",
    "numeric_data = train_street_info[['total_distance', 'total_travel_time', 'number_of_steps']]\n",
    "# sns.pairplot(data = numeric_data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "9DM3-bCtFweD"
   },
   "outputs": [],
   "source": [
    "# Merge the train and test dataset with the OSRM Dataset\n",
    "train_data = train_data.merge(train_street_info, how='left', on='id')\n",
    "test_data = test_data.merge(test_street_info, how='left', on='id')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "qTatyY5Hyk4C"
   },
   "source": [
    "## Additional Dataset: Weather Dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 197
    },
    "id": "6wP3umiLyslq",
    "outputId": "b86cc86b-0fa2-46c0-b10b-0b4c04e62743"
   },
   "outputs": [],
   "source": [
    "# Read in the weather dataset\n",
    "weather_data = pd.read_csv('https://firebasestorage.googleapis.com/v0/b/pizzarush-40b2f.appspot.com/o/weather_data_nyc_centralpark_2016(1).csv?alt=media&token=a6fb7ce8-b4bc-4bc5-ac07-132a078bea85') # Replace with firebase link when available\n",
    "weather_data.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 287
    },
    "id": "3EvjAe9ty4r-",
    "outputId": "49b5a55f-32ee-44d1-d445-1ad9fcf902dd"
   },
   "outputs": [],
   "source": [
    "weather_data.describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "kx4DF8tJy8vb",
    "outputId": "9d0d0e66-65be-467e-aa4a-05f5b911687c"
   },
   "outputs": [],
   "source": [
    "weather_data.dtypes"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "VWRelD68Ylo1"
   },
   "source": [
    "WEATHER DATASET: FORMAT DATA"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "6ynJrDYG0QD8"
   },
   "outputs": [],
   "source": [
    "# Convert the data values to float which could be plotted and convert the string values containing 'T' for 'Trace' to 0.00\n",
    "\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline\n",
    "\n",
    "weather_data.loc[weather_data['precipitation']== 'T', 'precipitation'] = '0.00'\n",
    "weather_data['precipitation'] = (weather_data['precipitation']).astype(float)\n",
    "\n",
    "weather_data.loc[weather_data['snow fall']== 'T', 'snow fall'] = '0.00'\n",
    "weather_data['snow fall'] = (weather_data['snow fall']).astype(float)\n",
    "\n",
    "weather_data.loc[weather_data['snow depth']== 'T', 'snow depth'] = '0.00'\n",
    "weather_data['snow depth'] = (weather_data['snow depth']).astype(float)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "ojn-uLmnW8se"
   },
   "outputs": [],
   "source": [
    "weather_data['date'] = pd.to_datetime(weather_data.date)\n",
    "weather_data['weather_dayofyear']= weather_data.date.dt.dayofyear"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "4yl6dV_SYfdG"
   },
   "source": [
    "WEATHER DATASET: UNIVARIATE ANALYSIS"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 542
    },
    "id": "TRyeA12NzGgR",
    "outputId": "f8b27854-8cec-46bb-cd78-a75df400d055"
   },
   "outputs": [],
   "source": [
    "# Plot min (yellow), avg (orange), and max (red) temperatures over time\n",
    "\n",
    "import plotly.express as px\n",
    "import plotly.graph_objects as go\n",
    "\n",
    "# plotly.graph_objects\n",
    "colors = ['yellow', 'orange', 'red']\n",
    "fig = go.Figure()\n",
    "fig.add_traces(go.Scatter(x=weather_data['date'], y = weather_data['minimum temperature'], mode = 'lines', line=dict(color=colors[0])))\n",
    "fig.add_traces(go.Scatter(x=weather_data['date'], y = weather_data['average temperature'], mode = 'lines', line=dict(color=colors[1])))\n",
    "fig.add_traces(go.Scatter(x=weather_data['date'], y = weather_data['maximum temperature'], mode = 'lines', line=dict(color=colors[2])))\n",
    "fig.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 542
    },
    "id": "F1Tv1FFn0maU",
    "outputId": "02a9da00-407c-4029-a6ce-f964628d266d"
   },
   "outputs": [],
   "source": [
    "# Plot precipitation, snow fall, and snow depth \n",
    "\n",
    "import plotly as py\n",
    "import plotly.graph_objs as go\n",
    "import plotly\n",
    "\n",
    "x_data = weather_data['date'].values\n",
    "y0_data = weather_data['precipitation']\n",
    "y1_data = weather_data['snow fall']\n",
    "y2_data = weather_data['snow depth']\n",
    "\n",
    "data0 = go.Scatter(\n",
    "    x = x_data,\n",
    "    y = y0_data,\n",
    "    mode = 'markers',\n",
    "    name = 'precipitation'\n",
    ")\n",
    "data1 = go.Scatter(\n",
    "    x = x_data,\n",
    "    y = y1_data,\n",
    "    mode = 'markers',\n",
    "    name = 'snow fall'\n",
    ")\n",
    "data2 = go.Scatter(\n",
    "    x = x_data,\n",
    "    y = y2_data,\n",
    "    mode = 'markers',\n",
    "    name = 'snow depth'\n",
    ")\n",
    "\n",
    "data = [data0, data1, data2]\n",
    "plotly.offline.iplot(data, filename='scatter-mode')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "MVGgrvnBz0P2"
   },
   "outputs": [],
   "source": [
    "# Merge train dataset with the weather dataset through date column\n",
    "train_data['date'] = train_data['pickup_datetime'].dt.date\n",
    "train_data['date'] = pd.to_datetime(train_data['date'])\n",
    "weather = weather_data[['date','minimum temperature', 'precipitation', 'snow fall', 'snow depth']]\n",
    "train_data = train_data.merge(weather, how='left', left_on='date', right_on='date')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "817Dc-u20FHG"
   },
   "outputs": [],
   "source": [
    "# Merge test dataset with the weather dataset through date column\n",
    "test_data['date'] = test_data['pickup_datetime'].dt.date\n",
    "test_data['date'] = pd.to_datetime(test_data['date'])\n",
    "test_data = test_data.merge(weather, how='left', left_on='date', right_on='date')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "ho6kwcG_kMOd"
   },
   "source": [
    "# FEATURE SELECTION"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "zhWwYe5TkREE"
   },
   "outputs": [],
   "source": [
    "# PCA - Reducing the dimentionality of the features\n",
    "\n",
    "from sklearn.decomposition import PCA\n",
    "\n",
    "coords = np.vstack((train_data[['pickup_latitude', 'pickup_longitude']].values,\n",
    "                    train_data[['dropoff_latitude', 'dropoff_longitude']].values,\n",
    "                    test_data[['pickup_latitude', 'pickup_longitude']].values,\n",
    "                    test_data[['dropoff_latitude', 'dropoff_longitude']].values))\n",
    "\n",
    "pca = PCA().fit(coords) # fit the PCA model according to the coordinate values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "y6k2uR9N_Dmx"
   },
   "outputs": [],
   "source": [
    "train_data['pickup_pca0'] = pca.transform(train_data[['pickup_latitude', 'pickup_longitude']])[:, 0] # Derive the PCA feature for Pickup Latitude in Train Dataset\n",
    "train_data['pickup_pca1'] = pca.transform(train_data[['pickup_latitude', 'pickup_longitude']])[:, 1] # Derive the PCA feature for Pickup Longitude in Train Dataset\n",
    "train_data['dropoff_pca0'] = pca.transform(train_data[['dropoff_latitude', 'dropoff_longitude']])[:, 0] # Derive the PCA feature for Dropoff Latitude in Train Dataset\n",
    "train_data['dropoff_pca1'] = pca.transform(train_data[['dropoff_latitude', 'dropoff_longitude']])[:, 1] # Derive the PCA feature for Dropoff Longitude in Train Dataset\n",
    "test_data['pickup_pca0'] = pca.transform(test_data[['pickup_latitude', 'pickup_longitude']])[:, 0] # Derive the PCA feature for Pickup Latitude in Test Dataset\n",
    "test_data['pickup_pca1'] = pca.transform(test_data[['pickup_latitude', 'pickup_longitude']])[:, 1] # Derive the PCA feature for Pickup Longitude in Test Dataset\n",
    "test_data['dropoff_pca0'] = pca.transform(test_data[['dropoff_latitude', 'dropoff_longitude']])[:, 0] # Derive the PCA feature for Dropoff Latitude in Test Dataset\n",
    "test_data['dropoff_pca1'] = pca.transform(test_data[['dropoff_latitude', 'dropoff_longitude']])[:, 1] # Derive the PCA feature for Dropoff Longitutde in Test Dataset\n",
    "\n",
    "train_data.loc[:, 'pca_manhattan'] = np.abs(train_data['dropoff_pca1'] - train_data['pickup_pca1']) + np.abs(train_data['dropoff_pca0'] - train_data['pickup_pca0']) # Calculate the Manhattan Distance with PCA Coordinates for Train Dataset\n",
    "test_data.loc[:, 'pca_manhattan'] = np.abs(test_data['dropoff_pca1'] - test_data['pickup_pca1']) + np.abs(test_data['dropoff_pca0'] - test_data['pickup_pca0']) #Calculate the Manhattan Distance with PCA Coordinated for Test Dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 278
    },
    "id": "82gwmaXcvyeq",
    "outputId": "b2415ba4-8e38-453c-ebbd-8d03ed5e7dd3"
   },
   "outputs": [],
   "source": [
    "# Visualising a PCA Feature - Pickup Longitude (pickup_pca1) from Train Dataset\n",
    "\n",
    "ax = plt.gca()\n",
    "\n",
    "train_data.reset_index().plot(kind='line', x='index', y='pickup_longitude',ax=ax)\n",
    "train_data.reset_index().plot(kind='line', x='index', y='pickup_pca1', color='red', ax=ax)\n",
    "\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "JYMvgiu_AkXx"
   },
   "source": [
    "# CLUSTERING"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "R2WDWWUs_BuN"
   },
   "outputs": [],
   "source": [
    "sample_ind = np.random.permutation(len(coords))[:500000]\n",
    "kmeans = MiniBatchKMeans(n_clusters=100, batch_size=10000).fit(coords[sample_ind])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "4TacdBJF-20n"
   },
   "outputs": [],
   "source": [
    "train_data.loc[:, 'pickup_cluster'] = kmeans.predict(train_data[['pickup_latitude', 'pickup_longitude']])\n",
    "train_data.loc[:, 'dropoff_cluster'] = kmeans.predict(train_data[['dropoff_latitude', 'dropoff_longitude']])\n",
    "test_data.loc[:, 'pickup_cluster'] = kmeans.predict(test_data[['pickup_latitude', 'pickup_longitude']])\n",
    "test_data.loc[:, 'dropoff_cluster'] = kmeans.predict(test_data[['dropoff_latitude', 'dropoff_longitude']])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "N3o4sgaOkSlP"
   },
   "source": [
    "# MODEL TRAINING"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "1TyplWnG0gDh",
    "outputId": "b56d7133-34de-4bfb-d4ea-2a7ff95d793f"
   },
   "outputs": [],
   "source": [
    "# Listing out the features that should be used for training the XGB Model.\n",
    "\n",
    "feature_names = list(train_data.columns)\n",
    "features_not_used = ['id', 'trip_duration_normalised', 'trip_duration', 'dropoff_datetime','dropoff_date','dropoff_hour',\n",
    "                           'dropoff_month','dropoff_time','dropoff_weekday', 'pickup_date', 'pickup_datetime', 'date','pickup_time','pickup_month']\n",
    "feature_names = [f for f in train_data.columns if f not in features_not_used]\n",
    "train_data[feature_names].count()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "vg26YJAd1CTr",
    "outputId": "5079a05b-57f7-4f90-b199-6fb4e4c6373c"
   },
   "outputs": [],
   "source": [
    "# Performing k-fold splits\n",
    "\n",
    "X = train_data[feature_names].values\n",
    "y = np.log(train_data['trip_duration'].values + 1)  \n",
    "kf = KFold(n_splits=10)\n",
    "kf.get_n_splits(X)\n",
    "\n",
    "print(kf)  \n",
    "\n",
    "KFold(n_splits=10, random_state=None, shuffle=False)\n",
    "for train_index, test_index in kf.split(X):\n",
    "    print(\"TRAIN:\", train_index, \"TEST:\", test_index)\n",
    "    X_train, X_test = X[train_index], X[test_index]\n",
    "    y_train, y_test = y[train_index], y[test_index]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "4DNUqn121GTl"
   },
   "outputs": [],
   "source": [
    "# Setting the XGB Parameters\n",
    "dtrain = xgb.DMatrix(X_train, label=y_train)\n",
    "dvalid = xgb.DMatrix(X_test, label=y_test)\n",
    "dtest = xgb.DMatrix(test_data[feature_names].values)\n",
    "watchlist = [(dtrain, 'train'), (dvalid, 'valid')]\n",
    "\n",
    "xgb_parameters = {'min_child_weight': 10, 'eta': 0.04, 'colsample_bytree': 0.8, 'max_depth': 15,\n",
    "            'subsample': 0.75, 'lambda': 2, 'nthread': -1, 'booster' : 'gbtree', 'silent': 1, 'gamma' : 0,\n",
    "            'eval_metric': 'rmse', 'objective': 'reg:linear'}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "FJNBqUQd1IMJ",
    "outputId": "67d27ea3-2ccc-4755-97b5-5d89af0df5d4"
   },
   "outputs": [],
   "source": [
    "# Training the model\n",
    "\n",
    "model = xgb.train(xgb_parameters, dtrain, 750, watchlist, early_stopping_rounds=250, maximize=False, verbose_eval=15)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "iPkIYam1kS_Z"
   },
   "source": [
    "# MODEL PREDICTION "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "pxL276WQAHnZ"
   },
   "outputs": [],
   "source": [
    "# Predicting the model on test data\n",
    "\n",
    "y_test = model.predict(dtest)\n",
    "y_pred = model.predict(dvalid)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 460
    },
    "id": "WjEbr77XkTHr",
    "outputId": "3f0ff916-7933-4cf5-a303-a8a1cd17788d"
   },
   "outputs": [],
   "source": [
    "# Check the number of rows of test data to be matching with predicted data.\n",
    "if test_data.shape[0] == y_test.shape[0]:\n",
    "  print('Number of rows for test data and predicted data are same')  \n",
    "else:\n",
    "  print('Error in Test Data Format')\n",
    "\n",
    "# Convert predicted dataframe into a csv file for submission\n",
    "test_data['trip_duration'] = np.exp(y_test) - 1\n",
    "test_data[['id', 'trip_duration']].to_csv('final_submission.csv', index=False)\n",
    "\n",
    "# Plot validation and test prediction mean\n",
    "print('Valid prediction mean: %.3f' % y_pred.mean())\n",
    "print('Test prediction mean: %.3f' % y_test.mean())\n",
    "fig, ax = plt.subplots(nrows=2, sharex=True, sharey=True)\n",
    "sns.distplot(y_pred, ax=ax[0], color='red', label='validation prediction')\n",
    "sns.distplot(y_test, ax=ax[1], color='blue', label='test prediction')\n",
    "ax[0].legend(loc=0)\n",
    "ax[1].legend(loc=0)\n",
    "plt.show()"
   ]
  }
 ],
 "metadata": {
  "colab": {
   "collapsed_sections": [],
   "name": "Main Notebook.ipynb",
   "provenance": []
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}