Intermediate_ML_Missing_values.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "Intermediate ML:Missing values.ipynb",
      "provenance": [],
      "collapsed_sections": [],
      "authorship_tag": "ABX9TyOKA0gp/mA6khaKY8EvjSQy",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/divyansh1195/Kaggle-ML-Exercise-Notebooks/blob/master/Intermediate_ML_Missing_values.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "5-VSWxQkOwz9",
        "colab_type": "text"
      },
      "source": [
        "**Intermediate ML:Missing values**"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "qX6KbCrJO4Vt",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import pandas as pd\n",
        "\n",
        "# Read the data\n",
        "train_data = pd.read_csv('train.csv', index_col='Id')\n",
        "test_data = pd.read_csv('test.csv', index_col='Id')\n",
        "#print(train_data)\n",
        "#print(test_data)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "keZ00G2hPV3F",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 493
        },
        "outputId": "92370de5-3205-42b5-efc0-a0db69d01770"
      },
      "source": [
        "# Remove rows with missing target values\n",
        "train_data.dropna(axis=0, subset=['SalePrice'], inplace=True)\n",
        "#axis=0 or, 'index' drops rows with missing values, axis=1 or, 'columns'drops columns which contain missing value\n",
        "#subset label along which you wish to drop\n",
        "\n",
        "#define y after removing missing values\n",
        "y = train_data.SalePrice\n",
        "print(y)\n",
        "\n",
        "#separate target from predictors\n",
        "#here, separate SalePrice from reamining columns\n",
        "train_data.drop(['SalePrice'], axis=1, inplace=True)\n",
        "#SalePrice column removed from X_full\n",
        "\n",
        "# To keep things simple, we'll use only numerical predictors\n",
        "X = train_data.select_dtypes(exclude=['object'])\n",
        "test = test_data.select_dtypes(exclude=['object'])\n",
        "\n",
        "print(X)\n",
        "#print(test)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Id\n",
            "1       208500\n",
            "2       181500\n",
            "3       223500\n",
            "4       140000\n",
            "5       250000\n",
            "         ...  \n",
            "1456    175000\n",
            "1457    210000\n",
            "1458    266500\n",
            "1459    142125\n",
            "1460    147500\n",
            "Name: SalePrice, Length: 1460, dtype: int64\n",
            "      MSSubClass  LotFrontage  LotArea  ...  MiscVal  MoSold  YrSold\n",
            "Id                                      ...                         \n",
            "1             60         65.0     8450  ...        0       2    2008\n",
            "2             20         80.0     9600  ...        0       5    2007\n",
            "3             60         68.0    11250  ...        0       9    2008\n",
            "4             70         60.0     9550  ...        0       2    2006\n",
            "5             60         84.0    14260  ...        0      12    2008\n",
            "...          ...          ...      ...  ...      ...     ...     ...\n",
            "1456          60         62.0     7917  ...        0       8    2007\n",
            "1457          20         85.0    13175  ...        0       2    2010\n",
            "1458          70         66.0     9042  ...     2500       5    2010\n",
            "1459          20         68.0     9717  ...        0       4    2010\n",
            "1460          20         75.0     9937  ...        0       6    2008\n",
            "\n",
            "[1460 rows x 36 columns]\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "DYawWBkHZ-ZB",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# Break off test set from training data\n",
        "from sklearn.model_selection import train_test_split\n",
        "X_train, X_test, y_train, y_test = train_test_split(X, y, train_size=0.8, test_size=0.2,\n",
        "                                                      random_state=0)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "A1v9Kp8MSVaI",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 162
        },
        "outputId": "bc20c777-299b-450b-f991-4a7c74cfb946"
      },
      "source": [
        "X_train.head(2)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>MSSubClass</th>\n",
              "      <th>LotFrontage</th>\n",
              "      <th>LotArea</th>\n",
              "      <th>OverallQual</th>\n",
              "      <th>OverallCond</th>\n",
              "      <th>YearBuilt</th>\n",
              "      <th>YearRemodAdd</th>\n",
              "      <th>MasVnrArea</th>\n",
              "      <th>BsmtFinSF1</th>\n",
              "      <th>BsmtFinSF2</th>\n",
              "      <th>BsmtUnfSF</th>\n",
              "      <th>TotalBsmtSF</th>\n",
              "      <th>1stFlrSF</th>\n",
              "      <th>2ndFlrSF</th>\n",
              "      <th>LowQualFinSF</th>\n",
              "      <th>GrLivArea</th>\n",
              "      <th>BsmtFullBath</th>\n",
              "      <th>BsmtHalfBath</th>\n",
              "      <th>FullBath</th>\n",
              "      <th>HalfBath</th>\n",
              "      <th>BedroomAbvGr</th>\n",
              "      <th>KitchenAbvGr</th>\n",
              "      <th>TotRmsAbvGrd</th>\n",
              "      <th>Fireplaces</th>\n",
              "      <th>GarageYrBlt</th>\n",
              "      <th>GarageCars</th>\n",
              "      <th>GarageArea</th>\n",
              "      <th>WoodDeckSF</th>\n",
              "      <th>OpenPorchSF</th>\n",
              "      <th>EnclosedPorch</th>\n",
              "      <th>3SsnPorch</th>\n",
              "      <th>ScreenPorch</th>\n",
              "      <th>PoolArea</th>\n",
              "      <th>MiscVal</th>\n",
              "      <th>MoSold</th>\n",
              "      <th>YrSold</th>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>Id</th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>619</th>\n",
              "      <td>20</td>\n",
              "      <td>90.0</td>\n",
              "      <td>11694</td>\n",
              "      <td>9</td>\n",
              "      <td>5</td>\n",
              "      <td>2007</td>\n",
              "      <td>2007</td>\n",
              "      <td>452.0</td>\n",
              "      <td>48</td>\n",
              "      <td>0</td>\n",
              "      <td>1774</td>\n",
              "      <td>1822</td>\n",
              "      <td>1828</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>1828</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>2</td>\n",
              "      <td>0</td>\n",
              "      <td>3</td>\n",
              "      <td>1</td>\n",
              "      <td>9</td>\n",
              "      <td>1</td>\n",
              "      <td>2007.0</td>\n",
              "      <td>3</td>\n",
              "      <td>774</td>\n",
              "      <td>0</td>\n",
              "      <td>108</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>260</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>7</td>\n",
              "      <td>2007</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>871</th>\n",
              "      <td>20</td>\n",
              "      <td>60.0</td>\n",
              "      <td>6600</td>\n",
              "      <td>5</td>\n",
              "      <td>5</td>\n",
              "      <td>1962</td>\n",
              "      <td>1962</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>894</td>\n",
              "      <td>894</td>\n",
              "      <td>894</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>894</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>2</td>\n",
              "      <td>1</td>\n",
              "      <td>5</td>\n",
              "      <td>0</td>\n",
              "      <td>1962.0</td>\n",
              "      <td>1</td>\n",
              "      <td>308</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>8</td>\n",
              "      <td>2009</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "     MSSubClass  LotFrontage  LotArea  ...  MiscVal  MoSold  YrSold\n",
              "Id                                     ...                         \n",
              "619          20         90.0    11694  ...        0       7    2007\n",
              "871          20         60.0     6600  ...        0       8    2009\n",
              "\n",
              "[2 rows x 36 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 4
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "xP4R6VM2UI70",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 102
        },
        "outputId": "0f8eed0c-cdb3-434a-8928-9d68c1f85f39"
      },
      "source": [
        "# Shape of training data (num_rows, num_columns)\n",
        "print(X_train.shape)\n",
        "#print(X_train.columns)\n",
        "# Number of missing values in each column of training data\n",
        "missing_val_count_by_column = (X_train.isnull().sum())\n",
        "#print(missing_val_count_by_column)\n",
        "print(missing_val_count_by_column[missing_val_count_by_column > 0])"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "(1168, 36)\n",
            "LotFrontage    212\n",
            "MasVnrArea       6\n",
            "GarageYrBlt     58\n",
            "dtype: int64\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "-03KFbnvU25W",
        "colab_type": "text"
      },
      "source": [
        "Best approach to dealing with missing values"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "5Xr4yzp5V2yU",
        "colab_type": "text"
      },
      "source": [
        "Approach 1 (Drop Columns with Missing Values)                   **(Usually this is a wrong approach)**"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "wrwjBklQkVcG",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "from sklearn.ensemble import RandomForestRegressor\n",
        "from sklearn.metrics import mean_absolute_error\n",
        "\n",
        "# Function for comparing different approaches\n",
        "def score_dataset(X_train, X_test, y_train, y_test):\n",
        "    model = RandomForestRegressor(n_estimators=100, random_state=0)\n",
        "    model.fit(X_train, y_train)\n",
        "    preds = model.predict(X_test)\n",
        "    return mean_absolute_error(y_test, preds)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "kObYMFQxVJJP",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 68
        },
        "outputId": "2c11cce0-a1b5-473d-baf2-3491ce31c88b"
      },
      "source": [
        "# Get names of columns with missing values\n",
        "cols_with_missing = [col for col in X_train.columns\n",
        "                     if X_train[col].isnull().any()]\n",
        "\n",
        "# Drop columns in training and validation data\n",
        "reduced_X_train = X_train.drop(cols_with_missing, axis=1)\n",
        "reduced_X_test = X_test.drop(cols_with_missing, axis=1)\n",
        "\n",
        "print(\"MAE from Approach 1 (Drop columns with missing values):\")\n",
        "print(score_dataset(reduced_X_train, reduced_X_test, y_train, y_test))\n",
        "print(reduced_X_train.shape)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "MAE from Approach 1 (Drop columns with missing values):\n",
            "17837.82570776256\n",
            "(1168, 33)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Wje2nyZqV6R0",
        "colab_type": "text"
      },
      "source": [
        "Approach 2 (Imputation)\n",
        "SimpleImputer to replace missing values with the mean value along each column"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "FB_ScRbGV-8r",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 680
        },
        "outputId": "0fd2c5b1-6adf-4b4b-9afe-a6da246ecffe"
      },
      "source": [
        "from sklearn.impute import SimpleImputer\n",
        "\n",
        "# Imputation\n",
        "imputer = SimpleImputer()\n",
        "imputed_X_train = pd.DataFrame(imputer.fit_transform(X_train))\n",
        "imputed_X_test = pd.DataFrame(imputer.transform(X_test))\n",
        "\n",
        "# Imputation removed column names; put them back\n",
        "imputed_X_train.columns = X_train.columns\n",
        "imputed_X_test.columns = X_test.columns\n",
        "\n",
        "print(\"MAE from Approach 2 (Imputation):\")\n",
        "print(score_dataset(imputed_X_train, imputed_X_test, y_train, y_test))\n",
        "\n",
        "#check if any missing value in any column\n",
        "missing_val_count_by_column = (imputed_X_train.isnull().sum())\n",
        "print(missing_val_count_by_column)\n",
        "#print(missing_val_count_by_column[missing_val_count_by_column > 0])"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "MAE from Approach 2 (Imputation):\n",
            "18062.894611872147\n",
            "MSSubClass       0\n",
            "LotFrontage      0\n",
            "LotArea          0\n",
            "OverallQual      0\n",
            "OverallCond      0\n",
            "YearBuilt        0\n",
            "YearRemodAdd     0\n",
            "MasVnrArea       0\n",
            "BsmtFinSF1       0\n",
            "BsmtFinSF2       0\n",
            "BsmtUnfSF        0\n",
            "TotalBsmtSF      0\n",
            "1stFlrSF         0\n",
            "2ndFlrSF         0\n",
            "LowQualFinSF     0\n",
            "GrLivArea        0\n",
            "BsmtFullBath     0\n",
            "BsmtHalfBath     0\n",
            "FullBath         0\n",
            "HalfBath         0\n",
            "BedroomAbvGr     0\n",
            "KitchenAbvGr     0\n",
            "TotRmsAbvGrd     0\n",
            "Fireplaces       0\n",
            "GarageYrBlt      0\n",
            "GarageCars       0\n",
            "GarageArea       0\n",
            "WoodDeckSF       0\n",
            "OpenPorchSF      0\n",
            "EnclosedPorch    0\n",
            "3SsnPorch        0\n",
            "ScreenPorch      0\n",
            "PoolArea         0\n",
            "MiscVal          0\n",
            "MoSold           0\n",
            "YrSold           0\n",
            "dtype: int64\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "beo4hs-hWpOf",
        "colab_type": "text"
      },
      "source": [
        "Preprocess the training and validation data using Imputer MEDIAN strategy"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "36E2VYUdWang",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 646
        },
        "outputId": "f5cd6f71-0670-4c22-933b-592d624caa9a"
      },
      "source": [
        "final_imputer = SimpleImputer(strategy='median')\n",
        "final_X_train = pd.DataFrame(final_imputer.fit_transform(X_train))\n",
        "final_X_test = pd.DataFrame(final_imputer.transform(X_test))\n",
        "\n",
        "# Imputation removed column names; put them back\n",
        "final_X_train.columns = X_train.columns\n",
        "final_X_test.columns = X_test.columns\n",
        "\n",
        "missing_val_count_by_column = (final_X_train.isnull().sum())\n",
        "print(missing_val_count_by_column)\n",
        "#print(missing_val_count_by_column[missing_val_count_by_column > 0])"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "MSSubClass       0\n",
            "LotFrontage      0\n",
            "LotArea          0\n",
            "OverallQual      0\n",
            "OverallCond      0\n",
            "YearBuilt        0\n",
            "YearRemodAdd     0\n",
            "MasVnrArea       0\n",
            "BsmtFinSF1       0\n",
            "BsmtFinSF2       0\n",
            "BsmtUnfSF        0\n",
            "TotalBsmtSF      0\n",
            "1stFlrSF         0\n",
            "2ndFlrSF         0\n",
            "LowQualFinSF     0\n",
            "GrLivArea        0\n",
            "BsmtFullBath     0\n",
            "BsmtHalfBath     0\n",
            "FullBath         0\n",
            "HalfBath         0\n",
            "BedroomAbvGr     0\n",
            "KitchenAbvGr     0\n",
            "TotRmsAbvGrd     0\n",
            "Fireplaces       0\n",
            "GarageYrBlt      0\n",
            "GarageCars       0\n",
            "GarageArea       0\n",
            "WoodDeckSF       0\n",
            "OpenPorchSF      0\n",
            "EnclosedPorch    0\n",
            "3SsnPorch        0\n",
            "ScreenPorch      0\n",
            "PoolArea         0\n",
            "MiscVal          0\n",
            "MoSold           0\n",
            "YrSold           0\n",
            "dtype: int64\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "A603xeGjXV1N",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 51
        },
        "outputId": "417e7c21-f685-495c-80b2-4d484c56621f"
      },
      "source": [
        "#Evaluate RANDOM FOREST REGRESSION model\n",
        "\n",
        "# Define and fit model\n",
        "\n",
        "model = RandomForestRegressor(n_estimators=150, random_state=0)\n",
        "model.fit(final_X_train, y_train)\n",
        "\n",
        "# Get validation predictions and MAE\n",
        "preds_test = model.predict(final_X_test)\n",
        "print(\"MAE (Your approach):\")\n",
        "print(mean_absolute_error(y_test, preds_test))"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "MAE (Your approach):\n",
            "17576.83299847793\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "-8ZZy1RMhtjt",
        "colab_type": "text"
      },
      "source": [
        "Applying on 'test' dataset"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "HSb5OBUbgXLU",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 51
        },
        "outputId": "bb7769e3-5a95-4f80-ea26-0879e5191104"
      },
      "source": [
        "final_test = pd.DataFrame(final_imputer.fit_transform(test))\n",
        "\n",
        "# get test predictions\n",
        "preds_test = model.predict(final_test)\n",
        "print(preds_test)\n",
        "#predictor values of houses for 'test' based on 'train'"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "[127353.72       154886.66666667 181673.82666667 ... 157433.36\n",
            " 108949.         231465.03333333]\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "E0tRubq-k9v5",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "outputId": "7d396e06-83bf-4cc3-823c-8e8f835e9654"
      },
      "source": [
        "output = pd.DataFrame({'Id': test.index,\n",
        "                       'SalePrice': preds_test})\n",
        "output.to_csv('submission.csv', index=False)\n",
        "z=pd.read_csv('submission.csv')\n",
        "z.shape"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(1459, 2)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 12
        }
      ]
    }
  ]
}