Intro_to_Python_for_ML .ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "be534e9c",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "0638a69b",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>record_id</th>\n",
       "      <th>utility_id_ferc1</th>\n",
       "      <th>report_year</th>\n",
       "      <th>plant_name_ferc1</th>\n",
       "      <th>fuel_type_code_pudl</th>\n",
       "      <th>fuel_unit</th>\n",
       "      <th>fuel_qty_burned</th>\n",
       "      <th>fuel_mmbtu_per_unit</th>\n",
       "      <th>fuel_cost_per_unit_burned</th>\n",
       "      <th>fuel_cost_per_unit_delivered</th>\n",
       "      <th>fuel_cost_per_mmbtu</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>f1_fuel_1994_12_1_0_7</td>\n",
       "      <td>1</td>\n",
       "      <td>1994</td>\n",
       "      <td>rockport</td>\n",
       "      <td>coal</td>\n",
       "      <td>ton</td>\n",
       "      <td>5377489.0</td>\n",
       "      <td>16.590</td>\n",
       "      <td>18.59</td>\n",
       "      <td>18.53</td>\n",
       "      <td>1.121</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>f1_fuel_1994_12_1_0_10</td>\n",
       "      <td>1</td>\n",
       "      <td>1994</td>\n",
       "      <td>rockport total plant</td>\n",
       "      <td>coal</td>\n",
       "      <td>ton</td>\n",
       "      <td>10486945.0</td>\n",
       "      <td>16.592</td>\n",
       "      <td>18.58</td>\n",
       "      <td>18.53</td>\n",
       "      <td>1.120</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>f1_fuel_1994_12_2_0_1</td>\n",
       "      <td>2</td>\n",
       "      <td>1994</td>\n",
       "      <td>gorgas</td>\n",
       "      <td>coal</td>\n",
       "      <td>ton</td>\n",
       "      <td>2978683.0</td>\n",
       "      <td>24.130</td>\n",
       "      <td>39.72</td>\n",
       "      <td>38.12</td>\n",
       "      <td>1.650</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>f1_fuel_1994_12_2_0_7</td>\n",
       "      <td>2</td>\n",
       "      <td>1994</td>\n",
       "      <td>barry</td>\n",
       "      <td>coal</td>\n",
       "      <td>ton</td>\n",
       "      <td>3739484.0</td>\n",
       "      <td>23.950</td>\n",
       "      <td>47.21</td>\n",
       "      <td>45.99</td>\n",
       "      <td>1.970</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>f1_fuel_1994_12_2_0_10</td>\n",
       "      <td>2</td>\n",
       "      <td>1994</td>\n",
       "      <td>chickasaw</td>\n",
       "      <td>gas</td>\n",
       "      <td>mcf</td>\n",
       "      <td>40533.0</td>\n",
       "      <td>1.000</td>\n",
       "      <td>2.77</td>\n",
       "      <td>2.77</td>\n",
       "      <td>2.570</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29518</th>\n",
       "      <td>f1_fuel_2018_12_12_0_13</td>\n",
       "      <td>12</td>\n",
       "      <td>2018</td>\n",
       "      <td>neil simpson ct #1</td>\n",
       "      <td>gas</td>\n",
       "      <td>mcf</td>\n",
       "      <td>18799.0</td>\n",
       "      <td>1.059</td>\n",
       "      <td>4.78</td>\n",
       "      <td>4.78</td>\n",
       "      <td>9.030</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29519</th>\n",
       "      <td>f1_fuel_2018_12_12_1_1</td>\n",
       "      <td>12</td>\n",
       "      <td>2018</td>\n",
       "      <td>cheyenne prairie 58%</td>\n",
       "      <td>gas</td>\n",
       "      <td>mcf</td>\n",
       "      <td>806730.0</td>\n",
       "      <td>1.050</td>\n",
       "      <td>3.65</td>\n",
       "      <td>3.65</td>\n",
       "      <td>6.950</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29520</th>\n",
       "      <td>f1_fuel_2018_12_12_1_10</td>\n",
       "      <td>12</td>\n",
       "      <td>2018</td>\n",
       "      <td>lange ct facility</td>\n",
       "      <td>gas</td>\n",
       "      <td>mcf</td>\n",
       "      <td>104554.0</td>\n",
       "      <td>1.060</td>\n",
       "      <td>4.77</td>\n",
       "      <td>4.77</td>\n",
       "      <td>8.990</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29521</th>\n",
       "      <td>f1_fuel_2018_12_12_1_13</td>\n",
       "      <td>12</td>\n",
       "      <td>2018</td>\n",
       "      <td>wygen 3 bhp 52%</td>\n",
       "      <td>coal</td>\n",
       "      <td>ton</td>\n",
       "      <td>315945.0</td>\n",
       "      <td>16.108</td>\n",
       "      <td>3.06</td>\n",
       "      <td>14.76</td>\n",
       "      <td>1.110</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29522</th>\n",
       "      <td>f1_fuel_2018_12_12_1_14</td>\n",
       "      <td>12</td>\n",
       "      <td>2018</td>\n",
       "      <td>wygen 3 bhp 52%</td>\n",
       "      <td>gas</td>\n",
       "      <td>mcf</td>\n",
       "      <td>17853.0</td>\n",
       "      <td>1.059</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>11.680</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>29523 rows × 11 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                     record_id  utility_id_ferc1  report_year  \\\n",
       "0        f1_fuel_1994_12_1_0_7                 1         1994   \n",
       "1       f1_fuel_1994_12_1_0_10                 1         1994   \n",
       "2        f1_fuel_1994_12_2_0_1                 2         1994   \n",
       "3        f1_fuel_1994_12_2_0_7                 2         1994   \n",
       "4       f1_fuel_1994_12_2_0_10                 2         1994   \n",
       "...                        ...               ...          ...   \n",
       "29518  f1_fuel_2018_12_12_0_13                12         2018   \n",
       "29519   f1_fuel_2018_12_12_1_1                12         2018   \n",
       "29520  f1_fuel_2018_12_12_1_10                12         2018   \n",
       "29521  f1_fuel_2018_12_12_1_13                12         2018   \n",
       "29522  f1_fuel_2018_12_12_1_14                12         2018   \n",
       "\n",
       "           plant_name_ferc1 fuel_type_code_pudl fuel_unit  fuel_qty_burned  \\\n",
       "0                  rockport                coal       ton        5377489.0   \n",
       "1      rockport total plant                coal       ton       10486945.0   \n",
       "2                    gorgas                coal       ton        2978683.0   \n",
       "3                     barry                coal       ton        3739484.0   \n",
       "4                 chickasaw                 gas       mcf          40533.0   \n",
       "...                     ...                 ...       ...              ...   \n",
       "29518    neil simpson ct #1                 gas       mcf          18799.0   \n",
       "29519  cheyenne prairie 58%                 gas       mcf         806730.0   \n",
       "29520     lange ct facility                 gas       mcf         104554.0   \n",
       "29521       wygen 3 bhp 52%                coal       ton         315945.0   \n",
       "29522       wygen 3 bhp 52%                 gas       mcf          17853.0   \n",
       "\n",
       "       fuel_mmbtu_per_unit  fuel_cost_per_unit_burned  \\\n",
       "0                   16.590                      18.59   \n",
       "1                   16.592                      18.58   \n",
       "2                   24.130                      39.72   \n",
       "3                   23.950                      47.21   \n",
       "4                    1.000                       2.77   \n",
       "...                    ...                        ...   \n",
       "29518                1.059                       4.78   \n",
       "29519                1.050                       3.65   \n",
       "29520                1.060                       4.77   \n",
       "29521               16.108                       3.06   \n",
       "29522                1.059                       0.00   \n",
       "\n",
       "       fuel_cost_per_unit_delivered  fuel_cost_per_mmbtu  \n",
       "0                             18.53                1.121  \n",
       "1                             18.53                1.120  \n",
       "2                             38.12                1.650  \n",
       "3                             45.99                1.970  \n",
       "4                              2.77                2.570  \n",
       "...                             ...                  ...  \n",
       "29518                          4.78                9.030  \n",
       "29519                          3.65                6.950  \n",
       "29520                          4.77                8.990  \n",
       "29521                         14.76                1.110  \n",
       "29522                          0.00               11.680  \n",
       "\n",
       "[29523 rows x 11 columns]"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "url=\"http://bit.ly/HDSC-Dataset\"\n",
    "df=pd.read_csv(url,  error_bad_lines=False)\n",
    "df"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "014c89fc",
   "metadata": {},
   "source": [
    "# Question 3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "04ec44ac",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "fuel_type_code_pudl\n",
       "coal         67.421830\n",
       "gas          13.659397\n",
       "nuclear    4955.157002\n",
       "oil         168.877086\n",
       "other        18.253856\n",
       "waste        19.518122\n",
       "Name: fuel_cost_per_unit_burned, dtype: float64"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#average fuel cost per unit burned for each fuel type code\n",
    "\n",
    "df.groupby(\"fuel_type_code_pudl\")[\"fuel_cost_per_unit_burned\"].mean()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "5b8c3217",
   "metadata": {},
   "source": [
    "# Question 4"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "65c4da98",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "count    29523.000000\n",
       "mean         8.492111\n",
       "std         10.600220\n",
       "min          0.000001\n",
       "25%          1.024000\n",
       "50%          5.762694\n",
       "75%         17.006000\n",
       "max        341.260000\n",
       "Name: fuel_mmbtu_per_unit, dtype: float64"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#std and 75th percentile of measure of energy per unit\n",
    "\n",
    "df[\"fuel_mmbtu_per_unit\"].describe()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "1db0e5fc",
   "metadata": {},
   "source": [
    "# Question 5"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "093e7a06",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "15.851495469109503\n",
      "651.3694501337732\n"
     ]
    }
   ],
   "source": [
    "#skewness and kurtosis for fuel qty burned\n",
    "\n",
    "print(df[\"fuel_qty_burned\"].skew())\n",
    "print(df[\"fuel_qty_burned\"].kurt())"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "7702fece",
   "metadata": {},
   "source": [
    "# Question 6"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "8dd9009d",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "record_id                         0\n",
      "utility_id_ferc1                  0\n",
      "report_year                       0\n",
      "plant_name_ferc1                  0\n",
      "fuel_type_code_pudl               0\n",
      "fuel_unit                       180\n",
      "fuel_qty_burned                   0\n",
      "fuel_mmbtu_per_unit               0\n",
      "fuel_cost_per_unit_burned         0\n",
      "fuel_cost_per_unit_delivered      0\n",
      "fuel_cost_per_mmbtu               0\n",
      "dtype: int64\n"
     ]
    }
   ],
   "source": [
    "#Feature with missing values\n",
    "\n",
    "print(df.isnull().sum())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "48d3dc6d",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.609694136774718\n"
     ]
    }
   ],
   "source": [
    "print(180/len(df)*100)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "12ff8636",
   "metadata": {},
   "source": [
    "# Question 8"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "971adc8d",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "utility_id_ferc1               -0.037863\n",
       "report_year                     0.013599\n",
       "fuel_qty_burned                -0.018535\n",
       "fuel_mmbtu_per_unit            -0.010034\n",
       "fuel_cost_per_unit_burned       1.000000\n",
       "fuel_cost_per_unit_delivered    0.011007\n",
       "fuel_cost_per_mmbtu            -0.000437\n",
       "Name: fuel_cost_per_unit_burned, dtype: float64"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.corr()[\"fuel_cost_per_unit_burned\"]"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "9606564b",
   "metadata": {},
   "source": [
    "# Question 9"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "5c6c1d18",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "report_year\n",
       "1994     14984.572\n",
       "1995     14571.785\n",
       "1996     12694.803\n",
       "1997     43742.178\n",
       "1998     11902.597\n",
       "1999     16303.468\n",
       "2000    100428.831\n",
       "2001     15109.012\n",
       "2002      9171.164\n",
       "2003     30954.110\n",
       "2004      9632.063\n",
       "2005     11107.182\n",
       "2006     12402.684\n",
       "2007     12965.171\n",
       "2008     14331.348\n",
       "2009     45554.858\n",
       "2010     88627.745\n",
       "2011     17001.015\n",
       "2012     17546.097\n",
       "2013     17629.860\n",
       "2014     15412.197\n",
       "2015     13338.235\n",
       "2016     11100.925\n",
       "2017     10254.284\n",
       "2018      9488.197\n",
       "Name: fuel_cost_per_unit_burned, dtype: float64"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#fuel cost per unit burned for coal \n",
    "\n",
    "coal_df = df[df[\"fuel_type_code_pudl\"] == \"coal\"]\n",
    "coal_df.groupby(\"report_year\")[\"fuel_cost_per_unit_burned\"].sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "b5b67223",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "-20.567654518260518"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "(11902.597-14984.572)/14984.572 * 100"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b0a96c1b",
   "metadata": {},
   "source": [
    "# Question 10"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "5712070d",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "report_year\n",
       "1994       63.636060\n",
       "1995       32.735269\n",
       "1996     9196.705948\n",
       "1997    11140.197239\n",
       "1998      287.154420\n",
       "1999       25.551627\n",
       "2000      985.362877\n",
       "2001       60.050396\n",
       "2002       47.594361\n",
       "2003       55.663493\n",
       "2004      139.524275\n",
       "2005       41.438184\n",
       "2006       38.657484\n",
       "2007       43.325023\n",
       "2008       58.588197\n",
       "2009      652.694163\n",
       "2010       91.862105\n",
       "2011       59.774667\n",
       "2012       60.994502\n",
       "2013      172.307591\n",
       "2014      192.737183\n",
       "2015      326.535511\n",
       "2016      103.901761\n",
       "2017       46.196861\n",
       "2018      499.269966\n",
       "Name: fuel_cost_per_unit_delivered, dtype: float64"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#fuel cost per unit delivered\n",
    "\n",
    "df.groupby(\"report_year\")[\"fuel_cost_per_unit_delivered\"].mean()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}