testing_the_estimator.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import statsmodels.api as sm\n",
    "from statsmodels.regression.mixed_linear_model import MixedLM\n",
    "import matplotlib.pyplot as plt\n",
    "from scipy import stats\n",
    "from viewser.operations import fetch\n",
    "from viewser import Queryset, Column\n",
    "import subprocess\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " .    "
     ]
    }
   ],
   "source": [
    "df = Queryset(\"month_to_annual_moving_sum\", \"country_month\").fetch()\n",
    "df = df.query('month_id > 240 & month_id < 511')\n",
    "df = df.query('country_id != [239,240,208,230,59]')\n",
    "# Remove countries that don't exist anymore\n",
    "df = df.query('country_id!=185')  # German Democratic Republic\n",
    "# Remove country months that have value zero for dependent variable\n",
    "df = df.query('wdi_sh_dyn_mort_fe!=0')\n",
    "fixed_effect_column = 'country_id'\n",
    "df[fixed_effect_column] = df.index.get_level_values(fixed_effect_column)\n",
    "fixed_effect_column = 'month_id'\n",
    "df[fixed_effect_column] = df.index.get_level_values(fixed_effect_column)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "independent_variable = ['wdi_ny_gdp_mktp_kd', 'wdi_sp_pop_totl', 'ged_sb_tlag1', 'ged_sb_tlag2', 'ged_sb_tlag3', 'ged_sb_tlag4', 'ged_sb_tlag5', 'ged_sb_tlag6', 'ged_sb_tlag7', 'ged_sb_tlag8', 'ged_sb_tlag9', 'ged_sb_tlag10', 'ged_sb_tlag11', 'ged_sb_tlag12', 'ged_sb_tlag13', 'ged_sb_tlag14', 'ged_sb_tlag15', 'ged_sb_tlag16', 'ged_sb_tlag17',\n",
    "                        'ged_sb_tlag18', 'ged_sb_tlag19', 'ged_sb_tlag20', 'ged_sb_tlag21', 'ged_sb_tlag22', 'ged_sb_tlag23', 'ged_sb_tlag24', 'ged_sb_tlag25', 'ged_sb_tlag26', 'ged_sb_tlag27', 'ged_sb_tlag28', 'ged_sb_tlag29', 'ged_sb_tlag30', 'ged_sb_tlag31', 'ged_sb_tlag32', 'ged_sb_tlag33', 'ged_sb_tlag34', 'ged_sb_tlag35', 'ged_sb_tlag36','country_id','month_id','ged_sb_nolag']\n",
    "dependent_variable = 'wdi_sh_dyn_mort_fe'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th>wdi_ny_gdp_mktp_kd</th>\n",
       "      <th>wdi_sp_pop_totl</th>\n",
       "      <th>ged_sb_tlag1</th>\n",
       "      <th>ged_sb_tlag2</th>\n",
       "      <th>ged_sb_tlag3</th>\n",
       "      <th>ged_sb_tlag4</th>\n",
       "      <th>ged_sb_tlag5</th>\n",
       "      <th>ged_sb_tlag6</th>\n",
       "      <th>ged_sb_tlag7</th>\n",
       "      <th>ged_sb_tlag8</th>\n",
       "      <th>...</th>\n",
       "      <th>ged_sb_tlag30</th>\n",
       "      <th>ged_sb_tlag31</th>\n",
       "      <th>ged_sb_tlag32</th>\n",
       "      <th>ged_sb_tlag33</th>\n",
       "      <th>ged_sb_tlag34</th>\n",
       "      <th>ged_sb_tlag35</th>\n",
       "      <th>ged_sb_tlag36</th>\n",
       "      <th>country_id</th>\n",
       "      <th>month_id</th>\n",
       "      <th>ged_sb_nolag</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>month_id</th>\n",
       "      <th>country_id</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th rowspan=\"5\" valign=\"top\">241</th>\n",
       "      <th>1</th>\n",
       "      <td>21.779484</td>\n",
       "      <td>13.539826</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>1</td>\n",
       "      <td>241</td>\n",
       "      <td>0.00000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>21.795169</td>\n",
       "      <td>13.079454</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>2</td>\n",
       "      <td>241</td>\n",
       "      <td>0.00000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>23.309243</td>\n",
       "      <td>14.102345</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>3</td>\n",
       "      <td>241</td>\n",
       "      <td>0.00000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>17.011230</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>4</td>\n",
       "      <td>241</td>\n",
       "      <td>0.00000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>20.098032</td>\n",
       "      <td>12.122740</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>5</td>\n",
       "      <td>241</td>\n",
       "      <td>0.00000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"5\" valign=\"top\">510</th>\n",
       "      <th>242</th>\n",
       "      <td>24.884573</td>\n",
       "      <td>17.967941</td>\n",
       "      <td>4.969813</td>\n",
       "      <td>4.969813</td>\n",
       "      <td>4.969813</td>\n",
       "      <td>4.969813</td>\n",
       "      <td>4.969813</td>\n",
       "      <td>4.969813</td>\n",
       "      <td>2.890372</td>\n",
       "      <td>1.791759</td>\n",
       "      <td>...</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>242</td>\n",
       "      <td>510</td>\n",
       "      <td>0.00000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>243</th>\n",
       "      <td>25.542467</td>\n",
       "      <td>17.428496</td>\n",
       "      <td>4.852030</td>\n",
       "      <td>4.852030</td>\n",
       "      <td>4.852030</td>\n",
       "      <td>4.852030</td>\n",
       "      <td>4.852030</td>\n",
       "      <td>2.772589</td>\n",
       "      <td>2.772589</td>\n",
       "      <td>2.484907</td>\n",
       "      <td>...</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>243</td>\n",
       "      <td>510</td>\n",
       "      <td>0.00000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>244</th>\n",
       "      <td>22.728763</td>\n",
       "      <td>15.344817</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>244</td>\n",
       "      <td>510</td>\n",
       "      <td>0.00000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>245</th>\n",
       "      <td>25.101865</td>\n",
       "      <td>17.636672</td>\n",
       "      <td>4.276666</td>\n",
       "      <td>4.276666</td>\n",
       "      <td>4.276666</td>\n",
       "      <td>4.276666</td>\n",
       "      <td>4.276666</td>\n",
       "      <td>6.984716</td>\n",
       "      <td>6.984716</td>\n",
       "      <td>6.984716</td>\n",
       "      <td>...</td>\n",
       "      <td>6.109248</td>\n",
       "      <td>8.411833</td>\n",
       "      <td>12.240474</td>\n",
       "      <td>13.849912</td>\n",
       "      <td>17.781738</td>\n",
       "      <td>19.168032</td>\n",
       "      <td>22.112471</td>\n",
       "      <td>245</td>\n",
       "      <td>510</td>\n",
       "      <td>0.00000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>246</th>\n",
       "      <td>23.207989</td>\n",
       "      <td>16.190256</td>\n",
       "      <td>17.794711</td>\n",
       "      <td>20.097296</td>\n",
       "      <td>20.097296</td>\n",
       "      <td>21.706734</td>\n",
       "      <td>21.013587</td>\n",
       "      <td>21.013587</td>\n",
       "      <td>16.886452</td>\n",
       "      <td>19.371359</td>\n",
       "      <td>...</td>\n",
       "      <td>20.620358</td>\n",
       "      <td>24.309238</td>\n",
       "      <td>24.309238</td>\n",
       "      <td>23.461940</td>\n",
       "      <td>23.886823</td>\n",
       "      <td>24.898424</td>\n",
       "      <td>25.186106</td>\n",
       "      <td>246</td>\n",
       "      <td>510</td>\n",
       "      <td>1.94591</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>50513 rows × 41 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                     wdi_ny_gdp_mktp_kd  wdi_sp_pop_totl  ged_sb_tlag1  \\\n",
       "month_id country_id                                                      \n",
       "241      1                    21.779484        13.539826      0.000000   \n",
       "         2                    21.795169        13.079454      0.000000   \n",
       "         3                    23.309243        14.102345      0.000000   \n",
       "         4                     0.000000        17.011230      0.000000   \n",
       "         5                    20.098032        12.122740      0.000000   \n",
       "...                                 ...              ...           ...   \n",
       "510      242                  24.884573        17.967941      4.969813   \n",
       "         243                  25.542467        17.428496      4.852030   \n",
       "         244                  22.728763        15.344817      0.000000   \n",
       "         245                  25.101865        17.636672      4.276666   \n",
       "         246                  23.207989        16.190256     17.794711   \n",
       "\n",
       "                     ged_sb_tlag2  ged_sb_tlag3  ged_sb_tlag4  ged_sb_tlag5  \\\n",
       "month_id country_id                                                           \n",
       "241      1               0.000000      0.000000      0.000000      0.000000   \n",
       "         2               0.000000      0.000000      0.000000      0.000000   \n",
       "         3               0.000000      0.000000      0.000000      0.000000   \n",
       "         4               0.000000      0.000000      0.000000      0.000000   \n",
       "         5               0.000000      0.000000      0.000000      0.000000   \n",
       "...                           ...           ...           ...           ...   \n",
       "510      242             4.969813      4.969813      4.969813      4.969813   \n",
       "         243             4.852030      4.852030      4.852030      4.852030   \n",
       "         244             0.000000      0.000000      0.000000      0.000000   \n",
       "         245             4.276666      4.276666      4.276666      4.276666   \n",
       "         246            20.097296     20.097296     21.706734     21.013587   \n",
       "\n",
       "                     ged_sb_tlag6  ged_sb_tlag7  ged_sb_tlag8  ...  \\\n",
       "month_id country_id                                            ...   \n",
       "241      1               0.000000      0.000000      0.000000  ...   \n",
       "         2               0.000000      0.000000      0.000000  ...   \n",
       "         3               0.000000      0.000000      0.000000  ...   \n",
       "         4               0.000000      0.000000      0.000000  ...   \n",
       "         5               0.000000      0.000000      0.000000  ...   \n",
       "...                           ...           ...           ...  ...   \n",
       "510      242             4.969813      2.890372      1.791759  ...   \n",
       "         243             2.772589      2.772589      2.484907  ...   \n",
       "         244             0.000000      0.000000      0.000000  ...   \n",
       "         245             6.984716      6.984716      6.984716  ...   \n",
       "         246            21.013587     16.886452     19.371359  ...   \n",
       "\n",
       "                     ged_sb_tlag30  ged_sb_tlag31  ged_sb_tlag32  \\\n",
       "month_id country_id                                                \n",
       "241      1                0.000000       0.000000       0.000000   \n",
       "         2                0.000000       0.000000       0.000000   \n",
       "         3                0.000000       0.000000       0.000000   \n",
       "         4                0.000000       0.000000       0.000000   \n",
       "         5                0.000000       0.000000       0.000000   \n",
       "...                            ...            ...            ...   \n",
       "510      242              0.000000       0.000000       0.000000   \n",
       "         243              0.000000       0.000000       0.000000   \n",
       "         244              0.000000       0.000000       0.000000   \n",
       "         245              6.109248       8.411833      12.240474   \n",
       "         246             20.620358      24.309238      24.309238   \n",
       "\n",
       "                     ged_sb_tlag33  ged_sb_tlag34  ged_sb_tlag35  \\\n",
       "month_id country_id                                                \n",
       "241      1                0.000000       0.000000       0.000000   \n",
       "         2                0.000000       0.000000       0.000000   \n",
       "         3                0.000000       0.000000       0.000000   \n",
       "         4                0.000000       0.000000       0.000000   \n",
       "         5                0.000000       0.000000       0.000000   \n",
       "...                            ...            ...            ...   \n",
       "510      242              0.000000       0.000000       0.000000   \n",
       "         243              0.000000       0.000000       0.000000   \n",
       "         244              0.000000       0.000000       0.000000   \n",
       "         245             13.849912      17.781738      19.168032   \n",
       "         246             23.461940      23.886823      24.898424   \n",
       "\n",
       "                     ged_sb_tlag36  country_id  month_id  ged_sb_nolag  \n",
       "month_id country_id                                                     \n",
       "241      1                0.000000           1       241       0.00000  \n",
       "         2                0.000000           2       241       0.00000  \n",
       "         3                0.000000           3       241       0.00000  \n",
       "         4                0.000000           4       241       0.00000  \n",
       "         5                0.000000           5       241       0.00000  \n",
       "...                            ...         ...       ...           ...  \n",
       "510      242              0.000000         242       510       0.00000  \n",
       "         243              0.000000         243       510       0.00000  \n",
       "         244              0.000000         244       510       0.00000  \n",
       "         245             22.112471         245       510       0.00000  \n",
       "         246             25.186106         246       510       1.94591  \n",
       "\n",
       "[50513 rows x 41 columns]"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df[independent_variable]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "from MonthToAnnualScikit import *"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.model_selection import train_test_split\n",
    "\n",
    "# Assuming X and y are your independent and dependent variables respectively\n",
    "X_train, X_test, y_train, y_test = train_test_split(\n",
    "    df[independent_variable], df[dependent_variable], test_size=0.2, random_state=42)\n",
    "\n",
    "# Create an instance with a specific algorithm\n",
    "model = MotanRegression('XGBRegressor') "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "month_id  country_id\n",
       "353       30            1.916923\n",
       "286       169           4.294561\n",
       "460       69            4.280824\n",
       "359       27            2.923162\n",
       "500       62            2.646175\n",
       "                          ...   \n",
       "302       74            4.753590\n",
       "480       89            1.960095\n",
       "445       133           4.127134\n",
       "245       153           3.529297\n",
       "326       143           3.397858\n",
       "Name: wdi_sh_dyn_mort_fe, Length: 40410, dtype: float64"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_train"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th>wdi_ny_gdp_mktp_kd</th>\n",
       "      <th>wdi_sp_pop_totl</th>\n",
       "      <th>ged_sb_tlag1</th>\n",
       "      <th>ged_sb_tlag2</th>\n",
       "      <th>ged_sb_tlag3</th>\n",
       "      <th>ged_sb_tlag4</th>\n",
       "      <th>ged_sb_tlag5</th>\n",
       "      <th>ged_sb_tlag6</th>\n",
       "      <th>ged_sb_tlag7</th>\n",
       "      <th>ged_sb_tlag8</th>\n",
       "      <th>...</th>\n",
       "      <th>ged_sb_tlag30</th>\n",
       "      <th>ged_sb_tlag31</th>\n",
       "      <th>ged_sb_tlag32</th>\n",
       "      <th>ged_sb_tlag33</th>\n",
       "      <th>ged_sb_tlag34</th>\n",
       "      <th>ged_sb_tlag35</th>\n",
       "      <th>ged_sb_tlag36</th>\n",
       "      <th>country_id</th>\n",
       "      <th>month_id</th>\n",
       "      <th>ged_sb_nolag</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>month_id</th>\n",
       "      <th>country_id</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>353</th>\n",
       "      <th>30</th>\n",
       "      <td>25.028787</td>\n",
       "      <td>16.238824</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>30</td>\n",
       "      <td>353</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>286</th>\n",
       "      <th>169</th>\n",
       "      <td>23.035154</td>\n",
       "      <td>14.136433</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>169</td>\n",
       "      <td>286</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>460</th>\n",
       "      <th>69</th>\n",
       "      <td>24.313501</td>\n",
       "      <td>17.037452</td>\n",
       "      <td>30.450724</td>\n",
       "      <td>29.791478</td>\n",
       "      <td>28.518512</td>\n",
       "      <td>26.860284</td>\n",
       "      <td>24.027071</td>\n",
       "      <td>23.413966</td>\n",
       "      <td>24.577117</td>\n",
       "      <td>24.353974</td>\n",
       "      <td>...</td>\n",
       "      <td>37.418486</td>\n",
       "      <td>41.419740</td>\n",
       "      <td>43.109221</td>\n",
       "      <td>43.495994</td>\n",
       "      <td>43.090529</td>\n",
       "      <td>45.324121</td>\n",
       "      <td>47.316551</td>\n",
       "      <td>69</td>\n",
       "      <td>460</td>\n",
       "      <td>4.077537</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>359</th>\n",
       "      <th>27</th>\n",
       "      <td>21.349698</td>\n",
       "      <td>12.657696</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>27</td>\n",
       "      <td>359</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>500</th>\n",
       "      <th>62</th>\n",
       "      <td>24.464679</td>\n",
       "      <td>16.226796</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>62</td>\n",
       "      <td>500</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>302</th>\n",
       "      <th>74</th>\n",
       "      <td>22.742461</td>\n",
       "      <td>15.913457</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>74</td>\n",
       "      <td>302</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>480</th>\n",
       "      <th>89</th>\n",
       "      <td>24.774081</td>\n",
       "      <td>15.757952</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>89</td>\n",
       "      <td>480</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>445</th>\n",
       "      <th>133</th>\n",
       "      <td>23.767383</td>\n",
       "      <td>17.389075</td>\n",
       "      <td>87.068329</td>\n",
       "      <td>87.160372</td>\n",
       "      <td>86.841207</td>\n",
       "      <td>86.641380</td>\n",
       "      <td>86.608917</td>\n",
       "      <td>86.413058</td>\n",
       "      <td>86.234557</td>\n",
       "      <td>86.173299</td>\n",
       "      <td>...</td>\n",
       "      <td>76.726898</td>\n",
       "      <td>76.921054</td>\n",
       "      <td>77.040807</td>\n",
       "      <td>77.368974</td>\n",
       "      <td>77.242715</td>\n",
       "      <td>77.395822</td>\n",
       "      <td>77.228571</td>\n",
       "      <td>133</td>\n",
       "      <td>445</td>\n",
       "      <td>7.126891</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>245</th>\n",
       "      <th>153</th>\n",
       "      <td>19.554979</td>\n",
       "      <td>11.623662</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>153</td>\n",
       "      <td>245</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>326</th>\n",
       "      <th>143</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>17.008310</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>143</td>\n",
       "      <td>326</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>40410 rows × 41 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                     wdi_ny_gdp_mktp_kd  wdi_sp_pop_totl  ged_sb_tlag1  \\\n",
       "month_id country_id                                                      \n",
       "353      30                   25.028787        16.238824      0.000000   \n",
       "286      169                  23.035154        14.136433      0.000000   \n",
       "460      69                   24.313501        17.037452     30.450724   \n",
       "359      27                   21.349698        12.657696      0.000000   \n",
       "500      62                   24.464679        16.226796      0.000000   \n",
       "...                                 ...              ...           ...   \n",
       "302      74                   22.742461        15.913457      0.000000   \n",
       "480      89                   24.774081        15.757952      0.000000   \n",
       "445      133                  23.767383        17.389075     87.068329   \n",
       "245      153                  19.554979        11.623662      0.000000   \n",
       "326      143                   0.000000        17.008310      0.000000   \n",
       "\n",
       "                     ged_sb_tlag2  ged_sb_tlag3  ged_sb_tlag4  ged_sb_tlag5  \\\n",
       "month_id country_id                                                           \n",
       "353      30              0.000000      0.000000      0.000000      0.000000   \n",
       "286      169             0.000000      0.000000      0.000000      0.000000   \n",
       "460      69             29.791478     28.518512     26.860284     24.027071   \n",
       "359      27              0.000000      0.000000      0.000000      0.000000   \n",
       "500      62              0.000000      0.000000      0.000000      0.000000   \n",
       "...                           ...           ...           ...           ...   \n",
       "302      74              0.000000      0.000000      0.000000      0.000000   \n",
       "480      89              0.000000      0.000000      0.000000      0.000000   \n",
       "445      133            87.160372     86.841207     86.641380     86.608917   \n",
       "245      153             0.000000      0.000000      0.000000      0.000000   \n",
       "326      143             0.000000      0.000000      0.000000      0.000000   \n",
       "\n",
       "                     ged_sb_tlag6  ged_sb_tlag7  ged_sb_tlag8  ...  \\\n",
       "month_id country_id                                            ...   \n",
       "353      30              0.000000      0.000000      0.000000  ...   \n",
       "286      169             0.000000      0.000000      0.000000  ...   \n",
       "460      69             23.413966     24.577117     24.353974  ...   \n",
       "359      27              0.000000      0.000000      0.000000  ...   \n",
       "500      62              0.000000      0.000000      0.000000  ...   \n",
       "...                           ...           ...           ...  ...   \n",
       "302      74              0.000000      0.000000      0.000000  ...   \n",
       "480      89              0.000000      0.000000      0.000000  ...   \n",
       "445      133            86.413058     86.234557     86.173299  ...   \n",
       "245      153             0.000000      0.000000      0.000000  ...   \n",
       "326      143             0.000000      0.000000      0.000000  ...   \n",
       "\n",
       "                     ged_sb_tlag30  ged_sb_tlag31  ged_sb_tlag32  \\\n",
       "month_id country_id                                                \n",
       "353      30               0.000000       0.000000       0.000000   \n",
       "286      169              0.000000       0.000000       0.000000   \n",
       "460      69              37.418486      41.419740      43.109221   \n",
       "359      27               0.000000       0.000000       0.000000   \n",
       "500      62               0.000000       0.000000       0.000000   \n",
       "...                            ...            ...            ...   \n",
       "302      74               0.000000       0.000000       0.000000   \n",
       "480      89               0.000000       0.000000       0.000000   \n",
       "445      133             76.726898      76.921054      77.040807   \n",
       "245      153              0.000000       0.000000       0.000000   \n",
       "326      143              0.000000       0.000000       0.000000   \n",
       "\n",
       "                     ged_sb_tlag33  ged_sb_tlag34  ged_sb_tlag35  \\\n",
       "month_id country_id                                                \n",
       "353      30               0.000000       0.000000       0.000000   \n",
       "286      169              0.000000       0.000000       0.000000   \n",
       "460      69              43.495994      43.090529      45.324121   \n",
       "359      27               0.000000       0.000000       0.000000   \n",
       "500      62               0.000000       0.000000       0.000000   \n",
       "...                            ...            ...            ...   \n",
       "302      74               0.000000       0.000000       0.000000   \n",
       "480      89               0.000000       0.000000       0.000000   \n",
       "445      133             77.368974      77.242715      77.395822   \n",
       "245      153              0.000000       0.000000       0.000000   \n",
       "326      143              0.000000       0.000000       0.000000   \n",
       "\n",
       "                     ged_sb_tlag36  country_id  month_id  ged_sb_nolag  \n",
       "month_id country_id                                                     \n",
       "353      30               0.000000          30       353      0.000000  \n",
       "286      169              0.000000         169       286      0.000000  \n",
       "460      69              47.316551          69       460      4.077537  \n",
       "359      27               0.000000          27       359      0.000000  \n",
       "500      62               0.000000          62       500      0.000000  \n",
       "...                            ...         ...       ...           ...  \n",
       "302      74               0.000000          74       302      0.000000  \n",
       "480      89               0.000000          89       480      0.000000  \n",
       "445      133             77.228571         133       445      7.126891  \n",
       "245      153              0.000000         153       245      0.000000  \n",
       "326      143              0.000000         143       326      0.000000  \n",
       "\n",
       "[40410 rows x 41 columns]"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_train"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th>wdi_ny_gdp_mktp_kd</th>\n",
       "      <th>wdi_sp_pop_totl</th>\n",
       "      <th>ged_sb_tlag1</th>\n",
       "      <th>ged_sb_tlag2</th>\n",
       "      <th>ged_sb_tlag3</th>\n",
       "      <th>ged_sb_tlag4</th>\n",
       "      <th>ged_sb_tlag5</th>\n",
       "      <th>ged_sb_tlag6</th>\n",
       "      <th>ged_sb_tlag7</th>\n",
       "      <th>ged_sb_tlag8</th>\n",
       "      <th>...</th>\n",
       "      <th>ged_sb_tlag30</th>\n",
       "      <th>ged_sb_tlag31</th>\n",
       "      <th>ged_sb_tlag32</th>\n",
       "      <th>ged_sb_tlag33</th>\n",
       "      <th>ged_sb_tlag34</th>\n",
       "      <th>ged_sb_tlag35</th>\n",
       "      <th>ged_sb_tlag36</th>\n",
       "      <th>country_id</th>\n",
       "      <th>month_id</th>\n",
       "      <th>ged_sb_nolag</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>month_id</th>\n",
       "      <th>country_id</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>336</th>\n",
       "      <th>30</th>\n",
       "      <td>24.974035</td>\n",
       "      <td>16.237645</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>30</td>\n",
       "      <td>336</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>360</th>\n",
       "      <th>83</th>\n",
       "      <td>23.433950</td>\n",
       "      <td>15.170766</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>83</td>\n",
       "      <td>360</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>408</th>\n",
       "      <th>47</th>\n",
       "      <td>23.113265</td>\n",
       "      <td>16.685476</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>47</td>\n",
       "      <td>408</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>324</th>\n",
       "      <th>130</th>\n",
       "      <td>24.865117</td>\n",
       "      <td>13.830459</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>130</td>\n",
       "      <td>324</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>252</th>\n",
       "      <th>98</th>\n",
       "      <td>25.561652</td>\n",
       "      <td>16.143282</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>98</td>\n",
       "      <td>252</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>300</th>\n",
       "      <th>183</th>\n",
       "      <td>27.878367</td>\n",
       "      <td>17.279390</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>183</td>\n",
       "      <td>300</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>288</th>\n",
       "      <th>179</th>\n",
       "      <td>23.141264</td>\n",
       "      <td>15.622320</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>179</td>\n",
       "      <td>288</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>348</th>\n",
       "      <th>2</th>\n",
       "      <td>22.197387</td>\n",
       "      <td>13.188037</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2</td>\n",
       "      <td>348</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>336</th>\n",
       "      <th>43</th>\n",
       "      <td>21.469134</td>\n",
       "      <td>15.105498</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>43</td>\n",
       "      <td>336</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>480</th>\n",
       "      <th>89</th>\n",
       "      <td>24.774081</td>\n",
       "      <td>15.757952</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>89</td>\n",
       "      <td>480</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>3343 rows × 41 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                     wdi_ny_gdp_mktp_kd  wdi_sp_pop_totl  ged_sb_tlag1  \\\n",
       "month_id country_id                                                      \n",
       "336      30                   24.974035        16.237645           0.0   \n",
       "360      83                   23.433950        15.170766           0.0   \n",
       "408      47                   23.113265        16.685476           0.0   \n",
       "324      130                  24.865117        13.830459           0.0   \n",
       "252      98                   25.561652        16.143282           0.0   \n",
       "...                                 ...              ...           ...   \n",
       "300      183                  27.878367        17.279390           0.0   \n",
       "288      179                  23.141264        15.622320           0.0   \n",
       "348      2                    22.197387        13.188037           0.0   \n",
       "336      43                   21.469134        15.105498           0.0   \n",
       "480      89                   24.774081        15.757952           0.0   \n",
       "\n",
       "                     ged_sb_tlag2  ged_sb_tlag3  ged_sb_tlag4  ged_sb_tlag5  \\\n",
       "month_id country_id                                                           \n",
       "336      30                   0.0           0.0           0.0           0.0   \n",
       "360      83                   0.0           0.0           0.0           0.0   \n",
       "408      47                   0.0           0.0           0.0           0.0   \n",
       "324      130                  0.0           0.0           0.0           0.0   \n",
       "252      98                   0.0           0.0           0.0           0.0   \n",
       "...                           ...           ...           ...           ...   \n",
       "300      183                  0.0           0.0           0.0           0.0   \n",
       "288      179                  0.0           0.0           0.0           0.0   \n",
       "348      2                    0.0           0.0           0.0           0.0   \n",
       "336      43                   0.0           0.0           0.0           0.0   \n",
       "480      89                   0.0           0.0           0.0           0.0   \n",
       "\n",
       "                     ged_sb_tlag6  ged_sb_tlag7  ged_sb_tlag8  ...  \\\n",
       "month_id country_id                                            ...   \n",
       "336      30                   0.0           0.0           0.0  ...   \n",
       "360      83                   0.0           0.0           0.0  ...   \n",
       "408      47                   0.0           0.0           0.0  ...   \n",
       "324      130                  0.0           0.0           0.0  ...   \n",
       "252      98                   0.0           0.0           0.0  ...   \n",
       "...                           ...           ...           ...  ...   \n",
       "300      183                  0.0           0.0           0.0  ...   \n",
       "288      179                  0.0           0.0           0.0  ...   \n",
       "348      2                    0.0           0.0           0.0  ...   \n",
       "336      43                   0.0           0.0           0.0  ...   \n",
       "480      89                   0.0           0.0           0.0  ...   \n",
       "\n",
       "                     ged_sb_tlag30  ged_sb_tlag31  ged_sb_tlag32  \\\n",
       "month_id country_id                                                \n",
       "336      30                    0.0            0.0            0.0   \n",
       "360      83                    0.0            0.0            0.0   \n",
       "408      47                    0.0            0.0            0.0   \n",
       "324      130                   0.0            0.0            0.0   \n",
       "252      98                    0.0            0.0            0.0   \n",
       "...                            ...            ...            ...   \n",
       "300      183                   0.0            0.0            0.0   \n",
       "288      179                   0.0            0.0            0.0   \n",
       "348      2                     0.0            0.0            0.0   \n",
       "336      43                    0.0            0.0            0.0   \n",
       "480      89                    0.0            0.0            0.0   \n",
       "\n",
       "                     ged_sb_tlag33  ged_sb_tlag34  ged_sb_tlag35  \\\n",
       "month_id country_id                                                \n",
       "336      30                    0.0            0.0            0.0   \n",
       "360      83                    0.0            0.0            0.0   \n",
       "408      47                    0.0            0.0            0.0   \n",
       "324      130                   0.0            0.0            0.0   \n",
       "252      98                    0.0            0.0            0.0   \n",
       "...                            ...            ...            ...   \n",
       "300      183                   0.0            0.0            0.0   \n",
       "288      179                   0.0            0.0            0.0   \n",
       "348      2                     0.0            0.0            0.0   \n",
       "336      43                    0.0            0.0            0.0   \n",
       "480      89                    0.0            0.0            0.0   \n",
       "\n",
       "                     ged_sb_tlag36  country_id  month_id  ged_sb_nolag  \n",
       "month_id country_id                                                     \n",
       "336      30                    0.0          30       336           0.0  \n",
       "360      83                    0.0          83       360           0.0  \n",
       "408      47                    0.0          47       408           0.0  \n",
       "324      130                   0.0         130       324           0.0  \n",
       "252      98                    0.0          98       252           0.0  \n",
       "...                            ...         ...       ...           ...  \n",
       "300      183                   0.0         183       300           0.0  \n",
       "288      179                   0.0         179       288           0.0  \n",
       "348      2                     0.0           2       348           0.0  \n",
       "336      43                    0.0          43       336           0.0  \n",
       "480      89                    0.0          89       480           0.0  \n",
       "\n",
       "[3343 rows x 41 columns]"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_train.query('month_id % 12 == 0')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/noorainkazmi/miniforge3/envs/viewser/lib/python3.9/site-packages/xgboost/data.py:297: FutureWarning: is_sparse is deprecated and will be removed in a future version. Check `isinstance(dtype, pd.SparseDtype)` instead.\n",
      "  if is_sparse(dtype):\n",
      "/Users/noorainkazmi/miniforge3/envs/viewser/lib/python3.9/site-packages/xgboost/data.py:299: FutureWarning: is_categorical_dtype is deprecated and will be removed in a future version. Use isinstance(dtype, CategoricalDtype) instead\n",
      "  elif is_categorical_dtype(dtype) and enable_categorical:\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/noorainkazmi/miniforge3/envs/viewser/lib/python3.9/site-packages/xgboost/data.py:364: FutureWarning: is_sparse is deprecated and will be removed in a future version. Check `isinstance(dtype, pd.SparseDtype)` instead.\n",
      "  if is_sparse(data):\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<style>#sk-container-id-2 {color: black;}#sk-container-id-2 pre{padding: 0;}#sk-container-id-2 div.sk-toggleable {background-color: white;}#sk-container-id-2 label.sk-toggleable__label {cursor: pointer;display: block;width: 100%;margin-bottom: 0;padding: 0.3em;box-sizing: border-box;text-align: center;}#sk-container-id-2 label.sk-toggleable__label-arrow:before {content: \"▸\";float: left;margin-right: 0.25em;color: #696969;}#sk-container-id-2 label.sk-toggleable__label-arrow:hover:before {color: black;}#sk-container-id-2 div.sk-estimator:hover label.sk-toggleable__label-arrow:before {color: black;}#sk-container-id-2 div.sk-toggleable__content {max-height: 0;max-width: 0;overflow: hidden;text-align: left;background-color: #f0f8ff;}#sk-container-id-2 div.sk-toggleable__content pre {margin: 0.2em;color: black;border-radius: 0.25em;background-color: #f0f8ff;}#sk-container-id-2 input.sk-toggleable__control:checked~div.sk-toggleable__content {max-height: 200px;max-width: 100%;overflow: auto;}#sk-container-id-2 input.sk-toggleable__control:checked~label.sk-toggleable__label-arrow:before {content: \"▾\";}#sk-container-id-2 div.sk-estimator input.sk-toggleable__control:checked~label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-2 div.sk-label input.sk-toggleable__control:checked~label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-2 input.sk-hidden--visually {border: 0;clip: rect(1px 1px 1px 1px);clip: rect(1px, 1px, 1px, 1px);height: 1px;margin: -1px;overflow: hidden;padding: 0;position: absolute;width: 1px;}#sk-container-id-2 div.sk-estimator {font-family: monospace;background-color: #f0f8ff;border: 1px dotted black;border-radius: 0.25em;box-sizing: border-box;margin-bottom: 0.5em;}#sk-container-id-2 div.sk-estimator:hover {background-color: #d4ebff;}#sk-container-id-2 div.sk-parallel-item::after {content: \"\";width: 100%;border-bottom: 1px solid gray;flex-grow: 1;}#sk-container-id-2 div.sk-label:hover label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-2 div.sk-serial::before {content: \"\";position: absolute;border-left: 1px solid gray;box-sizing: border-box;top: 0;bottom: 0;left: 50%;z-index: 0;}#sk-container-id-2 div.sk-serial {display: flex;flex-direction: column;align-items: center;background-color: white;padding-right: 0.2em;padding-left: 0.2em;position: relative;}#sk-container-id-2 div.sk-item {position: relative;z-index: 1;}#sk-container-id-2 div.sk-parallel {display: flex;align-items: stretch;justify-content: center;background-color: white;position: relative;}#sk-container-id-2 div.sk-item::before, #sk-container-id-2 div.sk-parallel-item::before {content: \"\";position: absolute;border-left: 1px solid gray;box-sizing: border-box;top: 0;bottom: 0;left: 50%;z-index: -1;}#sk-container-id-2 div.sk-parallel-item {display: flex;flex-direction: column;z-index: 1;position: relative;background-color: white;}#sk-container-id-2 div.sk-parallel-item:first-child::after {align-self: flex-end;width: 50%;}#sk-container-id-2 div.sk-parallel-item:last-child::after {align-self: flex-start;width: 50%;}#sk-container-id-2 div.sk-parallel-item:only-child::after {width: 0;}#sk-container-id-2 div.sk-dashed-wrapped {border: 1px dashed gray;margin: 0 0.4em 0.5em 0.4em;box-sizing: border-box;padding-bottom: 0.4em;background-color: white;}#sk-container-id-2 div.sk-label label {font-family: monospace;font-weight: bold;display: inline-block;line-height: 1.2em;}#sk-container-id-2 div.sk-label-container {text-align: center;}#sk-container-id-2 div.sk-container {/* jupyter's `normalize.less` sets `[hidden] { display: none; }` but bootstrap.min.css set `[hidden] { display: none !important; }` so we also need the `!important` here to be able to override the default hidden behavior on the sphinx rendered scikit-learn.org. See: https://github.com/scikit-learn/scikit-learn/issues/21755 */display: inline-block !important;position: relative;}#sk-container-id-2 div.sk-text-repr-fallback {display: none;}</style><div id=\"sk-container-id-2\" class=\"sk-top-container\"><div class=\"sk-text-repr-fallback\"><pre>MotanRegression(reg_name=&#x27;XGBRegressor&#x27;)</pre><b>In a Jupyter environment, please rerun this cell to show the HTML representation or trust the notebook. <br />On GitHub, the HTML representation is unable to render, please try loading this page with nbviewer.org.</b></div><div class=\"sk-container\" hidden><div class=\"sk-item\"><div class=\"sk-estimator sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"sk-estimator-id-2\" type=\"checkbox\" checked><label for=\"sk-estimator-id-2\" class=\"sk-toggleable__label sk-toggleable__label-arrow\">MotanRegression</label><div class=\"sk-toggleable__content\"><pre>MotanRegression(reg_name=&#x27;XGBRegressor&#x27;)</pre></div></div></div></div></div>"
      ],
      "text/plain": [
       "MotanRegression(reg_name='XGBRegressor')"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.fit(X_train, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/noorainkazmi/miniforge3/envs/viewser/lib/python3.9/site-packages/xgboost/data.py:297: FutureWarning: is_sparse is deprecated and will be removed in a future version. Check `isinstance(dtype, pd.SparseDtype)` instead.\n",
      "  if is_sparse(dtype):\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/noorainkazmi/miniforge3/envs/viewser/lib/python3.9/site-packages/xgboost/data.py:299: FutureWarning: is_categorical_dtype is deprecated and will be removed in a future version. Use isinstance(dtype, CategoricalDtype) instead\n",
      "  elif is_categorical_dtype(dtype) and enable_categorical:\n"
     ]
    }
   ],
   "source": [
    "from sklearn.utils.validation import check_is_fitted\n",
    "\n",
    "predictions = model.predict(X_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([3.8599923, 2.1096075, 1.9183037, ..., 2.7897387, 3.513937 ,\n",
       "       3.4430902], dtype=float32)"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "predictions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "10103"
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(predictions)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " .    "
     ]
    }
   ],
   "source": [
    "df = Queryset(\"month_to_annual_moving_sum\", \"country_month\").fetch()\n",
    "df = df.query('country_id != [239,240,208,230,59]')\n",
    "# Remove countries that don't exist anymore\n",
    "df = df.query('country_id!=185')  # German Democratic Republic\n",
    "# Remove country months that have value zero for dependent variable\n",
    "df = df.query('wdi_sh_dyn_mort_fe!=0')\n",
    "fixed_effect_column = 'country_id'\n",
    "df[fixed_effect_column] = df.index.get_level_values(fixed_effect_column)\n",
    "fixed_effect_column = 'month_id'\n",
    "df[fixed_effect_column] = df.index.get_level_values(fixed_effect_column)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/noorainkazmi/miniforge3/envs/viewser/lib/python3.9/site-packages/xgboost/data.py:297: FutureWarning: is_sparse is deprecated and will be removed in a future version. Check `isinstance(dtype, pd.SparseDtype)` instead.\n",
      "  if is_sparse(dtype):\n",
      "/Users/noorainkazmi/miniforge3/envs/viewser/lib/python3.9/site-packages/xgboost/data.py:299: FutureWarning: is_categorical_dtype is deprecated and will be removed in a future version. Use isinstance(dtype, CategoricalDtype) instead\n",
      "  elif is_categorical_dtype(dtype) and enable_categorical:\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "array([3.0386906, 2.7429001, 2.7421808, ..., 3.736035 , 3.782715 ,\n",
       "       3.997299 ], dtype=float32)"
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.predict(df[independent_variable].query('month_id > 517'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.18"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}