69_xgb_dgi_mhier_loop_start_thresh.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "4c65f557-651f-4958-a567-a1b6b5a82b2b",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Modify the file 00_setup.py to define input/output file paths on your system\n",
    "# The information in 00_setup.py will be used across notebooks\n",
    "from importlib.machinery import SourceFileLoader\n",
    "setup = SourceFileLoader(\"setup\", \"./00_setup.py\").load_module()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b01d0fdb-d4bf-465a-8541-997addc50b0c",
   "metadata": {},
   "source": [
    "# XGBoost, Modified DGI Hierarchical - Start Level, Higher Threshold for Blending\n",
    "Do an alternative hierarchical blending, NAICS + hierarcy above a certain level.  \n",
    "Use NAICS-like groups"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "56161f89-1f54-47f9-860d-3f945a941c1f",
   "metadata": {},
   "source": [
    "*This script takes about 5 minutes on my MacBook Air*"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "ec6b1778-7abc-4129-ab5f-ac2992bd8ca6",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import xgboost as xgb\n",
    "import warnings\n",
    "import matplotlib.pyplot as plt\n",
    "import pickle"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "785af927-a634-4e34-84c4-7a532236f2ca",
   "metadata": {},
   "outputs": [],
   "source": [
    "from pathlib import Path"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "fd09527b-32dd-47fa-b62f-f9307f89f553",
   "metadata": {},
   "outputs": [],
   "source": [
    "import sklearn as sk\n",
    "from sklearn.model_selection import RandomizedSearchCV\n",
    "from sklearn.inspection import permutation_importance, partial_dependence\n",
    "from scikitplot.metrics import plot_lift_curve"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "0732ba40-8318-457b-b29f-f9b6de3b70aa",
   "metadata": {},
   "outputs": [],
   "source": [
    "from sba_gnn.sba_gnn import sg_plot, sg_blender\n",
    "from sba_gnn.sba_gnn.sg_blender import HierarchicalEncoder # Target encoder"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d14a5947-6599-4253-acf3-91ddeffdedd3",
   "metadata": {},
   "source": [
    "## Import Processed Datasets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "f6bc5cc5-b635-420f-852c-6cc9bba68a52",
   "metadata": {},
   "outputs": [],
   "source": [
    "sba_loans = pd.read_parquet(Path(setup.temp_path).joinpath('01_DATA_transformed.parquet')) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "62d59b9d-e575-4a1a-9709-931bba75ebf3",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['NoEmp', 'CreateJob', 'LowDoc', 'DisbursementGross', 'new_business', 'urban_flag', 'franchise_flag', 'mhier_NAICS_alt']\n"
     ]
    }
   ],
   "source": [
    "predictor_features = setup.predictor_features + ['mhier_NAICS_alt']\n",
    "print(predictor_features)\n",
    "with open(Path(setup.temp_path).joinpath('69_DATA_features_predict.pkl'), 'wb') as fout:\n",
    "    pickle.dump(predictor_features, fout)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "3c752967-2379-4bf3-9a0e-9eb1217012b2",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'subsample': 0.8,\n",
       " 'scale_pos_weight': 1.0,\n",
       " 'reg_lambda': 0.01,\n",
       " 'reg_alpha': 0.001,\n",
       " 'min_child_weight': 50,\n",
       " 'max_depth': 8,\n",
       " 'learning_rate': 0.1,\n",
       " 'gamma': 2}"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "best_params_df = pd.read_csv(Path(setup.temp_path).joinpath('03_REPORT_fit_parameter_selected.csv'))\n",
    "best_params = best_params_df.to_dict(orient='records')[0]\n",
    "best_params"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "82a0011b-c42a-4aca-bb30-7285fde059d4",
   "metadata": {},
   "outputs": [],
   "source": [
    "xgb_params = dict(best_params, \n",
    "                   **{'objective':\"binary:logistic\", 'n_estimators':setup.xgb_n_estimators})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "e4686a76-4fac-418a-bdf0-c2dbd731b944",
   "metadata": {},
   "outputs": [],
   "source": [
    "embed_df = pd.read_parquet(Path(setup.temp_path).joinpath('63_DATA_embeddings_tsne_naics.parquet'))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "63164775-c633-4ab7-bec8-06b88d1d66ea",
   "metadata": {},
   "source": [
    "##### NAICS levels to select NAICS-like clusters"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "3f42a928-366a-47d4-9692-40d3ba1eb8a5",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[20.0, 106.0, 354.0, 834.0]"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "naics_grp_stats = pd.read_csv(Path(setup.temp_path).joinpath('02_REPORT_naics_grp_stats_all.csv'))\n",
    "naics_grp_k = naics_grp_stats['count_grp'].drop_duplicates().sort_values().to_list()\n",
    "naics_grp_k"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "21962074-1182-4e18-8f34-9e5e1e6cd77b",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['cluster_834',\n",
       " 'cluster_354',\n",
       " 'cluster_106',\n",
       " 'cluster_020',\n",
       " 'cluster_010',\n",
       " 'cluster_003']"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cluster_cols = sorted([c for c in embed_df if c.startswith('cluster_')],\n",
    "                      reverse=True)\n",
    "cluster_cols"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "bd9b2d14-ac7b-42b1-843a-2931ea05aede",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['cluster_834', 'cluster_354', 'cluster_106', 'cluster_020']"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cluster_cols_sel = sorted([c for c in cluster_cols if int(c.split('_')[1]) in naics_grp_k],\n",
    "                         reverse=True)\n",
    "cluster_cols_sel"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "81a9f88b-1673-4cb9-ba26-f52270976094",
   "metadata": {},
   "outputs": [],
   "source": [
    "sba_loans = sba_loans.drop(columns = cluster_cols_sel, errors='ignore') \\\n",
    "    .merge(embed_df[['NAICS_orig'] + cluster_cols_sel] \\\n",
    "               .rename(columns={'NAICS_orig':'NAICS'}),\n",
    "           on='NAICS', how='left')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "a42cf208-024a-4ea9-ac9e-0a4e498b601e",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index(['LoanNr_ChkDgt', 'Name', 'City', 'State', 'Zip', 'Bank', 'BankState',\n",
       "       'NAICS', 'ApprovalDate', 'ApprovalFY', 'Term', 'NoEmp', 'NewExist',\n",
       "       'CreateJob', 'RetainedJob', 'FranchiseCode', 'UrbanRural', 'RevLineCr',\n",
       "       'LowDoc', 'ChgOffDate', 'DisbursementDate', 'DisbursementGross',\n",
       "       'BalanceGross', 'MIS_Status', 'ChgOffPrinGr', 'GrAppv', 'SBA_Appv',\n",
       "       'target', 'guaranteed_fract', 'new_business', 'urban_flag',\n",
       "       'franchise_flag', 'NAICS_2', 'NAICS_sector', 'NAICS_sector_desc',\n",
       "       'NAICS_num', 'dset', 'dset_naics_holdout', 'NAICS_5', 'NAICS_4',\n",
       "       'NAICS_3', 'menc_NAICS', 'menc_NAICS_5', 'menc_NAICS_4', 'menc_NAICS_3',\n",
       "       'menc_NAICS_sector', 'mhier_NAICS',\n",
       "       'NS___Accommodation and Food Services',\n",
       "       'NS___Administrative and Support and Waste Management and Remediation Services',\n",
       "       'NS___Construction', 'NS___Health Care and Social Assistance',\n",
       "       'NS___Manufacturing',\n",
       "       'NS___Other Services (except Public Administration)',\n",
       "       'NS___Professional, Scientific, and Technical Services',\n",
       "       'NS___Retail Trade', 'NS___Wholesale Trade', 'NS___infrequent_sklearn',\n",
       "       'cenc_NAICS', 'cluster_834', 'cluster_354', 'cluster_106',\n",
       "       'cluster_020'],\n",
       "      dtype='object')"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sba_loans.columns"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "dad22af6-70e7-46e9-ae24-c0af8de1b822",
   "metadata": {},
   "source": [
    "## Function to get metrics"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "ca448a6e-df88-4d10-b19b-ba01c484301a",
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_metrics(data, enc_features = ['NAICS', 'NAICS_sector']):\n",
    "    \n",
    "    # Alternative encoding\n",
    "    train_df = data[data['dset'] == 'train']\n",
    "    hier_enc = HierarchicalEncoder(lambda_k = 50, lambda_f=50)\n",
    "    hier_enc.fit(train_df[enc_features],train_df['target'])\n",
    "    hier_col = hier_enc.transform(data[enc_features]) \\\n",
    "        .rename('mhier_NAICS_alt')\n",
    "    \n",
    "    # Append to data\n",
    "    data = pd.concat([data.drop(columns='mhier_NAICS_alt', errors='ignore'), \n",
    "                       hier_col], axis=1)\n",
    "    train_df = data[data['dset'] == 'train']\n",
    "    test_df = data[data['dset'] == 'test']\n",
    "    val_df = data[data['dset'] == 'val']\n",
    "    \n",
    "    # Model fit\n",
    "    xgb_model = xgb.XGBClassifier(objective=\"binary:logistic\",\n",
    "                             n_estimators =  setup.xgb_n_estimators)\n",
    "    xgb_model = xgb_model.set_params(**xgb_params)\n",
    "    xgb_model.fit(train_df[predictor_features], train_df['target'],\n",
    "              eval_set = [(train_df[predictor_features], train_df['target']),\n",
    "                         (val_df[predictor_features], val_df['target'])])\n",
    "    \n",
    "    # Predict\n",
    "    pred_df = pd.concat([data[['LoanNr_ChkDgt', 'dset', 'target', 'dset_naics_holdout']].reset_index(),\n",
    "                     pd.DataFrame({'predict_prob':\n",
    "                                   xgb_model.predict_proba(data[predictor_features])[:,1]})],\n",
    "                    axis=1) \\\n",
    "    .set_index('index')\n",
    "    \n",
    "    # Decision threshold\n",
    "    pred_train_df = pred_df[pred_df['dset'] == 'train']\n",
    "    thresh_tune_data = sg_plot.get_f1_frame(pred_train_df['target'], \n",
    "                                        pred_train_df['predict_prob'])\n",
    "    thresh_head = thresh_tune_data.sort_values('f1', ascending=False).head(2)\n",
    "    best_thresh = thresh_head['thresh'].iloc[0]\n",
    "    pred_df['predict_bin'] = np.where(pred_df['predict_prob'] >= best_thresh, 1, 0)\n",
    "    \n",
    "    # Metrics\n",
    "    metrics_dset_df = pred_df.groupby('dset') \\\n",
    "        .apply(lambda x: sg_plot.dset_metrics(x.target, x.predict_bin, x.predict_prob)) \\\n",
    "        .reset_index()\n",
    "    metrics_test_df = pred_df[pred_df['dset'] == 'test'] \\\n",
    "        .groupby(['dset', 'dset_naics_holdout']) \\\n",
    "        .apply(lambda x: sg_plot.dset_metrics(x.target, x.predict_bin, x.predict_prob)) \\\n",
    "        .reset_index()\n",
    "    metrics_df = pd.concat([metrics_dset_df, metrics_test_df])\n",
    "    \n",
    "    return pred_df, metrics_df"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "9c290356-3291-4406-90cf-d921ad845925",
   "metadata": {},
   "source": [
    "## Loop to get predictions, metrics"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "be3db577-c4bf-410f-a4fb-d926368a0519",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[['NAICS', 'cluster_834', 'cluster_354', 'cluster_106', 'cluster_020'],\n",
       " ['NAICS', 'cluster_354', 'cluster_106', 'cluster_020'],\n",
       " ['NAICS', 'cluster_106', 'cluster_020'],\n",
       " ['NAICS', 'cluster_020']]"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Hierarchical encode from different levels\n",
    "naics_sets = [['NAICS'] + cluster_cols_sel[i:] \\\n",
    "              for i in range(len(cluster_cols_sel))]\n",
    "naics_sets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "3ca0672f-819d-4bf5-b829-a481a852d65b",
   "metadata": {},
   "outputs": [],
   "source": [
    "%%capture\n",
    "pred_df = pd.DataFrame()\n",
    "metrics_df = pd.DataFrame()\n",
    "for this_set in naics_sets:\n",
    "    this_pred, this_met = get_metrics(sba_loans, this_set)\n",
    "    this_pred['start_naics'] = this_set[1]\n",
    "    pred_df = pd.concat([pred_df, this_pred])\n",
    "    this_met['start_naics'] = this_set[1]\n",
    "    metrics_df = pd.concat([metrics_df, this_met])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "0ff21702-e129-418b-b6c3-8ebd047186a0",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>dset</th>\n",
       "      <th>accuracy_score</th>\n",
       "      <th>f1_score</th>\n",
       "      <th>precision_score</th>\n",
       "      <th>recall_score</th>\n",
       "      <th>average_precision_score</th>\n",
       "      <th>roc_auc_score</th>\n",
       "      <th>dset_naics_holdout</th>\n",
       "      <th>start_naics</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>test</td>\n",
       "      <td>0.661195</td>\n",
       "      <td>0.449617</td>\n",
       "      <td>0.337623</td>\n",
       "      <td>0.672788</td>\n",
       "      <td>0.383752</td>\n",
       "      <td>0.728656</td>\n",
       "      <td>0.0</td>\n",
       "      <td>cluster_834</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>test</td>\n",
       "      <td>0.714844</td>\n",
       "      <td>0.397923</td>\n",
       "      <td>0.322818</td>\n",
       "      <td>0.518569</td>\n",
       "      <td>0.329501</td>\n",
       "      <td>0.715799</td>\n",
       "      <td>1.0</td>\n",
       "      <td>cluster_834</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>test</td>\n",
       "      <td>0.661013</td>\n",
       "      <td>0.449446</td>\n",
       "      <td>0.337457</td>\n",
       "      <td>0.672684</td>\n",
       "      <td>0.383306</td>\n",
       "      <td>0.728258</td>\n",
       "      <td>0.0</td>\n",
       "      <td>cluster_354</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>test</td>\n",
       "      <td>0.712691</td>\n",
       "      <td>0.401345</td>\n",
       "      <td>0.322955</td>\n",
       "      <td>0.529989</td>\n",
       "      <td>0.335796</td>\n",
       "      <td>0.723581</td>\n",
       "      <td>1.0</td>\n",
       "      <td>cluster_354</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>test</td>\n",
       "      <td>0.659002</td>\n",
       "      <td>0.448441</td>\n",
       "      <td>0.336013</td>\n",
       "      <td>0.673932</td>\n",
       "      <td>0.383081</td>\n",
       "      <td>0.728441</td>\n",
       "      <td>0.0</td>\n",
       "      <td>cluster_106</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>test</td>\n",
       "      <td>0.701116</td>\n",
       "      <td>0.412545</td>\n",
       "      <td>0.320878</td>\n",
       "      <td>0.577532</td>\n",
       "      <td>0.347778</td>\n",
       "      <td>0.729401</td>\n",
       "      <td>1.0</td>\n",
       "      <td>cluster_106</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>test</td>\n",
       "      <td>0.659826</td>\n",
       "      <td>0.449688</td>\n",
       "      <td>0.336974</td>\n",
       "      <td>0.675701</td>\n",
       "      <td>0.383443</td>\n",
       "      <td>0.728383</td>\n",
       "      <td>0.0</td>\n",
       "      <td>cluster_020</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>test</td>\n",
       "      <td>0.691124</td>\n",
       "      <td>0.425077</td>\n",
       "      <td>0.321169</td>\n",
       "      <td>0.628373</td>\n",
       "      <td>0.357393</td>\n",
       "      <td>0.736805</td>\n",
       "      <td>1.0</td>\n",
       "      <td>cluster_020</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   dset  accuracy_score  f1_score  precision_score  recall_score  \\\n",
       "0  test        0.661195  0.449617         0.337623      0.672788   \n",
       "1  test        0.714844  0.397923         0.322818      0.518569   \n",
       "0  test        0.661013  0.449446         0.337457      0.672684   \n",
       "1  test        0.712691  0.401345         0.322955      0.529989   \n",
       "0  test        0.659002  0.448441         0.336013      0.673932   \n",
       "1  test        0.701116  0.412545         0.320878      0.577532   \n",
       "0  test        0.659826  0.449688         0.336974      0.675701   \n",
       "1  test        0.691124  0.425077         0.321169      0.628373   \n",
       "\n",
       "   average_precision_score  roc_auc_score  dset_naics_holdout  start_naics  \n",
       "0                 0.383752       0.728656                 0.0  cluster_834  \n",
       "1                 0.329501       0.715799                 1.0  cluster_834  \n",
       "0                 0.383306       0.728258                 0.0  cluster_354  \n",
       "1                 0.335796       0.723581                 1.0  cluster_354  \n",
       "0                 0.383081       0.728441                 0.0  cluster_106  \n",
       "1                 0.347778       0.729401                 1.0  cluster_106  \n",
       "0                 0.383443       0.728383                 0.0  cluster_020  \n",
       "1                 0.357393       0.736805                 1.0  cluster_020  "
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "metrics_df[~metrics_df['dset_naics_holdout'].isna()]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "68669385-ef28-4f52-8987-7cc43ac626a6",
   "metadata": {},
   "outputs": [],
   "source": [
    "pred_df.to_parquet(Path(setup.temp_path).joinpath('69_DATA_combined_predictions.parquet'))\n",
    "metrics_df.to_csv(Path(setup.temp_path).joinpath('69_REPORT_metrics.csv'), index=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "5488c384-feb9-45b4-8d95-e2ea4954719b",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "83c4bf72-5710-489a-888f-bd2b7590e93b",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e2b6ef2b-9fe4-4595-9ba9-7fc205a864f6",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "tf_p39",
   "language": "python",
   "name": "tf_p39"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.18"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}