80_perf_summary.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "395c2ed9-318e-4be2-886b-61282d2085ba",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Modify the file 00_setup.py to define input/output file paths on your system\n",
    "# The information in 00_setup.py will be used across notebooks\n",
    "from importlib.machinery import SourceFileLoader\n",
    "setup = SourceFileLoader(\"setup\", \"./00_setup.py\").load_module()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "65368748-0ca7-44cc-a40f-09ae004de061",
   "metadata": {},
   "source": [
    "# Performance Summary\n",
    "Combine performance of models tested, and summarize"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "4beb4e5b-70f3-4ea8-9c75-287a7b0f4b0f",
   "metadata": {},
   "source": [
    "Also combine all metrics for easy comparisons"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "4ba148a6-16e3-4d60-9478-92aef8bb030f",
   "metadata": {},
   "source": [
    "*This script takes about 1 minutes on my MacBook Air*"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "ebb9974c-a2ab-4b1e-96f9-86f7696d4e68",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "4469060d-2483-4d27-8a0c-44d857962276",
   "metadata": {},
   "outputs": [],
   "source": [
    "from pathlib import Path"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "7fc1bcc0-2d16-44f8-83d4-4b8acc59d6b2",
   "metadata": {},
   "outputs": [],
   "source": [
    "from sba_gnn.sba_gnn import sg_plot "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "3db5b8db-1478-4f38-bc6f-e68891f395fa",
   "metadata": {},
   "outputs": [],
   "source": [
    "sg_plot.plot_defaults()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e7ab511d-db43-4643-9b2b-eeb8c7871c97",
   "metadata": {},
   "source": [
    "## File Dictionaries"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "5d02286b-85fb-4a63-982f-fc293d74067a",
   "metadata": {},
   "source": [
    "##### XGB basic tests"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "d12dd83c-e49e-47f7-b7b1-aab34f89a51c",
   "metadata": {},
   "outputs": [],
   "source": [
    "dict_pred_1_xgb = {\n",
    "    'xgb base': Path(setup.temp_path).joinpath('03_DATA_combined_predictions.parquet'),\n",
    "    'xgb mean':  Path(setup.temp_path).joinpath('04_DATA_combined_predictions.parquet'),\n",
    "    'xgb one hot':  Path(setup.temp_path).joinpath('05_DATA_combined_predictions.parquet'),\n",
    "    'xgb numeric naics':  Path(setup.temp_path).joinpath('06_DATA_combined_predictions.parquet'),\n",
    "    'xgb mean':  Path(setup.temp_path).joinpath('07_DATA_combined_predictions.parquet'),\n",
    "    'xgb hier': Path(setup.temp_path).joinpath('10_DATA_combined_predictions.parquet')\n",
    "}"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ee2b8033-2a49-47d1-95fc-3ad06964c28d",
   "metadata": {},
   "source": [
    "##### Neural network basic"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "354a0264-78fb-4e19-b0ab-76dd23f02e75",
   "metadata": {},
   "outputs": [],
   "source": [
    "dict_pred_2_nn_basic = {\n",
    "    'nn base': Path(setup.temp_path).joinpath('21_DATA_predictions.parquet'),\n",
    "    'nn emb': Path(setup.temp_path).joinpath('22_DATA_predictions.parquet'),\n",
    "    'nn menc': Path(setup.temp_path).joinpath('23_DATA_predictions.parquet'),\n",
    "    'nn hier': Path(setup.temp_path).joinpath('24_DATA_predictions.parquet'),\n",
    "    'nn emb remap': Path(setup.temp_path).joinpath('25_DATA_predictions.parquet'),\n",
    "}"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "7b75fce6-4bd7-4128-9400-9c8384ecded1",
   "metadata": {},
   "source": [
    "##### XGB + embeddings"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "6b00e2e6-882d-4410-9f5b-e8e8131b7deb",
   "metadata": {},
   "outputs": [],
   "source": [
    "dict_pred_3_emb = {\n",
    "    'xgb emb nn':  Path(setup.temp_path).joinpath('40_DATA_combined_predictions.parquet'),\n",
    "    'xgb dgi base':  Path(setup.temp_path).joinpath('50_DATA_combined_predictions.parquet'),\n",
    "    'xgb dgi base+label':  Path(setup.temp_path).joinpath('52_DATA_combined_predictions.parquet')\n",
    "}"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "aba059c1-61a5-4b52-a1bd-0282275156c8",
   "metadata": {},
   "source": [
    "##### Clustering - Mean Encoding"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "f9e4467a-5937-4164-bfdf-458f583b2b16",
   "metadata": {},
   "outputs": [],
   "source": [
    "dict_pred_4_clus_menc = {\n",
    "    'xgb clus menc nn':  Path(setup.temp_path).joinpath('62_DATA_combined_predictions.parquet'),\n",
    "    'xgb clus dgi hier':  Path(setup.temp_path).joinpath('65_DATA_combined_predictions.parquet')\n",
    "}"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "9228e5a6-c885-444b-b1f1-10d72a5ae4d5",
   "metadata": {},
   "source": [
    "##### Clustering - One Hot Encoding"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "9b82c171-3d1a-4683-8918-3dcb4fe85807",
   "metadata": {},
   "outputs": [],
   "source": [
    "dict_pred_5_oh = {\n",
    "    'xgb clus oh nn':  Path(setup.temp_path).joinpath('61_DATA_combined_predictions.parquet'),\n",
    "    'xgb clus oh dgi base':  Path(setup.temp_path).joinpath('64_DATA_combined_predictions.parquet'),\n",
    "    'xgb clus oh dgi base+label':  Path(setup.temp_path).joinpath('71_DATA_combined_predictions.parquet')\n",
    "}"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "65573c39-fef8-4af8-88fa-8c09fc40f2de",
   "metadata": {},
   "source": [
    "## Loop Dictionaries\n",
    "Some scripts have multiple models - pull these in a little differently"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "0d591cd5-dc8f-4b3b-bac4-cc64bc8f62e8",
   "metadata": {},
   "outputs": [],
   "source": [
    "dict_loop = {\n",
    "        'xgb hier vary start':  Path(setup.temp_path).joinpath('11_DATA_combined_predictions.parquet'),\n",
    "        'xgb hier pair':  Path(setup.temp_path).joinpath('12_DATA_combined_predictions.parquet'),\n",
    "        'xgb dgi hier v start':  Path(setup.temp_path).joinpath('66_DATA_combined_predictions.parquet'),\n",
    "        'xgb dgi hier v start (alt)':  Path(setup.temp_path).joinpath('67_DATA_combined_predictions.parquet'),\n",
    "        'xgb dgi hier pair':  Path(setup.temp_path).joinpath('68_DATA_combined_predictions.parquet')\n",
    "}"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "2a2394e6-85bd-4518-84e4-b69e19cd2566",
   "metadata": {},
   "source": [
    "## Combine predictions"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "5cb4b0ac-58ef-4c0f-927a-308084705bea",
   "metadata": {},
   "source": [
    "##### Get dictionary of all files"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "c4142fc5-b944-4d60-836d-ddc2ca5a6a25",
   "metadata": {},
   "outputs": [],
   "source": [
    "all_dict_list = [dict_pred_1_xgb, dict_pred_2_nn_basic, \n",
    "                 dict_pred_3_emb, dict_pred_4_clus_menc, dict_pred_5_oh, dict_loop]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "35464cb3-1895-482b-82c7-0dc1736335fb",
   "metadata": {},
   "outputs": [],
   "source": [
    "dict_pred= {k: v for d in all_dict_list for k, v in d.items()}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "d12441f9-882a-4993-b504-b017242d52ba",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Verify these all exist\n",
    "all([Path(v).exists() for k, v in dict_pred.items()])"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "1b76c3d9-1b06-4ffe-a27b-2e98a1048bd0",
   "metadata": {},
   "source": [
    "##### Standard metrics\n",
    "Get filenames for standard metrics calculated in all scripts. These all have similar file naming"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "c5a8782d-6437-4cb0-b9e4-f882f6a87cf4",
   "metadata": {},
   "outputs": [],
   "source": [
    "dict_metrics = {k:Path(setup.temp_path).joinpath(v.name[0:2] + '_REPORT_metrics.csv') \\\n",
    "                for k, v in dict_pred.items()}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "44006664-7b23-478a-8c22-34267e55fa2b",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Verify these all exist\n",
    "all([Path(v).exists() for k, v in dict_metrics.items()])"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "3a1ef909-9a12-4734-ab34-4c90675e7ded",
   "metadata": {},
   "source": [
    "## Combine standard metrics"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "3135c12f-f97f-464d-bee8-b08b666e0bfb",
   "metadata": {},
   "outputs": [],
   "source": [
    "all_metrics = pd.concat([pd.read_csv(v) for k, v in dict_metrics.items()],\n",
    "                        keys = dict_metrics.keys()) \\\n",
    "    .reset_index(level=0) \\\n",
    "    .drop(columns=['Unnamed: 0'], errors='ignore') \\\n",
    "    .rename(columns={'level_0':'model'})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "51353b86-465d-4cb6-803e-cede1f644cb2",
   "metadata": {},
   "outputs": [],
   "source": [
    "all_metrics['script_num'] = all_metrics['model'].apply(lambda x: dict_metrics[x].name[0:2])"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "149ec0a0-05dd-48b4-83f5-d997b78a75d9",
   "metadata": {},
   "source": [
    "##### Modify loop field names"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "20c0ddab-aa3d-4e26-9f41-1b313d6fa248",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>model</th>\n",
       "      <th>dset</th>\n",
       "      <th>accuracy_score</th>\n",
       "      <th>f1_score</th>\n",
       "      <th>precision_score</th>\n",
       "      <th>recall_score</th>\n",
       "      <th>average_precision_score</th>\n",
       "      <th>roc_auc_score</th>\n",
       "      <th>dset_naics_holdout</th>\n",
       "      <th>start_naics</th>\n",
       "      <th>script_num</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>xgb hier vary start</td>\n",
       "      <td>test</td>\n",
       "      <td>0.674488</td>\n",
       "      <td>0.442278</td>\n",
       "      <td>0.332824</td>\n",
       "      <td>0.659000</td>\n",
       "      <td>0.376333</td>\n",
       "      <td>0.731086</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NAICS_5</td>\n",
       "      <td>11</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>xgb hier vary start</td>\n",
       "      <td>train</td>\n",
       "      <td>0.664677</td>\n",
       "      <td>0.454880</td>\n",
       "      <td>0.340835</td>\n",
       "      <td>0.683622</td>\n",
       "      <td>0.392977</td>\n",
       "      <td>0.735772</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NAICS_5</td>\n",
       "      <td>11</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>xgb hier vary start</td>\n",
       "      <td>val</td>\n",
       "      <td>0.661267</td>\n",
       "      <td>0.451350</td>\n",
       "      <td>0.339903</td>\n",
       "      <td>0.671532</td>\n",
       "      <td>0.385388</td>\n",
       "      <td>0.726702</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NAICS_5</td>\n",
       "      <td>11</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>xgb hier vary start</td>\n",
       "      <td>test</td>\n",
       "      <td>0.660671</td>\n",
       "      <td>0.450608</td>\n",
       "      <td>0.337801</td>\n",
       "      <td>0.676533</td>\n",
       "      <td>0.382942</td>\n",
       "      <td>0.728478</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NAICS_5</td>\n",
       "      <td>11</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>xgb hier vary start</td>\n",
       "      <td>test</td>\n",
       "      <td>0.694337</td>\n",
       "      <td>0.428456</td>\n",
       "      <td>0.324481</td>\n",
       "      <td>0.630488</td>\n",
       "      <td>0.367300</td>\n",
       "      <td>0.732314</td>\n",
       "      <td>1.0</td>\n",
       "      <td>NAICS_5</td>\n",
       "      <td>11</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25</th>\n",
       "      <td>xgb dgi hier pair</td>\n",
       "      <td>test</td>\n",
       "      <td>0.672021</td>\n",
       "      <td>0.438179</td>\n",
       "      <td>0.329701</td>\n",
       "      <td>0.653041</td>\n",
       "      <td>0.371179</td>\n",
       "      <td>0.730046</td>\n",
       "      <td>NaN</td>\n",
       "      <td>cluster_003</td>\n",
       "      <td>68</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26</th>\n",
       "      <td>xgb dgi hier pair</td>\n",
       "      <td>train</td>\n",
       "      <td>0.666546</td>\n",
       "      <td>0.454316</td>\n",
       "      <td>0.341544</td>\n",
       "      <td>0.678267</td>\n",
       "      <td>0.393383</td>\n",
       "      <td>0.735652</td>\n",
       "      <td>NaN</td>\n",
       "      <td>cluster_003</td>\n",
       "      <td>68</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27</th>\n",
       "      <td>xgb dgi hier pair</td>\n",
       "      <td>val</td>\n",
       "      <td>0.662711</td>\n",
       "      <td>0.450020</td>\n",
       "      <td>0.340057</td>\n",
       "      <td>0.665085</td>\n",
       "      <td>0.385624</td>\n",
       "      <td>0.726945</td>\n",
       "      <td>NaN</td>\n",
       "      <td>cluster_003</td>\n",
       "      <td>68</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>28</th>\n",
       "      <td>xgb dgi hier pair</td>\n",
       "      <td>test</td>\n",
       "      <td>0.661687</td>\n",
       "      <td>0.448672</td>\n",
       "      <td>0.337451</td>\n",
       "      <td>0.669250</td>\n",
       "      <td>0.383301</td>\n",
       "      <td>0.728486</td>\n",
       "      <td>0.0</td>\n",
       "      <td>cluster_003</td>\n",
       "      <td>68</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29</th>\n",
       "      <td>xgb dgi hier pair</td>\n",
       "      <td>test</td>\n",
       "      <td>0.686866</td>\n",
       "      <td>0.421077</td>\n",
       "      <td>0.317055</td>\n",
       "      <td>0.626681</td>\n",
       "      <td>0.352150</td>\n",
       "      <td>0.731225</td>\n",
       "      <td>1.0</td>\n",
       "      <td>cluster_003</td>\n",
       "      <td>68</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>120 rows × 11 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                  model   dset  accuracy_score  f1_score  precision_score  \\\n",
       "0   xgb hier vary start   test        0.674488  0.442278         0.332824   \n",
       "1   xgb hier vary start  train        0.664677  0.454880         0.340835   \n",
       "2   xgb hier vary start    val        0.661267  0.451350         0.339903   \n",
       "3   xgb hier vary start   test        0.660671  0.450608         0.337801   \n",
       "4   xgb hier vary start   test        0.694337  0.428456         0.324481   \n",
       "..                  ...    ...             ...       ...              ...   \n",
       "25    xgb dgi hier pair   test        0.672021  0.438179         0.329701   \n",
       "26    xgb dgi hier pair  train        0.666546  0.454316         0.341544   \n",
       "27    xgb dgi hier pair    val        0.662711  0.450020         0.340057   \n",
       "28    xgb dgi hier pair   test        0.661687  0.448672         0.337451   \n",
       "29    xgb dgi hier pair   test        0.686866  0.421077         0.317055   \n",
       "\n",
       "    recall_score  average_precision_score  roc_auc_score  dset_naics_holdout  \\\n",
       "0       0.659000                 0.376333       0.731086                 NaN   \n",
       "1       0.683622                 0.392977       0.735772                 NaN   \n",
       "2       0.671532                 0.385388       0.726702                 NaN   \n",
       "3       0.676533                 0.382942       0.728478                 0.0   \n",
       "4       0.630488                 0.367300       0.732314                 1.0   \n",
       "..           ...                      ...            ...                 ...   \n",
       "25      0.653041                 0.371179       0.730046                 NaN   \n",
       "26      0.678267                 0.393383       0.735652                 NaN   \n",
       "27      0.665085                 0.385624       0.726945                 NaN   \n",
       "28      0.669250                 0.383301       0.728486                 0.0   \n",
       "29      0.626681                 0.352150       0.731225                 1.0   \n",
       "\n",
       "    start_naics script_num  \n",
       "0       NAICS_5         11  \n",
       "1       NAICS_5         11  \n",
       "2       NAICS_5         11  \n",
       "3       NAICS_5         11  \n",
       "4       NAICS_5         11  \n",
       "..          ...        ...  \n",
       "25  cluster_003         68  \n",
       "26  cluster_003         68  \n",
       "27  cluster_003         68  \n",
       "28  cluster_003         68  \n",
       "29  cluster_003         68  \n",
       "\n",
       "[120 rows x 11 columns]"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "all_metrics[all_metrics['model'].isin(dict_loop.keys())]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "30a57ee2-6ae7-4086-9164-2d9dc6211eca",
   "metadata": {},
   "outputs": [],
   "source": [
    "all_metrics['model'] = np.where(all_metrics['model'].isin(dict_loop.keys()),\n",
    "                                all_metrics['model'] + ':' + all_metrics['start_naics'],\n",
    "                                all_metrics['model'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "b9bf6f42-a12e-4390-945a-fea2e83afac5",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>model</th>\n",
       "      <th>dset</th>\n",
       "      <th>accuracy_score</th>\n",
       "      <th>f1_score</th>\n",
       "      <th>precision_score</th>\n",
       "      <th>recall_score</th>\n",
       "      <th>average_precision_score</th>\n",
       "      <th>roc_auc_score</th>\n",
       "      <th>dset_naics_holdout</th>\n",
       "      <th>start_naics</th>\n",
       "      <th>script_num</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>27</th>\n",
       "      <td>xgb dgi hier pair:cluster_003</td>\n",
       "      <td>val</td>\n",
       "      <td>0.662711</td>\n",
       "      <td>0.450020</td>\n",
       "      <td>0.340057</td>\n",
       "      <td>0.665085</td>\n",
       "      <td>0.385624</td>\n",
       "      <td>0.726945</td>\n",
       "      <td>NaN</td>\n",
       "      <td>cluster_003</td>\n",
       "      <td>68</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>28</th>\n",
       "      <td>xgb dgi hier pair:cluster_003</td>\n",
       "      <td>test</td>\n",
       "      <td>0.661687</td>\n",
       "      <td>0.448672</td>\n",
       "      <td>0.337451</td>\n",
       "      <td>0.669250</td>\n",
       "      <td>0.383301</td>\n",
       "      <td>0.728486</td>\n",
       "      <td>0.0</td>\n",
       "      <td>cluster_003</td>\n",
       "      <td>68</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29</th>\n",
       "      <td>xgb dgi hier pair:cluster_003</td>\n",
       "      <td>test</td>\n",
       "      <td>0.686866</td>\n",
       "      <td>0.421077</td>\n",
       "      <td>0.317055</td>\n",
       "      <td>0.626681</td>\n",
       "      <td>0.352150</td>\n",
       "      <td>0.731225</td>\n",
       "      <td>1.0</td>\n",
       "      <td>cluster_003</td>\n",
       "      <td>68</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                            model  dset  accuracy_score  f1_score  \\\n",
       "27  xgb dgi hier pair:cluster_003   val        0.662711  0.450020   \n",
       "28  xgb dgi hier pair:cluster_003  test        0.661687  0.448672   \n",
       "29  xgb dgi hier pair:cluster_003  test        0.686866  0.421077   \n",
       "\n",
       "    precision_score  recall_score  average_precision_score  roc_auc_score  \\\n",
       "27         0.340057      0.665085                 0.385624       0.726945   \n",
       "28         0.337451      0.669250                 0.383301       0.728486   \n",
       "29         0.317055      0.626681                 0.352150       0.731225   \n",
       "\n",
       "    dset_naics_holdout  start_naics script_num  \n",
       "27                 NaN  cluster_003         68  \n",
       "28                 0.0  cluster_003         68  \n",
       "29                 1.0  cluster_003         68  "
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "all_metrics.tail(3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "a14b8f8c-dc59-4b15-89c6-c9c4984b9e58",
   "metadata": {},
   "outputs": [],
   "source": [
    "all_metrics.to_csv(Path(setup.temp_path).joinpath('80_REPORT_metrics_combined.csv'), index=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "1886ccd1-2995-44b7-a328-699cf6d3a294",
   "metadata": {},
   "source": [
    "## Summary Data\n",
    "For test dataset only, show AUC and f1 scores for test dataset, along with f1 scores for holdout and low-volume NAICS"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "110e1837-38c6-4c99-9351-dd0ac5292942",
   "metadata": {},
   "outputs": [],
   "source": [
    "overall_stats =  all_metrics[(all_metrics['dset'] == 'test') & \\\n",
    "                             (all_metrics['dset_naics_holdout'] == 0)] \\\n",
    "    [['script_num', 'model', 'roc_auc_score', 'average_precision_score', 'f1_score', ]]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "a2632258-10af-41e0-b3f3-86b42d3fc085",
   "metadata": {},
   "outputs": [],
   "source": [
    "holdout_stats = all_metrics[(all_metrics['dset'] == 'test') & \\\n",
    "                            (all_metrics['dset_naics_holdout'] == 1)] \\\n",
    "    [['model', 'roc_auc_score', 'average_precision_score', 'f1_score']] \\\n",
    "    .rename(columns={'f1_score':'f1_score_ho', 'average_precision_score':'ap_ho',\n",
    "                    'roc_auc_score':'roc_ho'})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "2c3d6aa2-6b51-43c1-a145-eaf4871d35e5",
   "metadata": {},
   "outputs": [],
   "source": [
    "overall_stats = overall_stats.merge(holdout_stats, on='model')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "0879e80d-18f8-47de-829f-20b7eda551f8",
   "metadata": {},
   "source": [
    "##### Selected stats"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "e223aa00-ce56-4e18-b7be-d346ad5a323c",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>script_num</th>\n",
       "      <th>model</th>\n",
       "      <th>roc_auc_score</th>\n",
       "      <th>average_precision_score</th>\n",
       "      <th>f1_score</th>\n",
       "      <th>roc_ho</th>\n",
       "      <th>ap_ho</th>\n",
       "      <th>f1_score_ho</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>07</td>\n",
       "      <td>xgb mean</td>\n",
       "      <td>0.730515</td>\n",
       "      <td>0.386990</td>\n",
       "      <td>0.451451</td>\n",
       "      <td>0.732690</td>\n",
       "      <td>0.354598</td>\n",
       "      <td>0.423457</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>40</td>\n",
       "      <td>xgb emb nn</td>\n",
       "      <td>0.730254</td>\n",
       "      <td>0.385856</td>\n",
       "      <td>0.451501</td>\n",
       "      <td>0.717489</td>\n",
       "      <td>0.331100</td>\n",
       "      <td>0.391092</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>36</th>\n",
       "      <td>68</td>\n",
       "      <td>xgb dgi hier pair:cluster_834</td>\n",
       "      <td>0.728603</td>\n",
       "      <td>0.384275</td>\n",
       "      <td>0.449924</td>\n",
       "      <td>0.728745</td>\n",
       "      <td>0.339113</td>\n",
       "      <td>0.427372</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26</th>\n",
       "      <td>66</td>\n",
       "      <td>xgb dgi hier v start:cluster_834</td>\n",
       "      <td>0.728585</td>\n",
       "      <td>0.384263</td>\n",
       "      <td>0.450125</td>\n",
       "      <td>0.712529</td>\n",
       "      <td>0.327377</td>\n",
       "      <td>0.408818</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>30</th>\n",
       "      <td>67</td>\n",
       "      <td>xgb dgi hier v start (alt):cluster_834</td>\n",
       "      <td>0.728585</td>\n",
       "      <td>0.384263</td>\n",
       "      <td>0.450125</td>\n",
       "      <td>0.712529</td>\n",
       "      <td>0.327377</td>\n",
       "      <td>0.408818</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>11</td>\n",
       "      <td>xgb hier vary start:NAICS_4</td>\n",
       "      <td>0.728771</td>\n",
       "      <td>0.384041</td>\n",
       "      <td>0.450069</td>\n",
       "      <td>0.737514</td>\n",
       "      <td>0.368883</td>\n",
       "      <td>0.433219</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>40</th>\n",
       "      <td>68</td>\n",
       "      <td>xgb dgi hier pair:cluster_010</td>\n",
       "      <td>0.728885</td>\n",
       "      <td>0.383998</td>\n",
       "      <td>0.449660</td>\n",
       "      <td>0.735481</td>\n",
       "      <td>0.356854</td>\n",
       "      <td>0.422107</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>34</th>\n",
       "      <td>67</td>\n",
       "      <td>xgb dgi hier v start (alt):cluster_010</td>\n",
       "      <td>0.728885</td>\n",
       "      <td>0.383998</td>\n",
       "      <td>0.449660</td>\n",
       "      <td>0.735481</td>\n",
       "      <td>0.356854</td>\n",
       "      <td>0.422107</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>11</td>\n",
       "      <td>xgb hier vary start:NAICS_3</td>\n",
       "      <td>0.728817</td>\n",
       "      <td>0.383981</td>\n",
       "      <td>0.449740</td>\n",
       "      <td>0.735148</td>\n",
       "      <td>0.358774</td>\n",
       "      <td>0.426627</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24</th>\n",
       "      <td>12</td>\n",
       "      <td>xgb hier pair:NAICS_3</td>\n",
       "      <td>0.728789</td>\n",
       "      <td>0.383948</td>\n",
       "      <td>0.449660</td>\n",
       "      <td>0.735078</td>\n",
       "      <td>0.358998</td>\n",
       "      <td>0.427180</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>39</th>\n",
       "      <td>68</td>\n",
       "      <td>xgb dgi hier pair:cluster_020</td>\n",
       "      <td>0.728759</td>\n",
       "      <td>0.383938</td>\n",
       "      <td>0.449631</td>\n",
       "      <td>0.736225</td>\n",
       "      <td>0.356929</td>\n",
       "      <td>0.424739</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29</th>\n",
       "      <td>66</td>\n",
       "      <td>xgb dgi hier v start:cluster_020</td>\n",
       "      <td>0.728759</td>\n",
       "      <td>0.383938</td>\n",
       "      <td>0.449631</td>\n",
       "      <td>0.736225</td>\n",
       "      <td>0.356929</td>\n",
       "      <td>0.424739</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   script_num                                   model  roc_auc_score  \\\n",
       "1          07                                xgb mean       0.730515   \n",
       "10         40                              xgb emb nn       0.730254   \n",
       "36         68           xgb dgi hier pair:cluster_834       0.728603   \n",
       "26         66        xgb dgi hier v start:cluster_834       0.728585   \n",
       "30         67  xgb dgi hier v start (alt):cluster_834       0.728585   \n",
       "19         11             xgb hier vary start:NAICS_4       0.728771   \n",
       "40         68           xgb dgi hier pair:cluster_010       0.728885   \n",
       "34         67  xgb dgi hier v start (alt):cluster_010       0.728885   \n",
       "20         11             xgb hier vary start:NAICS_3       0.728817   \n",
       "24         12                   xgb hier pair:NAICS_3       0.728789   \n",
       "39         68           xgb dgi hier pair:cluster_020       0.728759   \n",
       "29         66        xgb dgi hier v start:cluster_020       0.728759   \n",
       "\n",
       "    average_precision_score  f1_score    roc_ho     ap_ho  f1_score_ho  \n",
       "1                  0.386990  0.451451  0.732690  0.354598     0.423457  \n",
       "10                 0.385856  0.451501  0.717489  0.331100     0.391092  \n",
       "36                 0.384275  0.449924  0.728745  0.339113     0.427372  \n",
       "26                 0.384263  0.450125  0.712529  0.327377     0.408818  \n",
       "30                 0.384263  0.450125  0.712529  0.327377     0.408818  \n",
       "19                 0.384041  0.450069  0.737514  0.368883     0.433219  \n",
       "40                 0.383998  0.449660  0.735481  0.356854     0.422107  \n",
       "34                 0.383998  0.449660  0.735481  0.356854     0.422107  \n",
       "20                 0.383981  0.449740  0.735148  0.358774     0.426627  \n",
       "24                 0.383948  0.449660  0.735078  0.358998     0.427180  \n",
       "39                 0.383938  0.449631  0.736225  0.356929     0.424739  \n",
       "29                 0.383938  0.449631  0.736225  0.356929     0.424739  "
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "overall_stats.sort_values('average_precision_score', ascending=False).head(12)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "id": "9f2e98f8-0849-43ba-b272-b7a44b751636",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>script_num</th>\n",
       "      <th>model</th>\n",
       "      <th>roc_auc_score</th>\n",
       "      <th>average_precision_score</th>\n",
       "      <th>f1_score</th>\n",
       "      <th>roc_ho</th>\n",
       "      <th>ap_ho</th>\n",
       "      <th>f1_score_ho</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>11</td>\n",
       "      <td>xgb hier vary start:NAICS_4</td>\n",
       "      <td>0.728771</td>\n",
       "      <td>0.384041</td>\n",
       "      <td>0.450069</td>\n",
       "      <td>0.737514</td>\n",
       "      <td>0.368883</td>\n",
       "      <td>0.433219</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23</th>\n",
       "      <td>12</td>\n",
       "      <td>xgb hier pair:NAICS_4</td>\n",
       "      <td>0.728652</td>\n",
       "      <td>0.383583</td>\n",
       "      <td>0.450148</td>\n",
       "      <td>0.737012</td>\n",
       "      <td>0.367824</td>\n",
       "      <td>0.430403</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>11</td>\n",
       "      <td>xgb hier vary start:NAICS_5</td>\n",
       "      <td>0.728478</td>\n",
       "      <td>0.382942</td>\n",
       "      <td>0.450608</td>\n",
       "      <td>0.732314</td>\n",
       "      <td>0.367300</td>\n",
       "      <td>0.428456</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>10</td>\n",
       "      <td>xgb hier</td>\n",
       "      <td>0.728478</td>\n",
       "      <td>0.382942</td>\n",
       "      <td>0.450608</td>\n",
       "      <td>0.732314</td>\n",
       "      <td>0.367300</td>\n",
       "      <td>0.428456</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>06</td>\n",
       "      <td>xgb numeric naics</td>\n",
       "      <td>0.723166</td>\n",
       "      <td>0.373796</td>\n",
       "      <td>0.446564</td>\n",
       "      <td>0.738718</td>\n",
       "      <td>0.362137</td>\n",
       "      <td>0.429141</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>64</td>\n",
       "      <td>xgb clus oh dgi base</td>\n",
       "      <td>0.711046</td>\n",
       "      <td>0.356489</td>\n",
       "      <td>0.438078</td>\n",
       "      <td>0.737600</td>\n",
       "      <td>0.360423</td>\n",
       "      <td>0.427910</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>24</td>\n",
       "      <td>nn hier</td>\n",
       "      <td>0.724727</td>\n",
       "      <td>0.375652</td>\n",
       "      <td>0.447923</td>\n",
       "      <td>0.732317</td>\n",
       "      <td>0.359838</td>\n",
       "      <td>0.430286</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24</th>\n",
       "      <td>12</td>\n",
       "      <td>xgb hier pair:NAICS_3</td>\n",
       "      <td>0.728789</td>\n",
       "      <td>0.383948</td>\n",
       "      <td>0.449660</td>\n",
       "      <td>0.735078</td>\n",
       "      <td>0.358998</td>\n",
       "      <td>0.427180</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>11</td>\n",
       "      <td>xgb hier vary start:NAICS_3</td>\n",
       "      <td>0.728817</td>\n",
       "      <td>0.383981</td>\n",
       "      <td>0.449740</td>\n",
       "      <td>0.735148</td>\n",
       "      <td>0.358774</td>\n",
       "      <td>0.426627</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22</th>\n",
       "      <td>12</td>\n",
       "      <td>xgb hier pair:NAICS_5</td>\n",
       "      <td>0.728569</td>\n",
       "      <td>0.383699</td>\n",
       "      <td>0.449787</td>\n",
       "      <td>0.730116</td>\n",
       "      <td>0.358658</td>\n",
       "      <td>0.422230</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>71</td>\n",
       "      <td>xgb clus oh dgi base+label</td>\n",
       "      <td>0.711826</td>\n",
       "      <td>0.359239</td>\n",
       "      <td>0.436635</td>\n",
       "      <td>0.737109</td>\n",
       "      <td>0.358599</td>\n",
       "      <td>0.428162</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29</th>\n",
       "      <td>66</td>\n",
       "      <td>xgb dgi hier v start:cluster_020</td>\n",
       "      <td>0.728759</td>\n",
       "      <td>0.383938</td>\n",
       "      <td>0.449631</td>\n",
       "      <td>0.736225</td>\n",
       "      <td>0.356929</td>\n",
       "      <td>0.424739</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   script_num                             model  roc_auc_score  \\\n",
       "19         11       xgb hier vary start:NAICS_4       0.728771   \n",
       "23         12             xgb hier pair:NAICS_4       0.728652   \n",
       "18         11       xgb hier vary start:NAICS_5       0.728478   \n",
       "4          10                          xgb hier       0.728478   \n",
       "3          06                 xgb numeric naics       0.723166   \n",
       "16         64              xgb clus oh dgi base       0.711046   \n",
       "8          24                           nn hier       0.724727   \n",
       "24         12             xgb hier pair:NAICS_3       0.728789   \n",
       "20         11       xgb hier vary start:NAICS_3       0.728817   \n",
       "22         12             xgb hier pair:NAICS_5       0.728569   \n",
       "17         71        xgb clus oh dgi base+label       0.711826   \n",
       "29         66  xgb dgi hier v start:cluster_020       0.728759   \n",
       "\n",
       "    average_precision_score  f1_score    roc_ho     ap_ho  f1_score_ho  \n",
       "19                 0.384041  0.450069  0.737514  0.368883     0.433219  \n",
       "23                 0.383583  0.450148  0.737012  0.367824     0.430403  \n",
       "18                 0.382942  0.450608  0.732314  0.367300     0.428456  \n",
       "4                  0.382942  0.450608  0.732314  0.367300     0.428456  \n",
       "3                  0.373796  0.446564  0.738718  0.362137     0.429141  \n",
       "16                 0.356489  0.438078  0.737600  0.360423     0.427910  \n",
       "8                  0.375652  0.447923  0.732317  0.359838     0.430286  \n",
       "24                 0.383948  0.449660  0.735078  0.358998     0.427180  \n",
       "20                 0.383981  0.449740  0.735148  0.358774     0.426627  \n",
       "22                 0.383699  0.449787  0.730116  0.358658     0.422230  \n",
       "17                 0.359239  0.436635  0.737109  0.358599     0.428162  \n",
       "29                 0.383938  0.449631  0.736225  0.356929     0.424739  "
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "overall_stats.sort_values('ap_ho', ascending=False).head(12)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "id": "b7850f44-08b6-4e2f-80a4-a20ff5acead5",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>script_num</th>\n",
       "      <th>model</th>\n",
       "      <th>roc_auc_score</th>\n",
       "      <th>average_precision_score</th>\n",
       "      <th>f1_score</th>\n",
       "      <th>roc_ho</th>\n",
       "      <th>ap_ho</th>\n",
       "      <th>f1_score_ho</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>22</td>\n",
       "      <td>nn emb</td>\n",
       "      <td>0.728407</td>\n",
       "      <td>0.383644</td>\n",
       "      <td>0.449330</td>\n",
       "      <td>0.716412</td>\n",
       "      <td>0.327247</td>\n",
       "      <td>0.395913</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>25</td>\n",
       "      <td>nn emb remap</td>\n",
       "      <td>0.727966</td>\n",
       "      <td>0.381034</td>\n",
       "      <td>0.451593</td>\n",
       "      <td>0.727708</td>\n",
       "      <td>0.343123</td>\n",
       "      <td>0.418752</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>23</td>\n",
       "      <td>nn menc</td>\n",
       "      <td>0.724844</td>\n",
       "      <td>0.376685</td>\n",
       "      <td>0.447950</td>\n",
       "      <td>0.725387</td>\n",
       "      <td>0.341251</td>\n",
       "      <td>0.417035</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>24</td>\n",
       "      <td>nn hier</td>\n",
       "      <td>0.724727</td>\n",
       "      <td>0.375652</td>\n",
       "      <td>0.447923</td>\n",
       "      <td>0.732317</td>\n",
       "      <td>0.359838</td>\n",
       "      <td>0.430286</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>21</td>\n",
       "      <td>nn base</td>\n",
       "      <td>0.699359</td>\n",
       "      <td>0.340217</td>\n",
       "      <td>0.432538</td>\n",
       "      <td>0.727186</td>\n",
       "      <td>0.341215</td>\n",
       "      <td>0.419558</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  script_num         model  roc_auc_score  average_precision_score  f1_score  \\\n",
       "6         22        nn emb       0.728407                 0.383644  0.449330   \n",
       "9         25  nn emb remap       0.727966                 0.381034  0.451593   \n",
       "7         23       nn menc       0.724844                 0.376685  0.447950   \n",
       "8         24       nn hier       0.724727                 0.375652  0.447923   \n",
       "5         21       nn base       0.699359                 0.340217  0.432538   \n",
       "\n",
       "     roc_ho     ap_ho  f1_score_ho  \n",
       "6  0.716412  0.327247     0.395913  \n",
       "9  0.727708  0.343123     0.418752  \n",
       "7  0.725387  0.341251     0.417035  \n",
       "8  0.732317  0.359838     0.430286  \n",
       "5  0.727186  0.341215     0.419558  "
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "overall_stats[overall_stats['model'].isin(list(dict_pred_2_nn_basic.keys()))] \\\n",
    "    .sort_values('average_precision_score', ascending=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "id": "9e84147c-8161-4019-8e80-2c5d100e9605",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>script_num</th>\n",
       "      <th>model</th>\n",
       "      <th>roc_auc_score</th>\n",
       "      <th>average_precision_score</th>\n",
       "      <th>f1_score</th>\n",
       "      <th>roc_ho</th>\n",
       "      <th>ap_ho</th>\n",
       "      <th>f1_score_ho</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>40</td>\n",
       "      <td>xgb emb nn</td>\n",
       "      <td>0.730254</td>\n",
       "      <td>0.385856</td>\n",
       "      <td>0.451501</td>\n",
       "      <td>0.717489</td>\n",
       "      <td>0.331100</td>\n",
       "      <td>0.391092</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>50</td>\n",
       "      <td>xgb dgi base</td>\n",
       "      <td>0.724011</td>\n",
       "      <td>0.375218</td>\n",
       "      <td>0.447336</td>\n",
       "      <td>0.733705</td>\n",
       "      <td>0.346721</td>\n",
       "      <td>0.425681</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>52</td>\n",
       "      <td>xgb dgi base+label</td>\n",
       "      <td>0.723915</td>\n",
       "      <td>0.373773</td>\n",
       "      <td>0.446626</td>\n",
       "      <td>0.735716</td>\n",
       "      <td>0.355020</td>\n",
       "      <td>0.429074</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   script_num               model  roc_auc_score  average_precision_score  \\\n",
       "10         40          xgb emb nn       0.730254                 0.385856   \n",
       "11         50        xgb dgi base       0.724011                 0.375218   \n",
       "12         52  xgb dgi base+label       0.723915                 0.373773   \n",
       "\n",
       "    f1_score    roc_ho     ap_ho  f1_score_ho  \n",
       "10  0.451501  0.717489  0.331100     0.391092  \n",
       "11  0.447336  0.733705  0.346721     0.425681  \n",
       "12  0.446626  0.735716  0.355020     0.429074  "
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "overall_stats[overall_stats['model'].isin(list(dict_pred_3_emb.keys()))] \\\n",
    "    .sort_values('average_precision_score', ascending=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "id": "897c0457-c8a1-4f20-8346-dbc7dbe71bf1",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>script_num</th>\n",
       "      <th>model</th>\n",
       "      <th>roc_auc_score</th>\n",
       "      <th>average_precision_score</th>\n",
       "      <th>f1_score</th>\n",
       "      <th>roc_ho</th>\n",
       "      <th>ap_ho</th>\n",
       "      <th>f1_score_ho</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>62</td>\n",
       "      <td>xgb clus menc nn</td>\n",
       "      <td>0.728576</td>\n",
       "      <td>0.383574</td>\n",
       "      <td>0.449642</td>\n",
       "      <td>0.714015</td>\n",
       "      <td>0.313323</td>\n",
       "      <td>0.393603</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>65</td>\n",
       "      <td>xgb clus dgi hier</td>\n",
       "      <td>0.726191</td>\n",
       "      <td>0.379871</td>\n",
       "      <td>0.448569</td>\n",
       "      <td>0.714484</td>\n",
       "      <td>0.329468</td>\n",
       "      <td>0.391496</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   script_num              model  roc_auc_score  average_precision_score  \\\n",
       "13         62   xgb clus menc nn       0.728576                 0.383574   \n",
       "14         65  xgb clus dgi hier       0.726191                 0.379871   \n",
       "\n",
       "    f1_score    roc_ho     ap_ho  f1_score_ho  \n",
       "13  0.449642  0.714015  0.313323     0.393603  \n",
       "14  0.448569  0.714484  0.329468     0.391496  "
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "overall_stats[overall_stats['model'].isin(dict_pred_4_clus_menc.keys())] \\\n",
    "    .sort_values('average_precision_score', ascending=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "id": "05e8bea6-1b1a-4694-956b-a6600150ae65",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>script_num</th>\n",
       "      <th>model</th>\n",
       "      <th>roc_auc_score</th>\n",
       "      <th>average_precision_score</th>\n",
       "      <th>f1_score</th>\n",
       "      <th>roc_ho</th>\n",
       "      <th>ap_ho</th>\n",
       "      <th>f1_score_ho</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>61</td>\n",
       "      <td>xgb clus oh nn</td>\n",
       "      <td>0.726223</td>\n",
       "      <td>0.374882</td>\n",
       "      <td>0.448602</td>\n",
       "      <td>0.727376</td>\n",
       "      <td>0.343861</td>\n",
       "      <td>0.162614</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>71</td>\n",
       "      <td>xgb clus oh dgi base+label</td>\n",
       "      <td>0.711826</td>\n",
       "      <td>0.359239</td>\n",
       "      <td>0.436635</td>\n",
       "      <td>0.737109</td>\n",
       "      <td>0.358599</td>\n",
       "      <td>0.428162</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>05</td>\n",
       "      <td>xgb one hot</td>\n",
       "      <td>0.713595</td>\n",
       "      <td>0.358867</td>\n",
       "      <td>0.439001</td>\n",
       "      <td>0.733087</td>\n",
       "      <td>0.350732</td>\n",
       "      <td>0.423228</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>64</td>\n",
       "      <td>xgb clus oh dgi base</td>\n",
       "      <td>0.711046</td>\n",
       "      <td>0.356489</td>\n",
       "      <td>0.438078</td>\n",
       "      <td>0.737600</td>\n",
       "      <td>0.360423</td>\n",
       "      <td>0.427910</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   script_num                       model  roc_auc_score  \\\n",
       "15         61              xgb clus oh nn       0.726223   \n",
       "17         71  xgb clus oh dgi base+label       0.711826   \n",
       "2          05                 xgb one hot       0.713595   \n",
       "16         64        xgb clus oh dgi base       0.711046   \n",
       "\n",
       "    average_precision_score  f1_score    roc_ho     ap_ho  f1_score_ho  \n",
       "15                 0.374882  0.448602  0.727376  0.343861     0.162614  \n",
       "17                 0.359239  0.436635  0.737109  0.358599     0.428162  \n",
       "2                  0.358867  0.439001  0.733087  0.350732     0.423228  \n",
       "16                 0.356489  0.438078  0.737600  0.360423     0.427910  "
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "overall_stats[overall_stats['model'].isin(list(dict_pred_5_oh.keys()) + \\\n",
    "                                          ['xgb one hot'])] \\\n",
    "    .sort_values('average_precision_score', ascending=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "id": "95a02a52-09af-4777-809d-0f0be44afb6d",
   "metadata": {},
   "outputs": [],
   "source": [
    "overall_stats.to_csv(Path(setup.temp_path).joinpath('80_REPORT_summary_stats.csv'),\n",
    "                    index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "1101325b-203a-4016-8ca3-93f655b8f7e3",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "tf_p39",
   "language": "python",
   "name": "tf_p39"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.18"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}