Diabetes Readmission prediction using ANN.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "source": [
    "# Diabetes Readmission prediction using Artificial Neural Network (MultiLayerPerceptron) model"
   ],
   "metadata": {
    "application/vnd.databricks.v1+cell": {
     "showTitle": false,
     "cellMetadata": {},
     "nuid": "a627be25-9fcd-4b57-84d1-c9f11a155876",
     "inputWidgets": {},
     "title": ""
    }
   }
  },
  {
   "cell_type": "code",
   "source": [
    "# Data processing\n",
    "from pyspark.sql.functions import log, col, exp\n",
    "\n",
    "# Modeling\n",
    "from pyspark.ml.feature import VectorAssembler\n",
    "from pyspark.ml.classification import MultilayerPerceptronClassifier\n",
    "from pyspark.ml.evaluation import BinaryClassificationEvaluator\n"
   ],
   "metadata": {
    "application/vnd.databricks.v1+cell": {
     "showTitle": false,
     "cellMetadata": {
      "rowLimit": 10000,
      "byteLimit": 2048000
     },
     "nuid": "3c72ddea-d6b7-426b-bed6-861fb814ff4b",
     "inputWidgets": {},
     "title": ""
    }
   },
   "outputs": [],
   "execution_count": 1
  },
  {
   "cell_type": "code",
   "source": [
    "write_path = 'dbfs:/tmp/reproducible_ml_uofl/diab_readmit_csv.delta'\n",
    "diabetes_readmit = spark.read.format('delta').load(write_path)\n",
    "\n",
    "#Show basic summary stats\n",
    "display(diabetes_readmit.summary())"
   ],
   "metadata": {
    "application/vnd.databricks.v1+cell": {
     "showTitle": false,
     "cellMetadata": {
      "rowLimit": 10000,
      "byteLimit": 2048000
     },
     "nuid": "64021ba7-9975-4aa7-865c-813f95a25cdd",
     "inputWidgets": {},
     "title": ""
    }
   },
   "outputs": [
    {
     "output_type": "display_data",
     "metadata": {
      "application/vnd.databricks.v1+output": {
       "overflow": false,
       "datasetInfos": [
        {
         "name": "diabetes_readmit",
         "typeStr": "pyspark.sql.dataframe.DataFrame",
         "schema": {
          "fields": [
           {
            "metadata": {},
            "name": "patient_nbr",
            "nullable": true,
            "type": "integer"
           },
           {
            "metadata": {},
            "name": "time_in_hospital",
            "nullable": true,
            "type": "integer"
           },
           {
            "metadata": {},
            "name": "num_procedures",
            "nullable": true,
            "type": "integer"
           },
           {
            "metadata": {},
            "name": "num_lab_procedures",
            "nullable": true,
            "type": "integer"
           },
           {
            "metadata": {},
            "name": "num_medications",
            "nullable": true,
            "type": "integer"
           },
           {
            "metadata": {},
            "name": "number_outpatient",
            "nullable": true,
            "type": "integer"
           },
           {
            "metadata": {},
            "name": "number_inpatient",
            "nullable": true,
            "type": "integer"
           },
           {
            "metadata": {},
            "name": "number_emergency",
            "nullable": true,
            "type": "integer"
           },
           {
            "metadata": {},
            "name": "number_diagnoses",
            "nullable": true,
            "type": "integer"
           },
           {
            "metadata": {},
            "name": "gender_cd",
            "nullable": true,
            "type": "integer"
           },
           {
            "metadata": {},
            "name": "DiabetesMedication",
            "nullable": true,
            "type": "integer"
           },
           {
            "metadata": {},
            "name": "readmit_flag",
            "nullable": true,
            "type": "integer"
           },
           {
            "metadata": {},
            "name": "race_cd",
            "nullable": true,
            "type": "string"
           }
          ],
          "type": "struct"
         },
         "tableIdentifier": "dbfs:/tmp/reproducible_ml_uofl/diab_readmit_csv.delta"
        }
       ],
       "data": [
        [
         "count",
         "101766",
         "101766",
         "101766",
         "101766",
         "101766",
         "101766",
         "101766",
         "101766",
         "101766",
         "101766",
         "101766",
         "101766",
         "101766"
        ],
        [
         "mean",
         "5.4330400694947235E7",
         "4.395986871843248",
         "1.339730361810428",
         "43.09564098028811",
         "16.021844230882614",
         "0.36935715268360747",
         "0.635565906098304",
         "0.19783621248747127",
         "7.422606764538254",
         "0.5375862272271682",
         "0.7700312481575379",
         "0.11159915885462728",
         null
        ],
        [
         "stddev",
         "3.869635934653421E7",
         "2.985107767471267",
         "1.705806979121172",
         "19.674362249142096",
         "8.127566209167309",
         "1.2672650965326817",
         "1.26286329009732",
         "0.9304722684224632",
         "1.9336001449974247",
         "0.49858772375671534",
         "0.420814525814695",
         "0.3148741984505526",
         null
        ],
        [
         "min",
         "135",
         "1",
         "0",
         "1",
         "1",
         "0",
         "0",
         "0",
         "1",
         "0",
         "0",
         "0",
         "AfrAmr"
        ],
        [
         "25%",
         "23412645",
         "2",
         "0",
         "31",
         "10",
         "0",
         "0",
         "0",
         "6",
         "0",
         "1",
         "0",
         null
        ],
        [
         "50%",
         "45500490",
         "4",
         "1",
         "44",
         "15",
         "0",
         "0",
         "0",
         "8",
         "1",
         "1",
         "0",
         null
        ],
        [
         "75%",
         "87532902",
         "6",
         "2",
         "57",
         "20",
         "0",
         "1",
         "0",
         "9",
         "1",
         "1",
         "0",
         null
        ],
        [
         "max",
         "189502619",
         "14",
         "6",
         "132",
         "81",
         "42",
         "21",
         "76",
         "16",
         "1",
         "1",
         "1",
         "White"
        ]
       ],
       "plotOptions": {
        "displayType": "table",
        "customPlotOptions": {},
        "pivotColumns": null,
        "pivotAggregation": null,
        "xColumns": null,
        "yColumns": null
       },
       "columnCustomDisplayInfos": {},
       "aggType": "",
       "isJsonSchema": true,
       "removedWidgets": [],
       "aggSchema": [],
       "schema": [
        {
         "name": "summary",
         "type": "\"string\"",
         "metadata": "{}"
        },
        {
         "name": "patient_nbr",
         "type": "\"string\"",
         "metadata": "{}"
        },
        {
         "name": "time_in_hospital",
         "type": "\"string\"",
         "metadata": "{}"
        },
        {
         "name": "num_procedures",
         "type": "\"string\"",
         "metadata": "{}"
        },
        {
         "name": "num_lab_procedures",
         "type": "\"string\"",
         "metadata": "{}"
        },
        {
         "name": "num_medications",
         "type": "\"string\"",
         "metadata": "{}"
        },
        {
         "name": "number_outpatient",
         "type": "\"string\"",
         "metadata": "{}"
        },
        {
         "name": "number_inpatient",
         "type": "\"string\"",
         "metadata": "{}"
        },
        {
         "name": "number_emergency",
         "type": "\"string\"",
         "metadata": "{}"
        },
        {
         "name": "number_diagnoses",
         "type": "\"string\"",
         "metadata": "{}"
        },
        {
         "name": "gender_cd",
         "type": "\"string\"",
         "metadata": "{}"
        },
        {
         "name": "DiabetesMedication",
         "type": "\"string\"",
         "metadata": "{}"
        },
        {
         "name": "readmit_flag",
         "type": "\"string\"",
         "metadata": "{}"
        },
        {
         "name": "race_cd",
         "type": "\"string\"",
         "metadata": "{}"
        }
       ],
       "aggError": "",
       "aggData": [],
       "addedWidgets": {},
       "metadata": {},
       "dbfsResultPath": null,
       "type": "table",
       "aggOverflow": false,
       "aggSeriesLimitReached": false,
       "arguments": {}
      }
     },
     "data": {
      "text/html": [
       "<style scoped>\n",
       "  .table-result-container {\n",
       "    max-height: 300px;\n",
       "    overflow: auto;\n",
       "  }\n",
       "  table, th, td {\n",
       "    border: 1px solid black;\n",
       "    border-collapse: collapse;\n",
       "  }\n",
       "  th, td {\n",
       "    padding: 5px;\n",
       "  }\n",
       "  th {\n",
       "    text-align: left;\n",
       "  }\n",
       "</style><div class='table-result-container'><table class='table-result'><thead style='background-color: white'><tr><th>summary</th><th>patient_nbr</th><th>time_in_hospital</th><th>num_procedures</th><th>num_lab_procedures</th><th>num_medications</th><th>number_outpatient</th><th>number_inpatient</th><th>number_emergency</th><th>number_diagnoses</th><th>gender_cd</th><th>DiabetesMedication</th><th>readmit_flag</th><th>race_cd</th></tr></thead><tbody><tr><td>count</td><td>101766</td><td>101766</td><td>101766</td><td>101766</td><td>101766</td><td>101766</td><td>101766</td><td>101766</td><td>101766</td><td>101766</td><td>101766</td><td>101766</td><td>101766</td></tr><tr><td>mean</td><td>5.4330400694947235E7</td><td>4.395986871843248</td><td>1.339730361810428</td><td>43.09564098028811</td><td>16.021844230882614</td><td>0.36935715268360747</td><td>0.635565906098304</td><td>0.19783621248747127</td><td>7.422606764538254</td><td>0.5375862272271682</td><td>0.7700312481575379</td><td>0.11159915885462728</td><td>null</td></tr><tr><td>stddev</td><td>3.869635934653421E7</td><td>2.985107767471267</td><td>1.705806979121172</td><td>19.674362249142096</td><td>8.127566209167309</td><td>1.2672650965326817</td><td>1.26286329009732</td><td>0.9304722684224632</td><td>1.9336001449974247</td><td>0.49858772375671534</td><td>0.420814525814695</td><td>0.3148741984505526</td><td>null</td></tr><tr><td>min</td><td>135</td><td>1</td><td>0</td><td>1</td><td>1</td><td>0</td><td>0</td><td>0</td><td>1</td><td>0</td><td>0</td><td>0</td><td>AfrAmr</td></tr><tr><td>25%</td><td>23412645</td><td>2</td><td>0</td><td>31</td><td>10</td><td>0</td><td>0</td><td>0</td><td>6</td><td>0</td><td>1</td><td>0</td><td>null</td></tr><tr><td>50%</td><td>45500490</td><td>4</td><td>1</td><td>44</td><td>15</td><td>0</td><td>0</td><td>0</td><td>8</td><td>1</td><td>1</td><td>0</td><td>null</td></tr><tr><td>75%</td><td>87532902</td><td>6</td><td>2</td><td>57</td><td>20</td><td>0</td><td>1</td><td>0</td><td>9</td><td>1</td><td>1</td><td>0</td><td>null</td></tr><tr><td>max</td><td>189502619</td><td>14</td><td>6</td><td>132</td><td>81</td><td>42</td><td>21</td><td>76</td><td>16</td><td>1</td><td>1</td><td>1</td><td>White</td></tr></tbody></table></div>"
      ]
     }
    }
   ],
   "execution_count": 0
  },
  {
   "cell_type": "code",
   "source": [
    "# Train test split\n",
    "trainDF, testDF = diabetes_readmit.randomSplit([.65, .35], seed=42)\n",
    "# Print the number of records\n",
    "print(f'There are {trainDF.cache().count()} records in the training dataset.')\n",
    "print(f'There are {testDF.cache().count()} records in the testing dataset.')"
   ],
   "metadata": {
    "application/vnd.databricks.v1+cell": {
     "showTitle": false,
     "cellMetadata": {
      "rowLimit": 10000,
      "byteLimit": 2048000
     },
     "nuid": "cbca723d-70fc-46d9-b996-cb36638f4b6d",
     "inputWidgets": {},
     "title": ""
    }
   },
   "outputs": [
    {
     "output_type": "display_data",
     "metadata": {
      "application/vnd.databricks.v1+output": {
       "datasetInfos": [
        {
         "name": "trainDF",
         "typeStr": "pyspark.sql.dataframe.DataFrame",
         "schema": {
          "fields": [
           {
            "metadata": {},
            "name": "patient_nbr",
            "nullable": true,
            "type": "integer"
           },
           {
            "metadata": {},
            "name": "time_in_hospital",
            "nullable": true,
            "type": "integer"
           },
           {
            "metadata": {},
            "name": "num_procedures",
            "nullable": true,
            "type": "integer"
           },
           {
            "metadata": {},
            "name": "num_lab_procedures",
            "nullable": true,
            "type": "integer"
           },
           {
            "metadata": {},
            "name": "num_medications",
            "nullable": true,
            "type": "integer"
           },
           {
            "metadata": {},
            "name": "number_outpatient",
            "nullable": true,
            "type": "integer"
           },
           {
            "metadata": {},
            "name": "number_inpatient",
            "nullable": true,
            "type": "integer"
           },
           {
            "metadata": {},
            "name": "number_emergency",
            "nullable": true,
            "type": "integer"
           },
           {
            "metadata": {},
            "name": "number_diagnoses",
            "nullable": true,
            "type": "integer"
           },
           {
            "metadata": {},
            "name": "gender_cd",
            "nullable": true,
            "type": "integer"
           },
           {
            "metadata": {},
            "name": "DiabetesMedication",
            "nullable": true,
            "type": "integer"
           },
           {
            "metadata": {},
            "name": "readmit_flag",
            "nullable": true,
            "type": "integer"
           },
           {
            "metadata": {},
            "name": "race_cd",
            "nullable": true,
            "type": "string"
           }
          ],
          "type": "struct"
         },
         "tableIdentifier": null
        },
        {
         "name": "testDF",
         "typeStr": "pyspark.sql.dataframe.DataFrame",
         "schema": {
          "fields": [
           {
            "metadata": {},
            "name": "patient_nbr",
            "nullable": true,
            "type": "integer"
           },
           {
            "metadata": {},
            "name": "time_in_hospital",
            "nullable": true,
            "type": "integer"
           },
           {
            "metadata": {},
            "name": "num_procedures",
            "nullable": true,
            "type": "integer"
           },
           {
            "metadata": {},
            "name": "num_lab_procedures",
            "nullable": true,
            "type": "integer"
           },
           {
            "metadata": {},
            "name": "num_medications",
            "nullable": true,
            "type": "integer"
           },
           {
            "metadata": {},
            "name": "number_outpatient",
            "nullable": true,
            "type": "integer"
           },
           {
            "metadata": {},
            "name": "number_inpatient",
            "nullable": true,
            "type": "integer"
           },
           {
            "metadata": {},
            "name": "number_emergency",
            "nullable": true,
            "type": "integer"
           },
           {
            "metadata": {},
            "name": "number_diagnoses",
            "nullable": true,
            "type": "integer"
           },
           {
            "metadata": {},
            "name": "gender_cd",
            "nullable": true,
            "type": "integer"
           },
           {
            "metadata": {},
            "name": "DiabetesMedication",
            "nullable": true,
            "type": "integer"
           },
           {
            "metadata": {},
            "name": "readmit_flag",
            "nullable": true,
            "type": "integer"
           },
           {
            "metadata": {},
            "name": "race_cd",
            "nullable": true,
            "type": "string"
           }
          ],
          "type": "struct"
         },
         "tableIdentifier": null
        }
       ],
       "data": "<div class=\"ansiout\">There are 66284 records in the training dataset.\nThere are 35482 records in the testing dataset.\n</div>",
       "removedWidgets": [],
       "addedWidgets": {},
       "metadata": {},
       "type": "html",
       "arguments": {}
      }
     },
     "data": {
      "text/html": [
       "<style scoped>\n",
       "  .ansiout {\n",
       "    display: block;\n",
       "    unicode-bidi: embed;\n",
       "    white-space: pre-wrap;\n",
       "    word-wrap: break-word;\n",
       "    word-break: break-all;\n",
       "    font-family: \"Source Code Pro\", \"Menlo\", monospace;;\n",
       "    font-size: 13px;\n",
       "    color: #555;\n",
       "    margin-left: 4px;\n",
       "    line-height: 19px;\n",
       "  }\n",
       "</style>\n",
       "<div class=\"ansiout\">There are 66284 records in the training dataset.\n",
       "There are 35482 records in the testing dataset.\n",
       "</div>"
      ]
     }
    }
   ],
   "execution_count": 0
  },
  {
   "cell_type": "markdown",
   "source": [
    "##Now we need to modify the categorical variable race_cd into one-hot-encoded version\n",
    " For this we can either use the StringIndexer and OneHotEncoder separately OR use a pipeline to do this in one step"
   ],
   "metadata": {
    "application/vnd.databricks.v1+cell": {
     "showTitle": false,
     "cellMetadata": {},
     "nuid": "49346026-09ef-4c39-99d4-cbec017e789e",
     "inputWidgets": {},
     "title": ""
    }
   }
  },
  {
   "cell_type": "code",
   "source": [
    "#You can also create a pipeline and do everything together in one easy fit and transform step\n",
    "from pyspark.ml import Pipeline\n",
    "from pyspark.ml.feature import StringIndexer, VectorAssembler\n",
    " \n",
    "categoricalColumns = [\"race_cd\"]\n",
    "stages = [] # stages in Pipeline\n",
    "for categoricalCol in categoricalColumns:\n",
    "    # Category Indexing with StringIndexer\n",
    "    stringIndexer = StringIndexer(inputCol=categoricalCol, outputCol=categoricalCol + \"Index\")\n",
    "    \n",
    "# Use OneHotEncoder to convert categorical variables into binary SparseVectors\n",
    "from pyspark.ml.feature import OneHotEncoder\n",
    "encoder = OneHotEncoder(inputCols=[stringIndexer.getOutputCol()], outputCols=[categoricalCol + \"classVec\"])\n",
    "\n",
    "# Define the pipeline based on the stages created in previous steps.\n",
    "pipeline = Pipeline(stages=[stringIndexer, encoder])\n",
    " \n",
    "# Define the pipeline model.\n",
    "transform_mdl = pipeline.fit(trainDF)\n",
    "trainDF21=transform_mdl.transform(trainDF)\n",
    "trainDF21.show()"
   ],
   "metadata": {
    "application/vnd.databricks.v1+cell": {
     "showTitle": false,
     "cellMetadata": {
      "rowLimit": 10000,
      "byteLimit": 2048000
     },
     "nuid": "c1805ce5-cbc3-4de4-9b0f-7685d61aee10",
     "inputWidgets": {},
     "title": ""
    }
   },
   "outputs": [
    {
     "output_type": "display_data",
     "metadata": {
      "application/vnd.databricks.v1+output": {
       "datasetInfos": [
        {
         "name": "trainDF21",
         "typeStr": "pyspark.sql.dataframe.DataFrame",
         "schema": {
          "fields": [
           {
            "metadata": {},
            "name": "patient_nbr",
            "nullable": true,
            "type": "integer"
           },
           {
            "metadata": {},
            "name": "time_in_hospital",
            "nullable": true,
            "type": "integer"
           },
           {
            "metadata": {},
            "name": "num_procedures",
            "nullable": true,
            "type": "integer"
           },
           {
            "metadata": {},
            "name": "num_lab_procedures",
            "nullable": true,
            "type": "integer"
           },
           {
            "metadata": {},
            "name": "num_medications",
            "nullable": true,
            "type": "integer"
           },
           {
            "metadata": {},
            "name": "number_outpatient",
            "nullable": true,
            "type": "integer"
           },
           {
            "metadata": {},
            "name": "number_inpatient",
            "nullable": true,
            "type": "integer"
           },
           {
            "metadata": {},
            "name": "number_emergency",
            "nullable": true,
            "type": "integer"
           },
           {
            "metadata": {},
            "name": "number_diagnoses",
            "nullable": true,
            "type": "integer"
           },
           {
            "metadata": {},
            "name": "gender_cd",
            "nullable": true,
            "type": "integer"
           },
           {
            "metadata": {},
            "name": "DiabetesMedication",
            "nullable": true,
            "type": "integer"
           },
           {
            "metadata": {},
            "name": "readmit_flag",
            "nullable": true,
            "type": "integer"
           },
           {
            "metadata": {},
            "name": "race_cd",
            "nullable": true,
            "type": "string"
           },
           {
            "metadata": {
             "ml_attr": {
              "name": "race_cdIndex",
              "type": "nominal",
              "vals": [
               "White",
               "AfrAmr",
               "Unk",
               "Hispanic",
               "Asian"
              ]
             }
            },
            "name": "race_cdIndex",
            "nullable": false,
            "type": "double"
           },
           {
            "metadata": {
             "ml_attr": {
              "attrs": {
               "binary": [
                {
                 "idx": 0,
                 "name": "White"
                },
                {
                 "idx": 1,
                 "name": "AfrAmr"
                },
                {
                 "idx": 2,
                 "name": "Unk"
                },
                {
                 "idx": 3,
                 "name": "Hispanic"
                }
               ]
              },
              "num_attrs": 4
             }
            },
            "name": "race_cdclassVec",
            "nullable": true,
            "type": {
             "class": "org.apache.spark.ml.linalg.VectorUDT",
             "pyClass": "pyspark.ml.linalg.VectorUDT",
             "sqlType": {
              "fields": [
               {
                "metadata": {},
                "name": "type",
                "nullable": false,
                "type": "byte"
               },
               {
                "metadata": {},
                "name": "size",
                "nullable": true,
                "type": "integer"
               },
               {
                "metadata": {},
                "name": "indices",
                "nullable": true,
                "type": {
                 "containsNull": false,
                 "elementType": "integer",
                 "type": "array"
                }
               },
               {
                "metadata": {},
                "name": "values",
                "nullable": true,
                "type": {
                 "containsNull": false,
                 "elementType": "double",
                 "type": "array"
                }
               }
              ],
              "type": "struct"
             },
             "type": "udt"
            }
           }
          ],
          "type": "struct"
         },
         "tableIdentifier": null
        }
       ],
       "data": "<div class=\"ansiout\">+-----------+----------------+--------------+------------------+---------------+-----------------+----------------+----------------+----------------+---------+------------------+------------+-------+------------+---------------+\n|patient_nbr|time_in_hospital|num_procedures|num_lab_procedures|num_medications|number_outpatient|number_inpatient|number_emergency|number_diagnoses|gender_cd|DiabetesMedication|readmit_flag|race_cd|race_cdIndex|race_cdclassVec|\n+-----------+----------------+--------------+------------------+---------------+-----------------+----------------+----------------+----------------+---------+------------------+------------+-------+------------+---------------+\n|        135|               3|             1|                31|             14|                0|               1|               0|               5|        1|                 1|           0|  White|         0.0|  (4,[0],[1.0])|\n|        135|               8|             6|                77|             33|                0|               0|               0|               8|        1|                 1|           1|  White|         0.0|  (4,[0],[1.0])|\n|        729|               4|             2|                68|             23|                0|               0|               0|               9|        1|                 1|           0|  White|         0.0|  (4,[0],[1.0])|\n|        927|               5|             0|                49|              5|                0|               0|               0|               3|        1|                 1|           0| AfrAmr|         1.0|  (4,[1],[1.0])|\n|       1152|               6|             4|                45|             15|                0|               2|               0|               6|        1|                 1|           0| AfrAmr|         1.0|  (4,[1],[1.0])|\n|       1152|              12|             1|                37|             18|                0|               2|               0|               5|        1|                 1|           0| AfrAmr|         1.0|  (4,[1],[1.0])|\n|       1314|               1|             5|                47|             12|                0|               1|               0|               7|        1|                 1|           1|  White|         0.0|  (4,[0],[1.0])|\n|       2025|              12|             2|                47|             18|                0|               0|               0|               9|        1|                 1|           0|  White|         0.0|  (4,[0],[1.0])|\n|       3069|               8|             6|                57|             31|                0|               0|               0|               9|        0|                 1|           0|  White|         0.0|  (4,[0],[1.0])|\n|       4302|               1|             1|                31|              9|                0|               0|               0|               7|        1|                 0|           0|  White|         0.0|  (4,[0],[1.0])|\n|       5220|               2|             0|                15|             14|                0|               0|               0|               9|        0|                 1|           0|  White|         0.0|  (4,[0],[1.0])|\n|       5220|              11|             4|                65|             19|                0|               2|               0|               9|        0|                 1|           0|  White|         0.0|  (4,[0],[1.0])|\n|       6174|               7|             0|                77|             12|                0|               0|               0|               9|        1|                 1|           0|  White|         0.0|  (4,[0],[1.0])|\n|       6228|               4|             4|                47|             16|                0|               0|               0|               7|        1|                 0|           0|  White|         0.0|  (4,[0],[1.0])|\n|       6309|               1|             5|                35|             13|                0|               0|               0|               8|        0|                 1|           0|  White|         0.0|  (4,[0],[1.0])|\n|       6939|               6|             2|                65|             19|                0|               0|               0|               7|        0|                 1|           0|  White|         0.0|  (4,[0],[1.0])|\n|       8154|               1|             0|                58|              5|                0|               0|               0|               8|        1|                 0|           0|  White|         0.0|  (4,[0],[1.0])|\n|       8748|               2|             0|                64|             10|                0|               0|               0|               9|        0|                 1|           0| AfrAmr|         1.0|  (4,[1],[1.0])|\n|      10377|               1|             2|                33|             12|                0|               0|               0|               5|        0|                 1|           0|  White|         0.0|  (4,[0],[1.0])|\n|      10422|               4|             2|                54|              3|                0|               0|               0|               8|        1|                 0|           0| AfrAmr|         1.0|  (4,[1],[1.0])|\n+-----------+----------------+--------------+------------------+---------------+-----------------+----------------+----------------+----------------+---------+------------------+------------+-------+------------+---------------+\nonly showing top 20 rows\n\n</div>",
       "removedWidgets": [],
       "addedWidgets": {},
       "metadata": {},
       "type": "html",
       "arguments": {}
      }
     },
     "data": {
      "text/html": [
       "<style scoped>\n",
       "  .ansiout {\n",
       "    display: block;\n",
       "    unicode-bidi: embed;\n",
       "    white-space: pre-wrap;\n",
       "    word-wrap: break-word;\n",
       "    word-break: break-all;\n",
       "    font-family: \"Source Code Pro\", \"Menlo\", monospace;;\n",
       "    font-size: 13px;\n",
       "    color: #555;\n",
       "    margin-left: 4px;\n",
       "    line-height: 19px;\n",
       "  }\n",
       "</style>\n",
       "<div class=\"ansiout\">+-----------+----------------+--------------+------------------+---------------+-----------------+----------------+----------------+----------------+---------+------------------+------------+-------+------------+---------------+\n",
       "patient_nbr|time_in_hospital|num_procedures|num_lab_procedures|num_medications|number_outpatient|number_inpatient|number_emergency|number_diagnoses|gender_cd|DiabetesMedication|readmit_flag|race_cd|race_cdIndex|race_cdclassVec|\n",
       "+-----------+----------------+--------------+------------------+---------------+-----------------+----------------+----------------+----------------+---------+------------------+------------+-------+------------+---------------+\n",
       "        135|               3|             1|                31|             14|                0|               1|               0|               5|        1|                 1|           0|  White|         0.0|  (4,[0],[1.0])|\n",
       "        135|               8|             6|                77|             33|                0|               0|               0|               8|        1|                 1|           1|  White|         0.0|  (4,[0],[1.0])|\n",
       "        729|               4|             2|                68|             23|                0|               0|               0|               9|        1|                 1|           0|  White|         0.0|  (4,[0],[1.0])|\n",
       "        927|               5|             0|                49|              5|                0|               0|               0|               3|        1|                 1|           0| AfrAmr|         1.0|  (4,[1],[1.0])|\n",
       "       1152|               6|             4|                45|             15|                0|               2|               0|               6|        1|                 1|           0| AfrAmr|         1.0|  (4,[1],[1.0])|\n",
       "       1152|              12|             1|                37|             18|                0|               2|               0|               5|        1|                 1|           0| AfrAmr|         1.0|  (4,[1],[1.0])|\n",
       "       1314|               1|             5|                47|             12|                0|               1|               0|               7|        1|                 1|           1|  White|         0.0|  (4,[0],[1.0])|\n",
       "       2025|              12|             2|                47|             18|                0|               0|               0|               9|        1|                 1|           0|  White|         0.0|  (4,[0],[1.0])|\n",
       "       3069|               8|             6|                57|             31|                0|               0|               0|               9|        0|                 1|           0|  White|         0.0|  (4,[0],[1.0])|\n",
       "       4302|               1|             1|                31|              9|                0|               0|               0|               7|        1|                 0|           0|  White|         0.0|  (4,[0],[1.0])|\n",
       "       5220|               2|             0|                15|             14|                0|               0|               0|               9|        0|                 1|           0|  White|         0.0|  (4,[0],[1.0])|\n",
       "       5220|              11|             4|                65|             19|                0|               2|               0|               9|        0|                 1|           0|  White|         0.0|  (4,[0],[1.0])|\n",
       "       6174|               7|             0|                77|             12|                0|               0|               0|               9|        1|                 1|           0|  White|         0.0|  (4,[0],[1.0])|\n",
       "       6228|               4|             4|                47|             16|                0|               0|               0|               7|        1|                 0|           0|  White|         0.0|  (4,[0],[1.0])|\n",
       "       6309|               1|             5|                35|             13|                0|               0|               0|               8|        0|                 1|           0|  White|         0.0|  (4,[0],[1.0])|\n",
       "       6939|               6|             2|                65|             19|                0|               0|               0|               7|        0|                 1|           0|  White|         0.0|  (4,[0],[1.0])|\n",
       "       8154|               1|             0|                58|              5|                0|               0|               0|               8|        1|                 0|           0|  White|         0.0|  (4,[0],[1.0])|\n",
       "       8748|               2|             0|                64|             10|                0|               0|               0|               9|        0|                 1|           0| AfrAmr|         1.0|  (4,[1],[1.0])|\n",
       "      10377|               1|             2|                33|             12|                0|               0|               0|               5|        0|                 1|           0|  White|         0.0|  (4,[0],[1.0])|\n",
       "      10422|               4|             2|                54|              3|                0|               0|               0|               8|        1|                 0|           0| AfrAmr|         1.0|  (4,[1],[1.0])|\n",
       "+-----------+----------------+--------------+------------------+---------------+-----------------+----------------+----------------+----------------+---------+------------------+------------+-------+------------+---------------+\n",
       "only showing top 20 rows\n",
       "\n",
       "</div>"
      ]
     }
    }
   ],
   "execution_count": 0
  },
  {
   "cell_type": "code",
   "source": [
    "# Linear regression expect a vector input\n",
    "vecAssembler = VectorAssembler(inputCols=['time_in_hospital','num_procedures','num_medications', 'number_inpatient','number_emergency','number_diagnoses','DiabetesMedication'], outputCol=\"features\")\n",
    "vecTrainDF = vecAssembler.transform(trainDF21)"
   ],
   "metadata": {
    "application/vnd.databricks.v1+cell": {
     "showTitle": false,
     "cellMetadata": {
      "rowLimit": 10000,
      "byteLimit": 2048000
     },
     "nuid": "f8fa852d-9b9d-459a-81a1-e921bdb941b1",
     "inputWidgets": {},
     "title": ""
    }
   },
   "outputs": [
    {
     "output_type": "display_data",
     "metadata": {
      "application/vnd.databricks.v1+output": {
       "datasetInfos": [
        {
         "name": "vecTrainDF",
         "typeStr": "pyspark.sql.dataframe.DataFrame",
         "schema": {
          "fields": [
           {
            "metadata": {},
            "name": "patient_nbr",
            "nullable": true,
            "type": "integer"
           },
           {
            "metadata": {},
            "name": "time_in_hospital",
            "nullable": true,
            "type": "integer"
           },
           {
            "metadata": {},
            "name": "num_procedures",
            "nullable": true,
            "type": "integer"
           },
           {
            "metadata": {},
            "name": "num_lab_procedures",
            "nullable": true,
            "type": "integer"
           },
           {
            "metadata": {},
            "name": "num_medications",
            "nullable": true,
            "type": "integer"
           },
           {
            "metadata": {},
            "name": "number_outpatient",
            "nullable": true,
            "type": "integer"
           },
           {
            "metadata": {},
            "name": "number_inpatient",
            "nullable": true,
            "type": "integer"
           },
           {
            "metadata": {},
            "name": "number_emergency",
            "nullable": true,
            "type": "integer"
           },
           {
            "metadata": {},
            "name": "number_diagnoses",
            "nullable": true,
            "type": "integer"
           },
           {
            "metadata": {},
            "name": "gender_cd",
            "nullable": true,
            "type": "integer"
           },
           {
            "metadata": {},
            "name": "DiabetesMedication",
            "nullable": true,
            "type": "integer"
           },
           {
            "metadata": {},
            "name": "readmit_flag",
            "nullable": true,
            "type": "integer"
           },
           {
            "metadata": {},
            "name": "race_cd",
            "nullable": true,
            "type": "string"
           },
           {
            "metadata": {
             "ml_attr": {
              "name": "race_cdIndex",
              "type": "nominal",
              "vals": [
               "White",
               "AfrAmr",
               "Unk",
               "Hispanic",
               "Asian"
              ]
             }
            },
            "name": "race_cdIndex",
            "nullable": false,
            "type": "double"
           },
           {
            "metadata": {
             "ml_attr": {
              "attrs": {
               "binary": [
                {
                 "idx": 0,
                 "name": "White"
                },
                {
                 "idx": 1,
                 "name": "AfrAmr"
                },
                {
                 "idx": 2,
                 "name": "Unk"
                },
                {
                 "idx": 3,
                 "name": "Hispanic"
                }
               ]
              },
              "num_attrs": 4
             }
            },
            "name": "race_cdclassVec",
            "nullable": true,
            "type": {
             "class": "org.apache.spark.ml.linalg.VectorUDT",
             "pyClass": "pyspark.ml.linalg.VectorUDT",
             "sqlType": {
              "fields": [
               {
                "metadata": {},
                "name": "type",
                "nullable": false,
                "type": "byte"
               },
               {
                "metadata": {},
                "name": "size",
                "nullable": true,
                "type": "integer"
               },
               {
                "metadata": {},
                "name": "indices",
                "nullable": true,
                "type": {
                 "containsNull": false,
                 "elementType": "integer",
                 "type": "array"
                }
               },
               {
                "metadata": {},
                "name": "values",
                "nullable": true,
                "type": {
                 "containsNull": false,
                 "elementType": "double",
                 "type": "array"
                }
               }
              ],
              "type": "struct"
             },
             "type": "udt"
            }
           },
           {
            "metadata": {
             "ml_attr": {
              "attrs": {
               "numeric": [
                {
                 "idx": 0,
                 "name": "time_in_hospital"
                },
                {
                 "idx": 1,
                 "name": "num_procedures"
                },
                {
                 "idx": 2,
                 "name": "num_medications"
                },
                {
                 "idx": 3,
                 "name": "number_inpatient"
                },
                {
                 "idx": 4,
                 "name": "number_emergency"
                },
                {
                 "idx": 5,
                 "name": "number_diagnoses"
                },
                {
                 "idx": 6,
                 "name": "DiabetesMedication"
                }
               ]
              },
              "num_attrs": 7
             }
            },
            "name": "features",
            "nullable": true,
            "type": {
             "class": "org.apache.spark.ml.linalg.VectorUDT",
             "pyClass": "pyspark.ml.linalg.VectorUDT",
             "sqlType": {
              "fields": [
               {
                "metadata": {},
                "name": "type",
                "nullable": false,
                "type": "byte"
               },
               {
                "metadata": {},
                "name": "size",
                "nullable": true,
                "type": "integer"
               },
               {
                "metadata": {},
                "name": "indices",
                "nullable": true,
                "type": {
                 "containsNull": false,
                 "elementType": "integer",
                 "type": "array"
                }
               },
               {
                "metadata": {},
                "name": "values",
                "nullable": true,
                "type": {
                 "containsNull": false,
                 "elementType": "double",
                 "type": "array"
                }
               }
              ],
              "type": "struct"
             },
             "type": "udt"
            }
           }
          ],
          "type": "struct"
         },
         "tableIdentifier": null
        }
       ],
       "data": "<div class=\"ansiout\"></div>",
       "removedWidgets": [],
       "addedWidgets": {},
       "metadata": {},
       "type": "html",
       "arguments": {}
      }
     },
     "data": {
      "text/html": [
       "<style scoped>\n",
       "  .ansiout {\n",
       "    display: block;\n",
       "    unicode-bidi: embed;\n",
       "    white-space: pre-wrap;\n",
       "    word-wrap: break-word;\n",
       "    word-break: break-all;\n",
       "    font-family: \"Source Code Pro\", \"Menlo\", monospace;;\n",
       "    font-size: 13px;\n",
       "    color: #555;\n",
       "    margin-left: 4px;\n",
       "    line-height: 19px;\n",
       "  }\n",
       "</style>\n",
       "<div class=\"ansiout\"></div>"
      ]
     }
    }
   ],
   "execution_count": 0
  },
  {
   "cell_type": "code",
   "source": [
    "# input layer of size 7 (features), one intermediate of size 10, and output of size 2 (classes)\n",
    "layers = [7,10,2]\n",
    "\n",
    "# Create Decision tree calssifier\n",
    "ann_mlpc = (MultilayerPerceptronClassifier(labelCol='readmit_flag',\n",
    "                                            featuresCol='features',\n",
    "                                            maxIter=100,\n",
    "                                            layers=layers,\n",
    "                                            blockSize=128,\n",
    "                                            seed=1234))\n",
    "ann_mdl = ann_mlpc.fit(vecTrainDF)\n",
    "predict_train = ann_mdl.transform(vecTrainDF)\n",
    "predict_train.select('readmit_flag', 'rawPrediction', 'prediction', 'probability').show(10)\n"
   ],
   "metadata": {
    "application/vnd.databricks.v1+cell": {
     "showTitle": false,
     "cellMetadata": {
      "rowLimit": 10000,
      "byteLimit": 2048000
     },
     "nuid": "2b2b03c5-11a2-4374-8d3c-c7ce7a6c4182",
     "inputWidgets": {},
     "title": ""
    }
   },
   "outputs": [
    {
     "output_type": "display_data",
     "metadata": {
      "application/vnd.databricks.v1+output": {
       "datasetInfos": [
        {
         "name": "predict_train",
         "typeStr": "pyspark.sql.dataframe.DataFrame",
         "schema": {
          "fields": [
           {
            "metadata": {},
            "name": "patient_nbr",
            "nullable": true,
            "type": "integer"
           },
           {
            "metadata": {},
            "name": "time_in_hospital",
            "nullable": true,
            "type": "integer"
           },
           {
            "metadata": {},
            "name": "num_procedures",
            "nullable": true,
            "type": "integer"
           },
           {
            "metadata": {},
            "name": "num_lab_procedures",
            "nullable": true,
            "type": "integer"
           },
           {
            "metadata": {},
            "name": "num_medications",
            "nullable": true,
            "type": "integer"
           },
           {
            "metadata": {},
            "name": "number_outpatient",
            "nullable": true,
            "type": "integer"
           },
           {
            "metadata": {},
            "name": "number_inpatient",
            "nullable": true,
            "type": "integer"
           },
           {
            "metadata": {},
            "name": "number_emergency",
            "nullable": true,
            "type": "integer"
           },
           {
            "metadata": {},
            "name": "number_diagnoses",
            "nullable": true,
            "type": "integer"
           },
           {
            "metadata": {},
            "name": "gender_cd",
            "nullable": true,
            "type": "integer"
           },
           {
            "metadata": {},
            "name": "DiabetesMedication",
            "nullable": true,
            "type": "integer"
           },
           {
            "metadata": {},
            "name": "readmit_flag",
            "nullable": true,
            "type": "integer"
           },
           {
            "metadata": {},
            "name": "race_cd",
            "nullable": true,
            "type": "string"
           },
           {
            "metadata": {
             "ml_attr": {
              "name": "race_cdIndex",
              "type": "nominal",
              "vals": [
               "White",
               "AfrAmr",
               "Unk",
               "Hispanic",
               "Asian"
              ]
             }
            },
            "name": "race_cdIndex",
            "nullable": false,
            "type": "double"
           },
           {
            "metadata": {
             "ml_attr": {
              "attrs": {
               "binary": [
                {
                 "idx": 0,
                 "name": "White"
                },
                {
                 "idx": 1,
                 "name": "AfrAmr"
                },
                {
                 "idx": 2,
                 "name": "Unk"
                },
                {
                 "idx": 3,
                 "name": "Hispanic"
                }
               ]
              },
              "num_attrs": 4
             }
            },
            "name": "race_cdclassVec",
            "nullable": true,
            "type": {
             "class": "org.apache.spark.ml.linalg.VectorUDT",
             "pyClass": "pyspark.ml.linalg.VectorUDT",
             "sqlType": {
              "fields": [
               {
                "metadata": {},
                "name": "type",
                "nullable": false,
                "type": "byte"
               },
               {
                "metadata": {},
                "name": "size",
                "nullable": true,
                "type": "integer"
               },
               {
                "metadata": {},
                "name": "indices",
                "nullable": true,
                "type": {
                 "containsNull": false,
                 "elementType": "integer",
                 "type": "array"
                }
               },
               {
                "metadata": {},
                "name": "values",
                "nullable": true,
                "type": {
                 "containsNull": false,
                 "elementType": "double",
                 "type": "array"
                }
               }
              ],
              "type": "struct"
             },
             "type": "udt"
            }
           },
           {
            "metadata": {
             "ml_attr": {
              "attrs": {
               "numeric": [
                {
                 "idx": 0,
                 "name": "time_in_hospital"
                },
                {
                 "idx": 1,
                 "name": "num_procedures"
                },
                {
                 "idx": 2,
                 "name": "num_medications"
                },
                {
                 "idx": 3,
                 "name": "number_inpatient"
                },
                {
                 "idx": 4,
                 "name": "number_emergency"
                },
                {
                 "idx": 5,
                 "name": "number_diagnoses"
                },
                {
                 "idx": 6,
                 "name": "DiabetesMedication"
                }
               ]
              },
              "num_attrs": 7
             }
            },
            "name": "features",
            "nullable": true,
            "type": {
             "class": "org.apache.spark.ml.linalg.VectorUDT",
             "pyClass": "pyspark.ml.linalg.VectorUDT",
             "sqlType": {
              "fields": [
               {
                "metadata": {},
                "name": "type",
                "nullable": false,
                "type": "byte"
               },
               {
                "metadata": {},
                "name": "size",
                "nullable": true,
                "type": "integer"
               },
               {
                "metadata": {},
                "name": "indices",
                "nullable": true,
                "type": {
                 "containsNull": false,
                 "elementType": "integer",
                 "type": "array"
                }
               },
               {
                "metadata": {},
                "name": "values",
                "nullable": true,
                "type": {
                 "containsNull": false,
                 "elementType": "double",
                 "type": "array"
                }
               }
              ],
              "type": "struct"
             },
             "type": "udt"
            }
           },
           {
            "metadata": {
             "ml_attr": {
              "num_attrs": 2
             }
            },
            "name": "rawPrediction",
            "nullable": true,
            "type": {
             "class": "org.apache.spark.ml.linalg.VectorUDT",
             "pyClass": "pyspark.ml.linalg.VectorUDT",
             "sqlType": {
              "fields": [
               {
                "metadata": {},
                "name": "type",
                "nullable": false,
                "type": "byte"
               },
               {
                "metadata": {},
                "name": "size",
                "nullable": true,
                "type": "integer"
               },
               {
                "metadata": {},
                "name": "indices",
                "nullable": true,
                "type": {
                 "containsNull": false,
                 "elementType": "integer",
                 "type": "array"
                }
               },
               {
                "metadata": {},
                "name": "values",
                "nullable": true,
                "type": {
                 "containsNull": false,
                 "elementType": "double",
                 "type": "array"
                }
               }
              ],
              "type": "struct"
             },
             "type": "udt"
            }
           },
           {
            "metadata": {
             "ml_attr": {
              "num_attrs": 2
             }
            },
            "name": "probability",
            "nullable": true,
            "type": {
             "class": "org.apache.spark.ml.linalg.VectorUDT",
             "pyClass": "pyspark.ml.linalg.VectorUDT",
             "sqlType": {
              "fields": [
               {
                "metadata": {},
                "name": "type",
                "nullable": false,
                "type": "byte"
               },
               {
                "metadata": {},
                "name": "size",
                "nullable": true,
                "type": "integer"
               },
               {
                "metadata": {},
                "name": "indices",
                "nullable": true,
                "type": {
                 "containsNull": false,
                 "elementType": "integer",
                 "type": "array"
                }
               },
               {
                "metadata": {},
                "name": "values",
                "nullable": true,
                "type": {
                 "containsNull": false,
                 "elementType": "double",
                 "type": "array"
                }
               }
              ],
              "type": "struct"
             },
             "type": "udt"
            }
           },
           {
            "metadata": {
             "ml_attr": {
              "num_vals": 2,
              "type": "nominal"
             }
            },
            "name": "prediction",
            "nullable": false,
            "type": "double"
           }
          ],
          "type": "struct"
         },
         "tableIdentifier": null
        }
       ],
       "data": "<div class=\"ansiout\">+------------+--------------------+----------+--------------------+\n|readmit_flag|       rawPrediction|prediction|         probability|\n+------------+--------------------+----------+--------------------+\n|           0|[1.62837038739449...|       0.0|[0.88771755036096...|\n|           1|[1.63238044136749...|       0.0|[0.88740691909947...|\n|           0|[1.69225982577089...|       0.0|[0.90483580832412...|\n|           0|[1.64457477068628...|       0.0|[0.92306362107251...|\n|           0|[1.36696366292519...|       0.0|[0.81017714479284...|\n|           0|[1.43063105347940...|       0.0|[0.85425130105974...|\n|           1|[1.78511982336359...|       0.0|[0.91037017168271...|\n|           0|[1.57047752770136...|       0.0|[0.89510268928386...|\n|           0|[1.59426559402237...|       0.0|[0.88095238421950...|\n|           0|[1.93066411360776...|       0.0|[0.94318175921958...|\n+------------+--------------------+----------+--------------------+\nonly showing top 10 rows\n\n</div>",
       "removedWidgets": [],
       "addedWidgets": {},
       "metadata": {},
       "type": "html",
       "arguments": {}
      }
     },
     "data": {
      "text/html": [
       "<style scoped>\n",
       "  .ansiout {\n",
       "    display: block;\n",
       "    unicode-bidi: embed;\n",
       "    white-space: pre-wrap;\n",
       "    word-wrap: break-word;\n",
       "    word-break: break-all;\n",
       "    font-family: \"Source Code Pro\", \"Menlo\", monospace;;\n",
       "    font-size: 13px;\n",
       "    color: #555;\n",
       "    margin-left: 4px;\n",
       "    line-height: 19px;\n",
       "  }\n",
       "</style>\n",
       "<div class=\"ansiout\">+------------+--------------------+----------+--------------------+\n",
       "readmit_flag|       rawPrediction|prediction|         probability|\n",
       "+------------+--------------------+----------+--------------------+\n",
       "           0|[1.62837038739449...|       0.0|[0.88771755036096...|\n",
       "           1|[1.63238044136749...|       0.0|[0.88740691909947...|\n",
       "           0|[1.69225982577089...|       0.0|[0.90483580832412...|\n",
       "           0|[1.64457477068628...|       0.0|[0.92306362107251...|\n",
       "           0|[1.36696366292519...|       0.0|[0.81017714479284...|\n",
       "           0|[1.43063105347940...|       0.0|[0.85425130105974...|\n",
       "           1|[1.78511982336359...|       0.0|[0.91037017168271...|\n",
       "           0|[1.57047752770136...|       0.0|[0.89510268928386...|\n",
       "           0|[1.59426559402237...|       0.0|[0.88095238421950...|\n",
       "           0|[1.93066411360776...|       0.0|[0.94318175921958...|\n",
       "+------------+--------------------+----------+--------------------+\n",
       "only showing top 10 rows\n",
       "\n",
       "</div>"
      ]
     }
    }
   ],
   "execution_count": 0
  },
  {
   "cell_type": "code",
   "source": [
    "testDF21=transform_mdl.transform(testDF) #do the data transformation using saved parameters from training\n",
    "vecTestDF = vecAssembler.transform(testDF21) #do the feature transformation using vector assembler\n",
    "# Make predictions on testing dataset\n",
    "predict_test = ann_mdl.transform(vecTestDF) #make predictions using the trained model"
   ],
   "metadata": {
    "application/vnd.databricks.v1+cell": {
     "showTitle": false,
     "cellMetadata": {
      "rowLimit": 10000,
      "byteLimit": 2048000
     },
     "nuid": "a03d2a6e-81c1-40ec-bb3e-95d9aebaa32c",
     "inputWidgets": {},
     "title": ""
    }
   },
   "outputs": [
    {
     "output_type": "display_data",
     "metadata": {
      "application/vnd.databricks.v1+output": {
       "datasetInfos": [
        {
         "name": "testDF21",
         "typeStr": "pyspark.sql.dataframe.DataFrame",
         "schema": {
          "fields": [
           {
            "metadata": {},
            "name": "patient_nbr",
            "nullable": true,
            "type": "integer"
           },
           {
            "metadata": {},
            "name": "time_in_hospital",
            "nullable": true,
            "type": "integer"
           },
           {
            "metadata": {},
            "name": "num_procedures",
            "nullable": true,
            "type": "integer"
           },
           {
            "metadata": {},
            "name": "num_lab_procedures",
            "nullable": true,
            "type": "integer"
           },
           {
            "metadata": {},
            "name": "num_medications",
            "nullable": true,
            "type": "integer"
           },
           {
            "metadata": {},
            "name": "number_outpatient",
            "nullable": true,
            "type": "integer"
           },
           {
            "metadata": {},
            "name": "number_inpatient",
            "nullable": true,
            "type": "integer"
           },
           {
            "metadata": {},
            "name": "number_emergency",
            "nullable": true,
            "type": "integer"
           },
           {
            "metadata": {},
            "name": "number_diagnoses",
            "nullable": true,
            "type": "integer"
           },
           {
            "metadata": {},
            "name": "gender_cd",
            "nullable": true,
            "type": "integer"
           },
           {
            "metadata": {},
            "name": "DiabetesMedication",
            "nullable": true,
            "type": "integer"
           },
           {
            "metadata": {},
            "name": "readmit_flag",
            "nullable": true,
            "type": "integer"
           },
           {
            "metadata": {},
            "name": "race_cd",
            "nullable": true,
            "type": "string"
           },
           {
            "metadata": {
             "ml_attr": {
              "name": "race_cdIndex",
              "type": "nominal",
              "vals": [
               "White",
               "AfrAmr",
               "Unk",
               "Hispanic",
               "Asian"
              ]
             }
            },
            "name": "race_cdIndex",
            "nullable": false,
            "type": "double"
           },
           {
            "metadata": {
             "ml_attr": {
              "attrs": {
               "binary": [
                {
                 "idx": 0,
                 "name": "White"
                },
                {
                 "idx": 1,
                 "name": "AfrAmr"
                },
                {
                 "idx": 2,
                 "name": "Unk"
                },
                {
                 "idx": 3,
                 "name": "Hispanic"
                }
               ]
              },
              "num_attrs": 4
             }
            },
            "name": "race_cdclassVec",
            "nullable": true,
            "type": {
             "class": "org.apache.spark.ml.linalg.VectorUDT",
             "pyClass": "pyspark.ml.linalg.VectorUDT",
             "sqlType": {
              "fields": [
               {
                "metadata": {},
                "name": "type",
                "nullable": false,
                "type": "byte"
               },
               {
                "metadata": {},
                "name": "size",
                "nullable": true,
                "type": "integer"
               },
               {
                "metadata": {},
                "name": "indices",
                "nullable": true,
                "type": {
                 "containsNull": false,
                 "elementType": "integer",
                 "type": "array"
                }
               },
               {
                "metadata": {},
                "name": "values",
                "nullable": true,
                "type": {
                 "containsNull": false,
                 "elementType": "double",
                 "type": "array"
                }
               }
              ],
              "type": "struct"
             },
             "type": "udt"
            }
           }
          ],
          "type": "struct"
         },
         "tableIdentifier": null
        },
        {
         "name": "vecTestDF",
         "typeStr": "pyspark.sql.dataframe.DataFrame",
         "schema": {
          "fields": [
           {
            "metadata": {},
            "name": "patient_nbr",
            "nullable": true,
            "type": "integer"
           },
           {
            "metadata": {},
            "name": "time_in_hospital",
            "nullable": true,
            "type": "integer"
           },
           {
            "metadata": {},
            "name": "num_procedures",
            "nullable": true,
            "type": "integer"
           },
           {
            "metadata": {},
            "name": "num_lab_procedures",
            "nullable": true,
            "type": "integer"
           },
           {
            "metadata": {},
            "name": "num_medications",
            "nullable": true,
            "type": "integer"
           },
           {
            "metadata": {},
            "name": "number_outpatient",
            "nullable": true,
            "type": "integer"
           },
           {
            "metadata": {},
            "name": "number_inpatient",
            "nullable": true,
            "type": "integer"
           },
           {
            "metadata": {},
            "name": "number_emergency",
            "nullable": true,
            "type": "integer"
           },
           {
            "metadata": {},
            "name": "number_diagnoses",
            "nullable": true,
            "type": "integer"
           },
           {
            "metadata": {},
            "name": "gender_cd",
            "nullable": true,
            "type": "integer"
           },
           {
            "metadata": {},
            "name": "DiabetesMedication",
            "nullable": true,
            "type": "integer"
           },
           {
            "metadata": {},
            "name": "readmit_flag",
            "nullable": true,
            "type": "integer"
           },
           {
            "metadata": {},
            "name": "race_cd",
            "nullable": true,
            "type": "string"
           },
           {
            "metadata": {
             "ml_attr": {
              "name": "race_cdIndex",
              "type": "nominal",
              "vals": [
               "White",
               "AfrAmr",
               "Unk",
               "Hispanic",
               "Asian"
              ]
             }
            },
            "name": "race_cdIndex",
            "nullable": false,
            "type": "double"
           },
           {
            "metadata": {
             "ml_attr": {
              "attrs": {
               "binary": [
                {
                 "idx": 0,
                 "name": "White"
                },
                {
                 "idx": 1,
                 "name": "AfrAmr"
                },
                {
                 "idx": 2,
                 "name": "Unk"
                },
                {
                 "idx": 3,
                 "name": "Hispanic"
                }
               ]
              },
              "num_attrs": 4
             }
            },
            "name": "race_cdclassVec",
            "nullable": true,
            "type": {
             "class": "org.apache.spark.ml.linalg.VectorUDT",
             "pyClass": "pyspark.ml.linalg.VectorUDT",
             "sqlType": {
              "fields": [
               {
                "metadata": {},
                "name": "type",
                "nullable": false,
                "type": "byte"
               },
               {
                "metadata": {},
                "name": "size",
                "nullable": true,
                "type": "integer"
               },
               {
                "metadata": {},
                "name": "indices",
                "nullable": true,
                "type": {
                 "containsNull": false,
                 "elementType": "integer",
                 "type": "array"
                }
               },
               {
                "metadata": {},
                "name": "values",
                "nullable": true,
                "type": {
                 "containsNull": false,
                 "elementType": "double",
                 "type": "array"
                }
               }
              ],
              "type": "struct"
             },
             "type": "udt"
            }
           },
           {
            "metadata": {
             "ml_attr": {
              "attrs": {
               "numeric": [
                {
                 "idx": 0,
                 "name": "time_in_hospital"
                },
                {
                 "idx": 1,
                 "name": "num_procedures"
                },
                {
                 "idx": 2,
                 "name": "num_medications"
                },
                {
                 "idx": 3,
                 "name": "number_inpatient"
                },
                {
                 "idx": 4,
                 "name": "number_emergency"
                },
                {
                 "idx": 5,
                 "name": "number_diagnoses"
                },
                {
                 "idx": 6,
                 "name": "DiabetesMedication"
                }
               ]
              },
              "num_attrs": 7
             }
            },
            "name": "features",
            "nullable": true,
            "type": {
             "class": "org.apache.spark.ml.linalg.VectorUDT",
             "pyClass": "pyspark.ml.linalg.VectorUDT",
             "sqlType": {
              "fields": [
               {
                "metadata": {},
                "name": "type",
                "nullable": false,
                "type": "byte"
               },
               {
                "metadata": {},
                "name": "size",
                "nullable": true,
                "type": "integer"
               },
               {
                "metadata": {},
                "name": "indices",
                "nullable": true,
                "type": {
                 "containsNull": false,
                 "elementType": "integer",
                 "type": "array"
                }
               },
               {
                "metadata": {},
                "name": "values",
                "nullable": true,
                "type": {
                 "containsNull": false,
                 "elementType": "double",
                 "type": "array"
                }
               }
              ],
              "type": "struct"
             },
             "type": "udt"
            }
           }
          ],
          "type": "struct"
         },
         "tableIdentifier": null
        },
        {
         "name": "predict_test",
         "typeStr": "pyspark.sql.dataframe.DataFrame",
         "schema": {
          "fields": [
           {
            "metadata": {},
            "name": "patient_nbr",
            "nullable": true,
            "type": "integer"
           },
           {
            "metadata": {},
            "name": "time_in_hospital",
            "nullable": true,
            "type": "integer"
           },
           {
            "metadata": {},
            "name": "num_procedures",
            "nullable": true,
            "type": "integer"
           },
           {
            "metadata": {},
            "name": "num_lab_procedures",
            "nullable": true,
            "type": "integer"
           },
           {
            "metadata": {},
            "name": "num_medications",
            "nullable": true,
            "type": "integer"
           },
           {
            "metadata": {},
            "name": "number_outpatient",
            "nullable": true,
            "type": "integer"
           },
           {
            "metadata": {},
            "name": "number_inpatient",
            "nullable": true,
            "type": "integer"
           },
           {
            "metadata": {},
            "name": "number_emergency",
            "nullable": true,
            "type": "integer"
           },
           {
            "metadata": {},
            "name": "number_diagnoses",
            "nullable": true,
            "type": "integer"
           },
           {
            "metadata": {},
            "name": "gender_cd",
            "nullable": true,
            "type": "integer"
           },
           {
            "metadata": {},
            "name": "DiabetesMedication",
            "nullable": true,
            "type": "integer"
           },
           {
            "metadata": {},
            "name": "readmit_flag",
            "nullable": true,
            "type": "integer"
           },
           {
            "metadata": {},
            "name": "race_cd",
            "nullable": true,
            "type": "string"
           },
           {
            "metadata": {
             "ml_attr": {
              "name": "race_cdIndex",
              "type": "nominal",
              "vals": [
               "White",
               "AfrAmr",
               "Unk",
               "Hispanic",
               "Asian"
              ]
             }
            },
            "name": "race_cdIndex",
            "nullable": false,
            "type": "double"
           },
           {
            "metadata": {
             "ml_attr": {
              "attrs": {
               "binary": [
                {
                 "idx": 0,
                 "name": "White"
                },
                {
                 "idx": 1,
                 "name": "AfrAmr"
                },
                {
                 "idx": 2,
                 "name": "Unk"
                },
                {
                 "idx": 3,
                 "name": "Hispanic"
                }
               ]
              },
              "num_attrs": 4
             }
            },
            "name": "race_cdclassVec",
            "nullable": true,
            "type": {
             "class": "org.apache.spark.ml.linalg.VectorUDT",
             "pyClass": "pyspark.ml.linalg.VectorUDT",
             "sqlType": {
              "fields": [
               {
                "metadata": {},
                "name": "type",
                "nullable": false,
                "type": "byte"
               },
               {
                "metadata": {},
                "name": "size",
                "nullable": true,
                "type": "integer"
               },
               {
                "metadata": {},
                "name": "indices",
                "nullable": true,
                "type": {
                 "containsNull": false,
                 "elementType": "integer",
                 "type": "array"
                }
               },
               {
                "metadata": {},
                "name": "values",
                "nullable": true,
                "type": {
                 "containsNull": false,
                 "elementType": "double",
                 "type": "array"
                }
               }
              ],
              "type": "struct"
             },
             "type": "udt"
            }
           },
           {
            "metadata": {
             "ml_attr": {
              "attrs": {
               "numeric": [
                {
                 "idx": 0,
                 "name": "time_in_hospital"
                },
                {
                 "idx": 1,
                 "name": "num_procedures"
                },
                {
                 "idx": 2,
                 "name": "num_medications"
                },
                {
                 "idx": 3,
                 "name": "number_inpatient"
                },
                {
                 "idx": 4,
                 "name": "number_emergency"
                },
                {
                 "idx": 5,
                 "name": "number_diagnoses"
                },
                {
                 "idx": 6,
                 "name": "DiabetesMedication"
                }
               ]
              },
              "num_attrs": 7
             }
            },
            "name": "features",
            "nullable": true,
            "type": {
             "class": "org.apache.spark.ml.linalg.VectorUDT",
             "pyClass": "pyspark.ml.linalg.VectorUDT",
             "sqlType": {
              "fields": [
               {
                "metadata": {},
                "name": "type",
                "nullable": false,
                "type": "byte"
               },
               {
                "metadata": {},
                "name": "size",
                "nullable": true,
                "type": "integer"
               },
               {
                "metadata": {},
                "name": "indices",
                "nullable": true,
                "type": {
                 "containsNull": false,
                 "elementType": "integer",
                 "type": "array"
                }
               },
               {
                "metadata": {},
                "name": "values",
                "nullable": true,
                "type": {
                 "containsNull": false,
                 "elementType": "double",
                 "type": "array"
                }
               }
              ],
              "type": "struct"
             },
             "type": "udt"
            }
           },
           {
            "metadata": {
             "ml_attr": {
              "num_attrs": 2
             }
            },
            "name": "rawPrediction",
            "nullable": true,
            "type": {
             "class": "org.apache.spark.ml.linalg.VectorUDT",
             "pyClass": "pyspark.ml.linalg.VectorUDT",
             "sqlType": {
              "fields": [
               {
                "metadata": {},
                "name": "type",
                "nullable": false,
                "type": "byte"
               },
               {
                "metadata": {},
                "name": "size",
                "nullable": true,
                "type": "integer"
               },
               {
                "metadata": {},
                "name": "indices",
                "nullable": true,
                "type": {
                 "containsNull": false,
                 "elementType": "integer",
                 "type": "array"
                }
               },
               {
                "metadata": {},
                "name": "values",
                "nullable": true,
                "type": {
                 "containsNull": false,
                 "elementType": "double",
                 "type": "array"
                }
               }
              ],
              "type": "struct"
             },
             "type": "udt"
            }
           },
           {
            "metadata": {
             "ml_attr": {
              "num_attrs": 2
             }
            },
            "name": "probability",
            "nullable": true,
            "type": {
             "class": "org.apache.spark.ml.linalg.VectorUDT",
             "pyClass": "pyspark.ml.linalg.VectorUDT",
             "sqlType": {
              "fields": [
               {
                "metadata": {},
                "name": "type",
                "nullable": false,
                "type": "byte"
               },
               {
                "metadata": {},
                "name": "size",
                "nullable": true,
                "type": "integer"
               },
               {
                "metadata": {},
                "name": "indices",
                "nullable": true,
                "type": {
                 "containsNull": false,
                 "elementType": "integer",
                 "type": "array"
                }
               },
               {
                "metadata": {},
                "name": "values",
                "nullable": true,
                "type": {
                 "containsNull": false,
                 "elementType": "double",
                 "type": "array"
                }
               }
              ],
              "type": "struct"
             },
             "type": "udt"
            }
           },
           {
            "metadata": {
             "ml_attr": {
              "num_vals": 2,
              "type": "nominal"
             }
            },
            "name": "prediction",
            "nullable": false,
            "type": "double"
           }
          ],
          "type": "struct"
         },
         "tableIdentifier": null
        }
       ],
       "data": "<div class=\"ansiout\"></div>",
       "removedWidgets": [],
       "addedWidgets": {},
       "metadata": {},
       "type": "html",
       "arguments": {}
      }
     },
     "data": {
      "text/html": [
       "<style scoped>\n",
       "  .ansiout {\n",
       "    display: block;\n",
       "    unicode-bidi: embed;\n",
       "    white-space: pre-wrap;\n",
       "    word-wrap: break-word;\n",
       "    word-break: break-all;\n",
       "    font-family: \"Source Code Pro\", \"Menlo\", monospace;;\n",
       "    font-size: 13px;\n",
       "    color: #555;\n",
       "    margin-left: 4px;\n",
       "    line-height: 19px;\n",
       "  }\n",
       "</style>\n",
       "<div class=\"ansiout\"></div>"
      ]
     }
    }
   ],
   "execution_count": 0
  },
  {
   "cell_type": "code",
   "source": [
    "eval_ann = BinaryClassificationEvaluator(labelCol = \"readmit_flag\")\n",
    "auc_train = eval_ann.evaluate(predict_train)\n",
    "print(auc_train)\n",
    "\n",
    "auc_test = eval_ann.evaluate(predict_test)\n",
    "print(auc_test)"
   ],
   "metadata": {
    "application/vnd.databricks.v1+cell": {
     "showTitle": false,
     "cellMetadata": {
      "rowLimit": 10000,
      "byteLimit": 2048000
     },
     "nuid": "98dd8d65-8d05-4ede-bd0e-ba9c70f2df3d",
     "inputWidgets": {},
     "title": ""
    }
   },
   "outputs": [
    {
     "output_type": "display_data",
     "metadata": {
      "application/vnd.databricks.v1+output": {
       "datasetInfos": [],
       "data": "<div class=\"ansiout\">0.634270504850411\n0.6345223971936924\n</div>",
       "removedWidgets": [],
       "addedWidgets": {},
       "metadata": {},
       "type": "html",
       "arguments": {}
      }
     },
     "data": {
      "text/html": [
       "<style scoped>\n",
       "  .ansiout {\n",
       "    display: block;\n",
       "    unicode-bidi: embed;\n",
       "    white-space: pre-wrap;\n",
       "    word-wrap: break-word;\n",
       "    word-break: break-all;\n",
       "    font-family: \"Source Code Pro\", \"Menlo\", monospace;;\n",
       "    font-size: 13px;\n",
       "    color: #555;\n",
       "    margin-left: 4px;\n",
       "    line-height: 19px;\n",
       "  }\n",
       "</style>\n",
       "<div class=\"ansiout\">0.634270504850411\n",
       "0.6345223971936924\n",
       "</div>"
      ]
     }
    }
   ],
   "execution_count": 0
  },
  {
   "cell_type": "code",
   "source": [
    "import pandas as pd\n",
    "import time \n",
    "import sys\n",
    "from pyspark.sql.window import *\n",
    "from pyspark.sql.types import * \n",
    "from pyspark.sql.functions import *\n",
    "import pyspark.sql.functions as f\n",
    "\n",
    "def get_model_stats(print_lable,dataset,evaluator,mbr_id,target):\n",
    "  print(print_lable)\n",
    "  AUROC=evaluator.evaluate(dataset)\n",
    "  \n",
    "  tot_event=dataset.count()\n",
    "  tot_target=dataset.select(target).filter(f.col(target)==1).count()\n",
    "  inc_rate=float(tot_target)/float(tot_event)\n",
    "  print('Incidence rate in dataset: ',inc_rate)\n",
    "  print('AUROC: ',AUROC)\n",
    "  \n",
    "\n",
    "  def mbr_prob1(prob_vec):\n",
    "    return float(prob_vec[1]) #Get the second value from probability vector of the prediction since that is probability of target 1\n",
    "  def mbr_prob2(rawp_vec):\n",
    "    prob_vec = np.exp(rawp_vector)/(1+ np.exp(rawp_vec)) #this is the same as the logit function e**x/1+e**x\n",
    "    return float(prob_vec[1])\n",
    "  get_prob_1= udf(mbr_prob1, FloatType())\n",
    "  get_prob_2= udf(mbr_prob2, FloatType())\n",
    "\n",
    "  try:\n",
    "    df_plot_tmp = dataset.withColumn('prob_score', get_prob_1('probability'))\n",
    "  except:\n",
    "    df_plot_tmp = dataset.withColumn('prob_score', get_prob_2('rawPrediction'))\n",
    "  df_reg1=(df_plot_tmp.select(mbr_id,\"rawPrediction\",\"probability\",\"prob_score\",target))\n",
    "\n",
    "  #Create Percentiles based on the predicted probability\n",
    "  df_reg_12=  df_reg1.sort(col(\"prob_score\").desc())\n",
    "  df_reg_12 = df_reg_12.withColumn(\"new_column\",lit(\"ABC\"))\n",
    "  w = Window().partitionBy(\"new_column\").orderBy(col(\"prob_score\").desc())  \n",
    "  df_reg_12 = df_reg_12.withColumn(\"row_num\",row_number().over(w)).drop(\"new_column\")\n",
    "  df_reg2 = (df_reg_12\n",
    "             .withColumn(\"score_pctl\", f.percent_rank().over(Window.orderBy(f.col(\"row_num\"))))\n",
    "             .withColumn(\"pctl_cat\", f.when(f.col(\"score_pctl\")==1, f.lit(100)).otherwise(f.floor(f.col(\"score_pctl\")*100) + 1))\n",
    "             )\n",
    "  df_reg22 = (df_reg2\n",
    "              .groupBy(\"pctl_cat\")\n",
    "              .agg(f.mean(target).alias(\"pctl_target_rate\"),\\\n",
    "                   (f.sum(target)).alias(\"pctl_target_total\"),\\\n",
    "                   (f.count(mbr_id)).alias(\"total_count\"),\\\n",
    "                   (f.mean(\"prob_score\")).alias(\"mean_predicted_prob\")\n",
    "                  ,)\n",
    "              .orderBy(f.col(\"pctl_cat\"))\n",
    "              )\n",
    "\n",
    "  #Now let's create the cumulative percentiles\n",
    "  cum_sum = (df_reg22\n",
    "              .withColumn('cum_target', f.sum(df_reg22.pctl_target_total).over(Window.partitionBy().orderBy().rowsBetween(-sys.maxsize, 0)))\n",
    "              .withColumn('cum_total', f.sum(df_reg22.total_count).over(Window.partitionBy().orderBy().rowsBetween(-sys.maxsize, 0)))\n",
    "              .withColumn('cum_target_cap_rate',f.col(\"cum_target\")/f.col(\"cum_total\"))\n",
    "              .withColumn('cum_target_cap_of_total',((f.col(\"cum_target\")/tot_target))*100)\n",
    "            )\n",
    "  return cum_sum"
   ],
   "metadata": {
    "application/vnd.databricks.v1+cell": {
     "showTitle": false,
     "cellMetadata": {
      "rowLimit": 10000,
      "byteLimit": 2048000
     },
     "nuid": "8da8ad76-b9da-449e-a580-32c555694f1e",
     "inputWidgets": {},
     "title": ""
    }
   },
   "outputs": [
    {
     "output_type": "display_data",
     "metadata": {
      "application/vnd.databricks.v1+output": {
       "datasetInfos": [],
       "data": "<div class=\"ansiout\"></div>",
       "removedWidgets": [],
       "addedWidgets": {},
       "metadata": {},
       "type": "html",
       "arguments": {}
      }
     },
     "data": {
      "text/html": [
       "<style scoped>\n",
       "  .ansiout {\n",
       "    display: block;\n",
       "    unicode-bidi: embed;\n",
       "    white-space: pre-wrap;\n",
       "    word-wrap: break-word;\n",
       "    word-break: break-all;\n",
       "    font-family: \"Source Code Pro\", \"Menlo\", monospace;;\n",
       "    font-size: 13px;\n",
       "    color: #555;\n",
       "    margin-left: 4px;\n",
       "    line-height: 19px;\n",
       "  }\n",
       "</style>\n",
       "<div class=\"ansiout\"></div>"
      ]
     }
    }
   ],
   "execution_count": 0
  },
  {
   "cell_type": "code",
   "source": [
    "cum_cap_dt = get_model_stats(\"Artificial Neural Network\",predict_test,eval_ann,\"patient_nbr\",\"readmit_flag\")"
   ],
   "metadata": {
    "application/vnd.databricks.v1+cell": {
     "showTitle": false,
     "cellMetadata": {
      "rowLimit": 10000,
      "byteLimit": 2048000
     },
     "nuid": "588518da-fdcd-44ae-b09f-c7ea44bef137",
     "inputWidgets": {},
     "title": ""
    }
   },
   "outputs": [
    {
     "output_type": "display_data",
     "metadata": {
      "application/vnd.databricks.v1+output": {
       "datasetInfos": [
        {
         "name": "cum_cap_dt",
         "typeStr": "pyspark.sql.dataframe.DataFrame",
         "schema": {
          "fields": [
           {
            "metadata": {},
            "name": "pctl_cat",
            "nullable": true,
            "type": "long"
           },
           {
            "metadata": {},
            "name": "pctl_target_rate",
            "nullable": true,
            "type": "double"
           },
           {
            "metadata": {},
            "name": "pctl_target_total",
            "nullable": true,
            "type": "long"
           },
           {
            "metadata": {},
            "name": "total_count",
            "nullable": false,
            "type": "long"
           },
           {
            "metadata": {},
            "name": "mean_predicted_prob",
            "nullable": true,
            "type": "double"
           },
           {
            "metadata": {},
            "name": "cum_target",
            "nullable": true,
            "type": "long"
           },
           {
            "metadata": {},
            "name": "cum_total",
            "nullable": true,
            "type": "long"
           },
           {
            "metadata": {},
            "name": "cum_target_cap_rate",
            "nullable": true,
            "type": "double"
           },
           {
            "metadata": {},
            "name": "cum_target_cap_of_total",
            "nullable": true,
            "type": "double"
           }
          ],
          "type": "struct"
         },
         "tableIdentifier": null
        }
       ],
       "data": "<div class=\"ansiout\">Artificial Neural Network\nIncidence rate in dataset:  0.11225410067076264\nAUROC:  0.6345223971936924\n</div>",
       "removedWidgets": [],
       "addedWidgets": {},
       "metadata": {},
       "type": "html",
       "arguments": {}
      }
     },
     "data": {
      "text/html": [
       "<style scoped>\n",
       "  .ansiout {\n",
       "    display: block;\n",
       "    unicode-bidi: embed;\n",
       "    white-space: pre-wrap;\n",
       "    word-wrap: break-word;\n",
       "    word-break: break-all;\n",
       "    font-family: \"Source Code Pro\", \"Menlo\", monospace;;\n",
       "    font-size: 13px;\n",
       "    color: #555;\n",
       "    margin-left: 4px;\n",
       "    line-height: 19px;\n",
       "  }\n",
       "</style>\n",
       "<div class=\"ansiout\">Artificial Neural Network\n",
       "Incidence rate in dataset:  0.11225410067076264\n",
       "AUROC:  0.6345223971936924\n",
       "</div>"
      ]
     }
    }
   ],
   "execution_count": 0
  },
  {
   "cell_type": "code",
   "source": [
    "display(cum_cap_dt)"
   ],
   "metadata": {
    "application/vnd.databricks.v1+cell": {
     "showTitle": false,
     "cellMetadata": {
      "rowLimit": 10000,
      "byteLimit": 2048000
     },
     "nuid": "1c932fbe-af75-4db8-80d2-1a42efc169f0",
     "inputWidgets": {},
     "title": ""
    }
   },
   "outputs": [
    {
     "output_type": "display_data",
     "metadata": {
      "application/vnd.databricks.v1+output": {
       "overflow": false,
       "datasetInfos": [],
       "data": [
        [
         1,
         0.4253521126760563,
         151,
         355,
         0.3884231478395596,
         151,
         355,
         0.4253521126760563,
         3.7911122269646
        ],
        [
         2,
         0.29577464788732394,
         105,
         355,
         0.32032184172684036,
         256,
         710,
         0.36056338028169016,
         6.427316093396937
        ],
        [
         3,
         0.2704225352112676,
         96,
         355,
         0.28197051102006937,
         352,
         1065,
         0.3305164319248826,
         8.837559628420788
        ],
        [
         4,
         0.22535211267605634,
         80,
         355,
         0.2534388258423604,
         432,
         1420,
         0.30422535211267604,
         10.846095907607332
        ],
        [
         5,
         0.2028169014084507,
         72,
         355,
         0.23392344494940528,
         504,
         1775,
         0.28394366197183096,
         12.65377855887522
        ],
        [
         6,
         0.192090395480226,
         68,
         354,
         0.22065006550085747,
         572,
         2129,
         0.2686707374354157,
         14.361034396183781
        ],
        [
         7,
         0.19718309859154928,
         70,
         355,
         0.2064826678222334,
         642,
         2484,
         0.2584541062801932,
         16.118503640472007
        ],
        [
         8,
         0.2084507042253521,
         74,
         355,
         0.19442139511377038,
         716,
         2839,
         0.2522014793941529,
         17.976399698719558
        ],
        [
         9,
         0.19436619718309858,
         69,
         355,
         0.18582005047462355,
         785,
         3194,
         0.24577332498434565,
         19.708762239517952
        ],
        [
         10,
         0.2028169014084507,
         72,
         355,
         0.17912935396315346,
         857,
         3549,
         0.241476472245703,
         21.51644489078584
        ],
        [
         11,
         0.14124293785310735,
         50,
         354,
         0.1731629910863052,
         907,
         3903,
         0.23238534460671278,
         22.77178006527743
        ],
        [
         12,
         0.17183098591549295,
         61,
         355,
         0.16734142630872592,
         968,
         4258,
         0.2273367778299671,
         24.303288978157166
        ],
        [
         13,
         0.17746478873239438,
         63,
         355,
         0.16181650585691693,
         1031,
         4613,
         0.2234988077173206,
         25.88501129801657
        ],
        [
         14,
         0.17746478873239438,
         63,
         355,
         0.15608739508709438,
         1094,
         4968,
         0.22020933977455717,
         27.46673361787597
        ],
        [
         15,
         0.14366197183098592,
         51,
         355,
         0.15095230638980867,
         1145,
         5323,
         0.21510426451249295,
         28.747175495857398
        ],
        [
         16,
         0.1807909604519774,
         64,
         354,
         0.14689100773657782,
         1209,
         5677,
         0.2129645939756914,
         30.35400451920663
        ],
        [
         17,
         0.10704225352112676,
         38,
         355,
         0.14341522853978922,
         1247,
         6032,
         0.20673076923076922,
         31.308059251820236
        ],
        [
         18,
         0.18309859154929578,
         65,
         355,
         0.14052970023222372,
         1312,
         6387,
         0.2054172537967747,
         32.93999497865931
        ],
        [
         19,
         0.14929577464788732,
         53,
         355,
         0.13800249326396996,
         1365,
         6742,
         0.20246217739543163,
         34.27065026362039
        ],
        [
         20,
         0.09577464788732394,
         34,
         355,
         0.1356502417527454,
         1399,
         7097,
         0.19712554600535437,
         35.124278182274665
        ],
        [
         21,
         0.14647887323943662,
         52,
         355,
         0.13351187798338876,
         1451,
         7452,
         0.1947128287707998,
         36.42982676374592
        ],
        [
         22,
         0.1271186440677966,
         45,
         354,
         0.13180361511343616,
         1496,
         7806,
         0.1916474506789649,
         37.55962842078835
        ],
        [
         23,
         0.15774647887323945,
         56,
         355,
         0.1299929379577368,
         1552,
         8161,
         0.19017277294449209,
         38.96560381621893
        ],
        [
         24,
         0.11549295774647887,
         41,
         355,
         0.1280814397502953,
         1593,
         8516,
         0.18705965241897604,
         39.994978659302035
        ],
        [
         25,
         0.10422535211267606,
         37,
         355,
         0.12608976599196314,
         1630,
         8871,
         0.18374478638259498,
         40.92392668842581
        ],
        [
         26,
         0.09295774647887324,
         33,
         355,
         0.12449656172957219,
         1663,
         9226,
         0.1802514632560156,
         41.75244790359026
        ],
        [
         27,
         0.13559322033898305,
         48,
         354,
         0.12266049279415675,
         1711,
         9580,
         0.17860125260960333,
         42.957569671102185
        ],
        [
         28,
         0.1352112676056338,
         48,
         355,
         0.12072432662399722,
         1759,
         9935,
         0.1770508303975843,
         44.16269143861411
        ],
        [
         29,
         0.10140845070422536,
         36,
         355,
         0.11895495829447894,
         1795,
         10290,
         0.17444120505344995,
         45.06653276424805
        ],
        [
         30,
         0.12112676056338029,
         43,
         355,
         0.11726084183639204,
         1838,
         10645,
         0.17266322217003288,
         46.14612101431082
        ],
        [
         31,
         0.14366197183098592,
         51,
         355,
         0.11583640317262059,
         1889,
         11000,
         0.17172727272727273,
         47.42656289229224
        ],
        [
         32,
         0.12429378531073447,
         44,
         354,
         0.11450824766593465,
         1933,
         11354,
         0.1702483706182843,
         48.53125784584484
        ],
        [
         33,
         0.10985915492957747,
         39,
         355,
         0.11323486572839844,
         1972,
         11709,
         0.16841745665727217,
         49.51041928194828
        ],
        [
         34,
         0.11267605633802817,
         40,
         355,
         0.11202819926637998,
         2012,
         12064,
         0.16677718832891247,
         50.51468742154155
        ],
        [
         35,
         0.1352112676056338,
         48,
         355,
         0.11081546912310829,
         2060,
         12419,
         0.16587486915210564,
         51.71980918905348
        ],
        [
         36,
         0.1323943661971831,
         47,
         355,
         0.10972124850246268,
         2107,
         12774,
         0.16494441834977297,
         52.899824253075565
        ],
        [
         37,
         0.11016949152542373,
         39,
         354,
         0.10873234025755171,
         2146,
         13128,
         0.16346739792809262,
         53.87898568917902
        ],
        [
         38,
         0.11267605633802817,
         40,
         355,
         0.10789228809971205,
         2186,
         13483,
         0.16213008974263887,
         54.88325382877228
        ],
        [
         39,
         0.11267605633802817,
         40,
         355,
         0.10702600502212282,
         2226,
         13838,
         0.1608613961555138,
         55.88752196836555
        ],
        [
         40,
         0.13802816901408452,
         49,
         355,
         0.10612402333759925,
         2275,
         14193,
         0.16029028394278869,
         57.11775043936731
        ],
        [
         41,
         0.11267605633802817,
         40,
         355,
         0.10526854622531945,
         2315,
         14548,
         0.15912840252955732,
         58.12201857896058
        ],
        [
         42,
         0.12112676056338029,
         43,
         355,
         0.1044047215035264,
         2358,
         14903,
         0.15822317654163592,
         59.20160682902335
        ],
        [
         43,
         0.1440677966101695,
         51,
         354,
         0.10375828363296002,
         2409,
         15257,
         0.15789473684210525,
         60.482048707004765
        ],
        [
         44,
         0.10422535211267606,
         37,
         355,
         0.10309044899235309,
         2446,
         15612,
         0.15667435306174737,
         61.410996736128546
        ],
        [
         45,
         0.08169014084507042,
         29,
         355,
         0.10237520570066613,
         2475,
         15967,
         0.1550072023548569,
         62.139091137333665
        ],
        [
         46,
         0.09859154929577464,
         35,
         355,
         0.1014477218750497,
         2510,
         16322,
         0.15378017399828453,
         63.01782575947779
        ],
        [
         47,
         0.12394366197183099,
         44,
         355,
         0.10046739817505151,
         2554,
         16677,
         0.15314505006895723,
         64.12252071303038
        ],
        [
         48,
         0.12994350282485875,
         46,
         354,
         0.09946332939263791,
         2600,
         17031,
         0.15266279138042393,
         65.27742907356264
        ],
        [
         49,
         0.10704225352112676,
         38,
         355,
         0.09895537925857893,
         2638,
         17386,
         0.15173127803980213,
         66.23148380617624
        ],
        [
         50,
         0.09295774647887324,
         33,
         355,
         0.09836249003108119,
         2671,
         17741,
         0.15055521109294853,
         67.0600050213407
        ],
        [
         51,
         0.09295774647887324,
         33,
         355,
         0.09763700968362915,
         2704,
         18096,
         0.14942528735632185,
         67.88852623650514
        ],
        [
         52,
         0.09014084507042254,
         32,
         355,
         0.09678679179977363,
         2736,
         18451,
         0.1482846458186548,
         68.69194074817977
        ],
        [
         53,
         0.11016949152542373,
         39,
         354,
         0.09593509689057614,
         2775,
         18805,
         0.14756713639989363,
         69.6711021842832
        ],
        [
         54,
         0.09014084507042254,
         32,
         355,
         0.0949755105124393,
         2807,
         19160,
         0.14650313152400835,
         70.47451669595782
        ],
        [
         55,
         0.09295774647887324,
         33,
         355,
         0.09406310284221676,
         2840,
         19515,
         0.14552908019472202,
         71.30303791112227
        ],
        [
         56,
         0.11267605633802817,
         40,
         355,
         0.0936338892705004,
         2880,
         19870,
         0.14494212380473076,
         72.30730605071554
        ],
        [
         57,
         0.07605633802816901,
         27,
         355,
         0.0930126912786927,
         2907,
         20225,
         0.14373300370828182,
         72.985187044941
        ],
        [
         58,
         0.08757062146892655,
         31,
         354,
         0.09226315952626998,
         2938,
         20579,
         0.14276689829437778,
         73.76349485312579
        ],
        [
         59,
         0.07323943661971831,
         26,
         355,
         0.09139051500340582,
         2964,
         20934,
         0.1415878475207796,
         74.41626914386141
        ],
        [
         60,
         0.10704225352112676,
         38,
         355,
         0.09059923121207197,
         3002,
         21289,
         0.14101179012635634,
         75.37032387647503
        ],
        [
         61,
         0.09014084507042254,
         32,
         355,
         0.08986382553694953,
         3034,
         21644,
         0.14017741637405284,
         76.17373838814963
        ],
        [
         62,
         0.07605633802816901,
         27,
         355,
         0.0889231166999105,
         3061,
         21999,
         0.13914268830401383,
         76.85161938237509
        ],
        [
         63,
         0.0647887323943662,
         23,
         355,
         0.0877928635184194,
         3084,
         22354,
         0.13796188601592557,
         77.42907356264122
        ],
        [
         64,
         0.07627118644067797,
         27,
         354,
         0.08717691626841739,
         3111,
         22708,
         0.13700017614937468,
         78.10695455686668
        ],
        [
         65,
         0.08732394366197183,
         31,
         355,
         0.08656463461442733,
         3142,
         23063,
         0.13623552876902398,
         78.88526236505146
        ],
        [
         66,
         0.07323943661971831,
         26,
         355,
         0.08586267900299019,
         3168,
         23418,
         0.13528055342044581,
         79.5380366557871
        ],
        [
         67,
         0.07605633802816901,
         27,
         355,
         0.08503971217383802,
         3195,
         23773,
         0.13439616371513902,
         80.21591765001256
        ],
        [
         68,
         0.10704225352112676,
         38,
         355,
         0.0841493159952298,
         3233,
         24128,
         0.13399370026525198,
         81.16997238262617
        ],
        [
         69,
         0.09322033898305085,
         33,
         354,
         0.08322593401380851,
         3266,
         24482,
         0.13340413364921166,
         81.99849359779061
        ],
        [
         70,
         0.09577464788732394,
         34,
         355,
         0.08224150597209662,
         3300,
         24837,
         0.13286628819905785,
         82.85212151644488
        ],
        [
         71,
         0.10140845070422536,
         36,
         355,
         0.08132559860676107,
         3336,
         25192,
         0.13242299142584948,
         83.75596284207883
        ],
        [
         72,
         0.0647887323943662,
         23,
         355,
         0.08038749350628382,
         3359,
         25547,
         0.13148314870630604,
         84.33341702234497
        ],
        [
         73,
         0.08732394366197183,
         31,
         355,
         0.07950322779131608,
         3390,
         25902,
         0.1308779244845958,
         85.11172483052975
        ],
        [
         74,
         0.0847457627118644,
         30,
         354,
         0.07893058379827919,
         3420,
         26256,
         0.13025594149908593,
         85.8649259352247
        ],
        [
         75,
         0.061971830985915494,
         22,
         355,
         0.07819679750942848,
         3442,
         26611,
         0.12934500770358123,
         86.41727341200101
        ],
        [
         76,
         0.08169014084507042,
         29,
         355,
         0.0773164225506111,
         3471,
         26966,
         0.1287176444411481,
         87.14536781320614
        ],
        [
         77,
         0.061971830985915494,
         22,
         355,
         0.07629270540996337,
         3493,
         27321,
         0.12785037150909556,
         87.69771528998243
        ],
        [
         78,
         0.08169014084507042,
         29,
         355,
         0.07535752498767745,
         3522,
         27676,
         0.12725827431709785,
         88.42580969118755
        ],
        [
         79,
         0.0847457627118644,
         30,
         354,
         0.07463387364132257,
         3552,
         28030,
         0.12672136996075634,
         89.17901079588249
        ],
        [
         80,
         0.07323943661971831,
         26,
         355,
         0.07377993229409338,
         3578,
         28385,
         0.12605249251365158,
         89.83178508661813
        ],
        [
         81,
         0.056338028169014086,
         20,
         355,
         0.07272688353985128,
         3598,
         28740,
         0.12519137091162144,
         90.33391915641477
        ],
        [
         82,
         0.07042253521126761,
         25,
         355,
         0.07156384410992475,
         3623,
         29095,
         0.1245231139371026,
         90.96158674366056
        ],
        [
         83,
         0.08169014084507042,
         29,
         355,
         0.07086619982417201,
         3652,
         29450,
         0.12400679117147707,
         91.68968114486567
        ],
        [
         84,
         0.05070422535211268,
         18,
         355,
         0.06985554164144354,
         3670,
         29805,
         0.12313370239892635,
         92.14160180768265
        ],
        [
         85,
         0.06497175141242938,
         23,
         354,
         0.06870297548797845,
         3693,
         30159,
         0.12245100964886103,
         92.71905598794878
        ],
        [
         86,
         0.061971830985915494,
         22,
         355,
         0.06758536031128655,
         3715,
         30514,
         0.12174739463852657,
         93.27140346472508
        ],
        [
         87,
         0.07323943661971831,
         26,
         355,
         0.06674885984877466,
         3741,
         30869,
         0.12118954290712365,
         93.9241777554607
        ],
        [
         88,
         0.056338028169014086,
         20,
         355,
         0.06559654929688279,
         3761,
         31224,
         0.12045221624391493,
         94.42631182525734
        ],
        [
         89,
         0.059154929577464786,
         21,
         355,
         0.0642276300930641,
         3782,
         31579,
         0.11976313372811045,
         94.95355259854381
        ],
        [
         90,
         0.05649717514124294,
         20,
         354,
         0.06326573575704784,
         3802,
         31933,
         0.11906178561362854,
         95.45568666834045
        ],
        [
         91,
         0.05352112676056338,
         19,
         355,
         0.062236306732389286,
         3821,
         32288,
         0.11834117938553022,
         95.93271403464725
        ],
        [
         92,
         0.0647887323943662,
         23,
         355,
         0.06090217318753122,
         3844,
         32643,
         0.1177587844254511,
         96.51016821491338
        ],
        [
         93,
         0.04507042253521127,
         16,
         355,
         0.05965756260803048,
         3860,
         32998,
         0.11697678647190739,
         96.91187547075069
        ],
        [
         94,
         0.07323943661971831,
         26,
         355,
         0.05846308316563217,
         3886,
         33353,
         0.11651125835756904,
         97.56464976148632
        ],
        [
         95,
         0.03954802259887006,
         14,
         354,
         0.057092468975122365,
         3900,
         33707,
         0.11570296970955588,
         97.91614361034397
        ],
        [
         96,
         0.0647887323943662,
         23,
         355,
         0.05564146268535668,
         3923,
         34062,
         0.1151723328048852,
         98.49359779061008
        ],
        [
         97,
         0.04507042253521127,
         16,
         355,
         0.05374305768751762,
         3939,
         34417,
         0.11444925472876775,
         98.8953050464474
        ],
        [
         98,
         0.04225352112676056,
         15,
         355,
         0.051717562165478584,
         3954,
         34772,
         0.11371218221557575,
         99.27190559879487
        ],
        [
         99,
         0.03380281690140845,
         12,
         355,
         0.048381851067845254,
         3966,
         35127,
         0.11290460329660945,
         99.57318604067285
        ],
        [
         100,
         0.04788732394366197,
         17,
         355,
         0.04269634957464648,
         3983,
         35482,
         0.11225410067076264,
         100.0
        ]
       ],
       "plotOptions": {
        "displayType": "table",
        "customPlotOptions": {},
        "pivotColumns": null,
        "pivotAggregation": null,
        "xColumns": null,
        "yColumns": null
       },
       "columnCustomDisplayInfos": {},
       "aggType": "",
       "isJsonSchema": true,
       "removedWidgets": [],
       "aggSchema": [],
       "schema": [
        {
         "name": "pctl_cat",
         "type": "\"long\"",
         "metadata": "{}"
        },
        {
         "name": "pctl_target_rate",
         "type": "\"double\"",
         "metadata": "{}"
        },
        {
         "name": "pctl_target_total",
         "type": "\"long\"",
         "metadata": "{}"
        },
        {
         "name": "total_count",
         "type": "\"long\"",
         "metadata": "{}"
        },
        {
         "name": "mean_predicted_prob",
         "type": "\"double\"",
         "metadata": "{}"
        },
        {
         "name": "cum_target",
         "type": "\"long\"",
         "metadata": "{}"
        },
        {
         "name": "cum_total",
         "type": "\"long\"",
         "metadata": "{}"
        },
        {
         "name": "cum_target_cap_rate",
         "type": "\"double\"",
         "metadata": "{}"
        },
        {
         "name": "cum_target_cap_of_total",
         "type": "\"double\"",
         "metadata": "{}"
        }
       ],
       "aggError": "",
       "aggData": [],
       "addedWidgets": {},
       "metadata": {},
       "dbfsResultPath": null,
       "type": "table",
       "aggOverflow": false,
       "aggSeriesLimitReached": false,
       "arguments": {}
      }
     },
     "data": {
      "text/html": [
       "<style scoped>\n",
       "  .table-result-container {\n",
       "    max-height: 300px;\n",
       "    overflow: auto;\n",
       "  }\n",
       "  table, th, td {\n",
       "    border: 1px solid black;\n",
       "    border-collapse: collapse;\n",
       "  }\n",
       "  th, td {\n",
       "    padding: 5px;\n",
       "  }\n",
       "  th {\n",
       "    text-align: left;\n",
       "  }\n",
       "</style><div class='table-result-container'><table class='table-result'><thead style='background-color: white'><tr><th>pctl_cat</th><th>pctl_target_rate</th><th>pctl_target_total</th><th>total_count</th><th>mean_predicted_prob</th><th>cum_target</th><th>cum_total</th><th>cum_target_cap_rate</th><th>cum_target_cap_of_total</th></tr></thead><tbody><tr><td>1</td><td>0.4253521126760563</td><td>151</td><td>355</td><td>0.3884231478395596</td><td>151</td><td>355</td><td>0.4253521126760563</td><td>3.7911122269646</td></tr><tr><td>2</td><td>0.29577464788732394</td><td>105</td><td>355</td><td>0.32032184172684036</td><td>256</td><td>710</td><td>0.36056338028169016</td><td>6.427316093396937</td></tr><tr><td>3</td><td>0.2704225352112676</td><td>96</td><td>355</td><td>0.28197051102006937</td><td>352</td><td>1065</td><td>0.3305164319248826</td><td>8.837559628420788</td></tr><tr><td>4</td><td>0.22535211267605634</td><td>80</td><td>355</td><td>0.2534388258423604</td><td>432</td><td>1420</td><td>0.30422535211267604</td><td>10.846095907607332</td></tr><tr><td>5</td><td>0.2028169014084507</td><td>72</td><td>355</td><td>0.23392344494940528</td><td>504</td><td>1775</td><td>0.28394366197183096</td><td>12.65377855887522</td></tr><tr><td>6</td><td>0.192090395480226</td><td>68</td><td>354</td><td>0.22065006550085747</td><td>572</td><td>2129</td><td>0.2686707374354157</td><td>14.361034396183781</td></tr><tr><td>7</td><td>0.19718309859154928</td><td>70</td><td>355</td><td>0.2064826678222334</td><td>642</td><td>2484</td><td>0.2584541062801932</td><td>16.118503640472007</td></tr><tr><td>8</td><td>0.2084507042253521</td><td>74</td><td>355</td><td>0.19442139511377038</td><td>716</td><td>2839</td><td>0.2522014793941529</td><td>17.976399698719558</td></tr><tr><td>9</td><td>0.19436619718309858</td><td>69</td><td>355</td><td>0.18582005047462355</td><td>785</td><td>3194</td><td>0.24577332498434565</td><td>19.708762239517952</td></tr><tr><td>10</td><td>0.2028169014084507</td><td>72</td><td>355</td><td>0.17912935396315346</td><td>857</td><td>3549</td><td>0.241476472245703</td><td>21.51644489078584</td></tr><tr><td>11</td><td>0.14124293785310735</td><td>50</td><td>354</td><td>0.1731629910863052</td><td>907</td><td>3903</td><td>0.23238534460671278</td><td>22.77178006527743</td></tr><tr><td>12</td><td>0.17183098591549295</td><td>61</td><td>355</td><td>0.16734142630872592</td><td>968</td><td>4258</td><td>0.2273367778299671</td><td>24.303288978157166</td></tr><tr><td>13</td><td>0.17746478873239438</td><td>63</td><td>355</td><td>0.16181650585691693</td><td>1031</td><td>4613</td><td>0.2234988077173206</td><td>25.88501129801657</td></tr><tr><td>14</td><td>0.17746478873239438</td><td>63</td><td>355</td><td>0.15608739508709438</td><td>1094</td><td>4968</td><td>0.22020933977455717</td><td>27.46673361787597</td></tr><tr><td>15</td><td>0.14366197183098592</td><td>51</td><td>355</td><td>0.15095230638980867</td><td>1145</td><td>5323</td><td>0.21510426451249295</td><td>28.747175495857398</td></tr><tr><td>16</td><td>0.1807909604519774</td><td>64</td><td>354</td><td>0.14689100773657782</td><td>1209</td><td>5677</td><td>0.2129645939756914</td><td>30.35400451920663</td></tr><tr><td>17</td><td>0.10704225352112676</td><td>38</td><td>355</td><td>0.14341522853978922</td><td>1247</td><td>6032</td><td>0.20673076923076922</td><td>31.308059251820236</td></tr><tr><td>18</td><td>0.18309859154929578</td><td>65</td><td>355</td><td>0.14052970023222372</td><td>1312</td><td>6387</td><td>0.2054172537967747</td><td>32.93999497865931</td></tr><tr><td>19</td><td>0.14929577464788732</td><td>53</td><td>355</td><td>0.13800249326396996</td><td>1365</td><td>6742</td><td>0.20246217739543163</td><td>34.27065026362039</td></tr><tr><td>20</td><td>0.09577464788732394</td><td>34</td><td>355</td><td>0.1356502417527454</td><td>1399</td><td>7097</td><td>0.19712554600535437</td><td>35.124278182274665</td></tr><tr><td>21</td><td>0.14647887323943662</td><td>52</td><td>355</td><td>0.13351187798338876</td><td>1451</td><td>7452</td><td>0.1947128287707998</td><td>36.42982676374592</td></tr><tr><td>22</td><td>0.1271186440677966</td><td>45</td><td>354</td><td>0.13180361511343616</td><td>1496</td><td>7806</td><td>0.1916474506789649</td><td>37.55962842078835</td></tr><tr><td>23</td><td>0.15774647887323945</td><td>56</td><td>355</td><td>0.1299929379577368</td><td>1552</td><td>8161</td><td>0.19017277294449209</td><td>38.96560381621893</td></tr><tr><td>24</td><td>0.11549295774647887</td><td>41</td><td>355</td><td>0.1280814397502953</td><td>1593</td><td>8516</td><td>0.18705965241897604</td><td>39.994978659302035</td></tr><tr><td>25</td><td>0.10422535211267606</td><td>37</td><td>355</td><td>0.12608976599196314</td><td>1630</td><td>8871</td><td>0.18374478638259498</td><td>40.92392668842581</td></tr><tr><td>26</td><td>0.09295774647887324</td><td>33</td><td>355</td><td>0.12449656172957219</td><td>1663</td><td>9226</td><td>0.1802514632560156</td><td>41.75244790359026</td></tr><tr><td>27</td><td>0.13559322033898305</td><td>48</td><td>354</td><td>0.12266049279415675</td><td>1711</td><td>9580</td><td>0.17860125260960333</td><td>42.957569671102185</td></tr><tr><td>28</td><td>0.1352112676056338</td><td>48</td><td>355</td><td>0.12072432662399722</td><td>1759</td><td>9935</td><td>0.1770508303975843</td><td>44.16269143861411</td></tr><tr><td>29</td><td>0.10140845070422536</td><td>36</td><td>355</td><td>0.11895495829447894</td><td>1795</td><td>10290</td><td>0.17444120505344995</td><td>45.06653276424805</td></tr><tr><td>30</td><td>0.12112676056338029</td><td>43</td><td>355</td><td>0.11726084183639204</td><td>1838</td><td>10645</td><td>0.17266322217003288</td><td>46.14612101431082</td></tr><tr><td>31</td><td>0.14366197183098592</td><td>51</td><td>355</td><td>0.11583640317262059</td><td>1889</td><td>11000</td><td>0.17172727272727273</td><td>47.42656289229224</td></tr><tr><td>32</td><td>0.12429378531073447</td><td>44</td><td>354</td><td>0.11450824766593465</td><td>1933</td><td>11354</td><td>0.1702483706182843</td><td>48.53125784584484</td></tr><tr><td>33</td><td>0.10985915492957747</td><td>39</td><td>355</td><td>0.11323486572839844</td><td>1972</td><td>11709</td><td>0.16841745665727217</td><td>49.51041928194828</td></tr><tr><td>34</td><td>0.11267605633802817</td><td>40</td><td>355</td><td>0.11202819926637998</td><td>2012</td><td>12064</td><td>0.16677718832891247</td><td>50.51468742154155</td></tr><tr><td>35</td><td>0.1352112676056338</td><td>48</td><td>355</td><td>0.11081546912310829</td><td>2060</td><td>12419</td><td>0.16587486915210564</td><td>51.71980918905348</td></tr><tr><td>36</td><td>0.1323943661971831</td><td>47</td><td>355</td><td>0.10972124850246268</td><td>2107</td><td>12774</td><td>0.16494441834977297</td><td>52.899824253075565</td></tr><tr><td>37</td><td>0.11016949152542373</td><td>39</td><td>354</td><td>0.10873234025755171</td><td>2146</td><td>13128</td><td>0.16346739792809262</td><td>53.87898568917902</td></tr><tr><td>38</td><td>0.11267605633802817</td><td>40</td><td>355</td><td>0.10789228809971205</td><td>2186</td><td>13483</td><td>0.16213008974263887</td><td>54.88325382877228</td></tr><tr><td>39</td><td>0.11267605633802817</td><td>40</td><td>355</td><td>0.10702600502212282</td><td>2226</td><td>13838</td><td>0.1608613961555138</td><td>55.88752196836555</td></tr><tr><td>40</td><td>0.13802816901408452</td><td>49</td><td>355</td><td>0.10612402333759925</td><td>2275</td><td>14193</td><td>0.16029028394278869</td><td>57.11775043936731</td></tr><tr><td>41</td><td>0.11267605633802817</td><td>40</td><td>355</td><td>0.10526854622531945</td><td>2315</td><td>14548</td><td>0.15912840252955732</td><td>58.12201857896058</td></tr><tr><td>42</td><td>0.12112676056338029</td><td>43</td><td>355</td><td>0.1044047215035264</td><td>2358</td><td>14903</td><td>0.15822317654163592</td><td>59.20160682902335</td></tr><tr><td>43</td><td>0.1440677966101695</td><td>51</td><td>354</td><td>0.10375828363296002</td><td>2409</td><td>15257</td><td>0.15789473684210525</td><td>60.482048707004765</td></tr><tr><td>44</td><td>0.10422535211267606</td><td>37</td><td>355</td><td>0.10309044899235309</td><td>2446</td><td>15612</td><td>0.15667435306174737</td><td>61.410996736128546</td></tr><tr><td>45</td><td>0.08169014084507042</td><td>29</td><td>355</td><td>0.10237520570066613</td><td>2475</td><td>15967</td><td>0.1550072023548569</td><td>62.139091137333665</td></tr><tr><td>46</td><td>0.09859154929577464</td><td>35</td><td>355</td><td>0.1014477218750497</td><td>2510</td><td>16322</td><td>0.15378017399828453</td><td>63.01782575947779</td></tr><tr><td>47</td><td>0.12394366197183099</td><td>44</td><td>355</td><td>0.10046739817505151</td><td>2554</td><td>16677</td><td>0.15314505006895723</td><td>64.12252071303038</td></tr><tr><td>48</td><td>0.12994350282485875</td><td>46</td><td>354</td><td>0.09946332939263791</td><td>2600</td><td>17031</td><td>0.15266279138042393</td><td>65.27742907356264</td></tr><tr><td>49</td><td>0.10704225352112676</td><td>38</td><td>355</td><td>0.09895537925857893</td><td>2638</td><td>17386</td><td>0.15173127803980213</td><td>66.23148380617624</td></tr><tr><td>50</td><td>0.09295774647887324</td><td>33</td><td>355</td><td>0.09836249003108119</td><td>2671</td><td>17741</td><td>0.15055521109294853</td><td>67.0600050213407</td></tr><tr><td>51</td><td>0.09295774647887324</td><td>33</td><td>355</td><td>0.09763700968362915</td><td>2704</td><td>18096</td><td>0.14942528735632185</td><td>67.88852623650514</td></tr><tr><td>52</td><td>0.09014084507042254</td><td>32</td><td>355</td><td>0.09678679179977363</td><td>2736</td><td>18451</td><td>0.1482846458186548</td><td>68.69194074817977</td></tr><tr><td>53</td><td>0.11016949152542373</td><td>39</td><td>354</td><td>0.09593509689057614</td><td>2775</td><td>18805</td><td>0.14756713639989363</td><td>69.6711021842832</td></tr><tr><td>54</td><td>0.09014084507042254</td><td>32</td><td>355</td><td>0.0949755105124393</td><td>2807</td><td>19160</td><td>0.14650313152400835</td><td>70.47451669595782</td></tr><tr><td>55</td><td>0.09295774647887324</td><td>33</td><td>355</td><td>0.09406310284221676</td><td>2840</td><td>19515</td><td>0.14552908019472202</td><td>71.30303791112227</td></tr><tr><td>56</td><td>0.11267605633802817</td><td>40</td><td>355</td><td>0.0936338892705004</td><td>2880</td><td>19870</td><td>0.14494212380473076</td><td>72.30730605071554</td></tr><tr><td>57</td><td>0.07605633802816901</td><td>27</td><td>355</td><td>0.0930126912786927</td><td>2907</td><td>20225</td><td>0.14373300370828182</td><td>72.985187044941</td></tr><tr><td>58</td><td>0.08757062146892655</td><td>31</td><td>354</td><td>0.09226315952626998</td><td>2938</td><td>20579</td><td>0.14276689829437778</td><td>73.76349485312579</td></tr><tr><td>59</td><td>0.07323943661971831</td><td>26</td><td>355</td><td>0.09139051500340582</td><td>2964</td><td>20934</td><td>0.1415878475207796</td><td>74.41626914386141</td></tr><tr><td>60</td><td>0.10704225352112676</td><td>38</td><td>355</td><td>0.09059923121207197</td><td>3002</td><td>21289</td><td>0.14101179012635634</td><td>75.37032387647503</td></tr><tr><td>61</td><td>0.09014084507042254</td><td>32</td><td>355</td><td>0.08986382553694953</td><td>3034</td><td>21644</td><td>0.14017741637405284</td><td>76.17373838814963</td></tr><tr><td>62</td><td>0.07605633802816901</td><td>27</td><td>355</td><td>0.0889231166999105</td><td>3061</td><td>21999</td><td>0.13914268830401383</td><td>76.85161938237509</td></tr><tr><td>63</td><td>0.0647887323943662</td><td>23</td><td>355</td><td>0.0877928635184194</td><td>3084</td><td>22354</td><td>0.13796188601592557</td><td>77.42907356264122</td></tr><tr><td>64</td><td>0.07627118644067797</td><td>27</td><td>354</td><td>0.08717691626841739</td><td>3111</td><td>22708</td><td>0.13700017614937468</td><td>78.10695455686668</td></tr><tr><td>65</td><td>0.08732394366197183</td><td>31</td><td>355</td><td>0.08656463461442733</td><td>3142</td><td>23063</td><td>0.13623552876902398</td><td>78.88526236505146</td></tr><tr><td>66</td><td>0.07323943661971831</td><td>26</td><td>355</td><td>0.08586267900299019</td><td>3168</td><td>23418</td><td>0.13528055342044581</td><td>79.5380366557871</td></tr><tr><td>67</td><td>0.07605633802816901</td><td>27</td><td>355</td><td>0.08503971217383802</td><td>3195</td><td>23773</td><td>0.13439616371513902</td><td>80.21591765001256</td></tr><tr><td>68</td><td>0.10704225352112676</td><td>38</td><td>355</td><td>0.0841493159952298</td><td>3233</td><td>24128</td><td>0.13399370026525198</td><td>81.16997238262617</td></tr><tr><td>69</td><td>0.09322033898305085</td><td>33</td><td>354</td><td>0.08322593401380851</td><td>3266</td><td>24482</td><td>0.13340413364921166</td><td>81.99849359779061</td></tr><tr><td>70</td><td>0.09577464788732394</td><td>34</td><td>355</td><td>0.08224150597209662</td><td>3300</td><td>24837</td><td>0.13286628819905785</td><td>82.85212151644488</td></tr><tr><td>71</td><td>0.10140845070422536</td><td>36</td><td>355</td><td>0.08132559860676107</td><td>3336</td><td>25192</td><td>0.13242299142584948</td><td>83.75596284207883</td></tr><tr><td>72</td><td>0.0647887323943662</td><td>23</td><td>355</td><td>0.08038749350628382</td><td>3359</td><td>25547</td><td>0.13148314870630604</td><td>84.33341702234497</td></tr><tr><td>73</td><td>0.08732394366197183</td><td>31</td><td>355</td><td>0.07950322779131608</td><td>3390</td><td>25902</td><td>0.1308779244845958</td><td>85.11172483052975</td></tr><tr><td>74</td><td>0.0847457627118644</td><td>30</td><td>354</td><td>0.07893058379827919</td><td>3420</td><td>26256</td><td>0.13025594149908593</td><td>85.8649259352247</td></tr><tr><td>75</td><td>0.061971830985915494</td><td>22</td><td>355</td><td>0.07819679750942848</td><td>3442</td><td>26611</td><td>0.12934500770358123</td><td>86.41727341200101</td></tr><tr><td>76</td><td>0.08169014084507042</td><td>29</td><td>355</td><td>0.0773164225506111</td><td>3471</td><td>26966</td><td>0.1287176444411481</td><td>87.14536781320614</td></tr><tr><td>77</td><td>0.061971830985915494</td><td>22</td><td>355</td><td>0.07629270540996337</td><td>3493</td><td>27321</td><td>0.12785037150909556</td><td>87.69771528998243</td></tr><tr><td>78</td><td>0.08169014084507042</td><td>29</td><td>355</td><td>0.07535752498767745</td><td>3522</td><td>27676</td><td>0.12725827431709785</td><td>88.42580969118755</td></tr><tr><td>79</td><td>0.0847457627118644</td><td>30</td><td>354</td><td>0.07463387364132257</td><td>3552</td><td>28030</td><td>0.12672136996075634</td><td>89.17901079588249</td></tr><tr><td>80</td><td>0.07323943661971831</td><td>26</td><td>355</td><td>0.07377993229409338</td><td>3578</td><td>28385</td><td>0.12605249251365158</td><td>89.83178508661813</td></tr><tr><td>81</td><td>0.056338028169014086</td><td>20</td><td>355</td><td>0.07272688353985128</td><td>3598</td><td>28740</td><td>0.12519137091162144</td><td>90.33391915641477</td></tr><tr><td>82</td><td>0.07042253521126761</td><td>25</td><td>355</td><td>0.07156384410992475</td><td>3623</td><td>29095</td><td>0.1245231139371026</td><td>90.96158674366056</td></tr><tr><td>83</td><td>0.08169014084507042</td><td>29</td><td>355</td><td>0.07086619982417201</td><td>3652</td><td>29450</td><td>0.12400679117147707</td><td>91.68968114486567</td></tr><tr><td>84</td><td>0.05070422535211268</td><td>18</td><td>355</td><td>0.06985554164144354</td><td>3670</td><td>29805</td><td>0.12313370239892635</td><td>92.14160180768265</td></tr><tr><td>85</td><td>0.06497175141242938</td><td>23</td><td>354</td><td>0.06870297548797845</td><td>3693</td><td>30159</td><td>0.12245100964886103</td><td>92.71905598794878</td></tr><tr><td>86</td><td>0.061971830985915494</td><td>22</td><td>355</td><td>0.06758536031128655</td><td>3715</td><td>30514</td><td>0.12174739463852657</td><td>93.27140346472508</td></tr><tr><td>87</td><td>0.07323943661971831</td><td>26</td><td>355</td><td>0.06674885984877466</td><td>3741</td><td>30869</td><td>0.12118954290712365</td><td>93.9241777554607</td></tr><tr><td>88</td><td>0.056338028169014086</td><td>20</td><td>355</td><td>0.06559654929688279</td><td>3761</td><td>31224</td><td>0.12045221624391493</td><td>94.42631182525734</td></tr><tr><td>89</td><td>0.059154929577464786</td><td>21</td><td>355</td><td>0.0642276300930641</td><td>3782</td><td>31579</td><td>0.11976313372811045</td><td>94.95355259854381</td></tr><tr><td>90</td><td>0.05649717514124294</td><td>20</td><td>354</td><td>0.06326573575704784</td><td>3802</td><td>31933</td><td>0.11906178561362854</td><td>95.45568666834045</td></tr><tr><td>91</td><td>0.05352112676056338</td><td>19</td><td>355</td><td>0.062236306732389286</td><td>3821</td><td>32288</td><td>0.11834117938553022</td><td>95.93271403464725</td></tr><tr><td>92</td><td>0.0647887323943662</td><td>23</td><td>355</td><td>0.06090217318753122</td><td>3844</td><td>32643</td><td>0.1177587844254511</td><td>96.51016821491338</td></tr><tr><td>93</td><td>0.04507042253521127</td><td>16</td><td>355</td><td>0.05965756260803048</td><td>3860</td><td>32998</td><td>0.11697678647190739</td><td>96.91187547075069</td></tr><tr><td>94</td><td>0.07323943661971831</td><td>26</td><td>355</td><td>0.05846308316563217</td><td>3886</td><td>33353</td><td>0.11651125835756904</td><td>97.56464976148632</td></tr><tr><td>95</td><td>0.03954802259887006</td><td>14</td><td>354</td><td>0.057092468975122365</td><td>3900</td><td>33707</td><td>0.11570296970955588</td><td>97.91614361034397</td></tr><tr><td>96</td><td>0.0647887323943662</td><td>23</td><td>355</td><td>0.05564146268535668</td><td>3923</td><td>34062</td><td>0.1151723328048852</td><td>98.49359779061008</td></tr><tr><td>97</td><td>0.04507042253521127</td><td>16</td><td>355</td><td>0.05374305768751762</td><td>3939</td><td>34417</td><td>0.11444925472876775</td><td>98.8953050464474</td></tr><tr><td>98</td><td>0.04225352112676056</td><td>15</td><td>355</td><td>0.051717562165478584</td><td>3954</td><td>34772</td><td>0.11371218221557575</td><td>99.27190559879487</td></tr><tr><td>99</td><td>0.03380281690140845</td><td>12</td><td>355</td><td>0.048381851067845254</td><td>3966</td><td>35127</td><td>0.11290460329660945</td><td>99.57318604067285</td></tr><tr><td>100</td><td>0.04788732394366197</td><td>17</td><td>355</td><td>0.04269634957464648</td><td>3983</td><td>35482</td><td>0.11225410067076264</td><td>100.0</td></tr></tbody></table></div>"
      ]
     }
    }
   ],
   "execution_count": 0
  }
 ],
 "metadata": {
  "application/vnd.databricks.v1+notebook": {
   "notebookName": "Diabetes Readmission prediction using ANN",
   "dashboards": [],
   "notebookMetadata": {
    "pythonIndentUnit": 2,
    "mostRecentlyExecutedCommandWithImplicitDF": {
     "commandId": 4107560357685426,
     "dataframes": [
      "_sqldf"
     ]
    }
   },
   "language": "python",
   "widgets": {}
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}