07. Tree Based Models.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Random Forest Using PySpark\n",
    "\n",
    "This is just a quick walkthrough of the Documentation's Example of Random Forest:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import findspark\n",
    "findspark.init()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "from pyspark.ml import Pipeline\n",
    "from pyspark.ml.classification import RandomForestClassifier\n",
    "from pyspark.ml.evaluation import MulticlassClassificationEvaluator"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "from pyspark.sql import SparkSession\n",
    "spark = SparkSession.builder.appName('rf').getOrCreate()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Load and parse the data file, converting it to a DataFrame.\n",
    "data = spark.read.format(\"libsvm\").load(\"data/sample_libsvm_data.txt\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+-----+--------------------+\n",
      "|label|            features|\n",
      "+-----+--------------------+\n",
      "|  0.0|(692,[127,128,129...|\n",
      "|  1.0|(692,[158,159,160...|\n",
      "|  1.0|(692,[124,125,126...|\n",
      "|  1.0|(692,[152,153,154...|\n",
      "|  1.0|(692,[151,152,153...|\n",
      "|  0.0|(692,[129,130,131...|\n",
      "|  1.0|(692,[158,159,160...|\n",
      "|  1.0|(692,[99,100,101,...|\n",
      "|  0.0|(692,[154,155,156...|\n",
      "|  0.0|(692,[127,128,129...|\n",
      "|  1.0|(692,[154,155,156...|\n",
      "|  0.0|(692,[153,154,155...|\n",
      "|  0.0|(692,[151,152,153...|\n",
      "|  1.0|(692,[129,130,131...|\n",
      "|  0.0|(692,[154,155,156...|\n",
      "|  1.0|(692,[150,151,152...|\n",
      "|  0.0|(692,[124,125,126...|\n",
      "|  0.0|(692,[152,153,154...|\n",
      "|  1.0|(692,[97,98,99,12...|\n",
      "|  1.0|(692,[124,125,126...|\n",
      "+-----+--------------------+\n",
      "only showing top 20 rows\n",
      "\n"
     ]
    }
   ],
   "source": [
    "data.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Row(label=0.0, features=SparseVector(692, {127: 51.0, 128: 159.0, 129: 253.0, 130: 159.0, 131: 50.0, 154: 48.0, 155: 238.0, 156: 252.0, 157: 252.0, 158: 252.0, 159: 237.0, 181: 54.0, 182: 227.0, 183: 253.0, 184: 252.0, 185: 239.0, 186: 233.0, 187: 252.0, 188: 57.0, 189: 6.0, 207: 10.0, 208: 60.0, 209: 224.0, 210: 252.0, 211: 253.0, 212: 252.0, 213: 202.0, 214: 84.0, 215: 252.0, 216: 253.0, 217: 122.0, 235: 163.0, 236: 252.0, 237: 252.0, 238: 252.0, 239: 253.0, 240: 252.0, 241: 252.0, 242: 96.0, 243: 189.0, 244: 253.0, 245: 167.0, 262: 51.0, 263: 238.0, 264: 253.0, 265: 253.0, 266: 190.0, 267: 114.0, 268: 253.0, 269: 228.0, 270: 47.0, 271: 79.0, 272: 255.0, 273: 168.0, 289: 48.0, 290: 238.0, 291: 252.0, 292: 252.0, 293: 179.0, 294: 12.0, 295: 75.0, 296: 121.0, 297: 21.0, 300: 253.0, 301: 243.0, 302: 50.0, 316: 38.0, 317: 165.0, 318: 253.0, 319: 233.0, 320: 208.0, 321: 84.0, 328: 253.0, 329: 252.0, 330: 165.0, 343: 7.0, 344: 178.0, 345: 252.0, 346: 240.0, 347: 71.0, 348: 19.0, 349: 28.0, 356: 253.0, 357: 252.0, 358: 195.0, 371: 57.0, 372: 252.0, 373: 252.0, 374: 63.0, 384: 253.0, 385: 252.0, 386: 195.0, 399: 198.0, 400: 253.0, 401: 190.0, 412: 255.0, 413: 253.0, 414: 196.0, 426: 76.0, 427: 246.0, 428: 252.0, 429: 112.0, 440: 253.0, 441: 252.0, 442: 148.0, 454: 85.0, 455: 252.0, 456: 230.0, 457: 25.0, 466: 7.0, 467: 135.0, 468: 253.0, 469: 186.0, 470: 12.0, 482: 85.0, 483: 252.0, 484: 223.0, 493: 7.0, 494: 131.0, 495: 252.0, 496: 225.0, 497: 71.0, 510: 85.0, 511: 252.0, 512: 145.0, 520: 48.0, 521: 165.0, 522: 252.0, 523: 173.0, 538: 86.0, 539: 253.0, 540: 225.0, 547: 114.0, 548: 238.0, 549: 253.0, 550: 162.0, 566: 85.0, 567: 252.0, 568: 249.0, 569: 146.0, 570: 48.0, 571: 29.0, 572: 85.0, 573: 178.0, 574: 225.0, 575: 253.0, 576: 223.0, 577: 167.0, 578: 56.0, 594: 85.0, 595: 252.0, 596: 252.0, 597: 252.0, 598: 229.0, 599: 215.0, 600: 252.0, 601: 252.0, 602: 252.0, 603: 196.0, 604: 130.0, 622: 28.0, 623: 199.0, 624: 252.0, 625: 252.0, 626: 253.0, 627: 252.0, 628: 252.0, 629: 233.0, 630: 145.0, 651: 25.0, 652: 128.0, 653: 252.0, 654: 253.0, 655: 252.0, 656: 141.0, 657: 37.0}))"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Split the data into training and test sets (30% held out for testing)\n",
    "(trainingData, testData) = data.randomSplit([0.7, 0.3])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "root\n",
      " |-- label: double (nullable = true)\n",
      " |-- features: vector (nullable = true)\n",
      "\n"
     ]
    }
   ],
   "source": [
    "trainingData.printSchema()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Train a RandomForest model.\n",
    "rf = RandomForestClassifier(labelCol=\"label\", featuresCol=\"features\", numTrees=20)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Train model.  This also runs the indexers.\n",
    "model = rf.fit(trainingData)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Make predictions.\n",
    "predictions = model.transform(testData)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "root\n",
      " |-- label: double (nullable = true)\n",
      " |-- features: vector (nullable = true)\n",
      " |-- rawPrediction: vector (nullable = true)\n",
      " |-- probability: vector (nullable = true)\n",
      " |-- prediction: double (nullable = false)\n",
      "\n"
     ]
    }
   ],
   "source": [
    "predictions.printSchema()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+----------+-----+--------------------+\n",
      "|prediction|label|            features|\n",
      "+----------+-----+--------------------+\n",
      "|       0.0|  0.0|(692,[122,123,148...|\n",
      "|       0.0|  0.0|(692,[124,125,126...|\n",
      "|       0.0|  0.0|(692,[124,125,126...|\n",
      "|       0.0|  0.0|(692,[124,125,126...|\n",
      "|       0.0|  0.0|(692,[127,128,129...|\n",
      "+----------+-----+--------------------+\n",
      "only showing top 5 rows\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# Select example rows to display.\n",
    "predictions.select(\"prediction\", \"label\", \"features\").show(5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Select (prediction, true label) and compute test error\n",
    "evaluator = MulticlassClassificationEvaluator(labelCol=\"label\", predictionCol=\"prediction\", metricName=\"accuracy\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Test Error = 0\n"
     ]
    }
   ],
   "source": [
    "accuracy = evaluator.evaluate(predictions)\n",
    "print(\"Test Error = %g\" % (1.0 - accuracy))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "SparseVector(692, {161: 0.003, 208: 0.0053, 209: 0.0078, 236: 0.0363, 241: 0.0036, 243: 0.0097, 263: 0.0317, 271: 0.0351, 295: 0.0029, 322: 0.0164, 351: 0.044, 353: 0.0021, 373: 0.0029, 374: 0.0071, 377: 0.0471, 378: 0.0166, 379: 0.0421, 384: 0.0336, 386: 0.0041, 405: 0.003, 407: 0.0539, 413: 0.0463, 431: 0.0029, 433: 0.006, 452: 0.004, 460: 0.0403, 482: 0.0031, 484: 0.0471, 488: 0.0349, 489: 0.0136, 490: 0.0108, 491: 0.0151, 496: 0.0037, 509: 0.01, 511: 0.0089, 517: 0.0911, 518: 0.0727, 539: 0.05, 540: 0.0849, 544: 0.0047, 547: 0.0043, 596: 0.0038, 628: 0.0028, 633: 0.0305})"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Not a very good example to show this!\n",
    "model.featureImportances"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Gradient Boosted Trees\n",
    "\n",
    "Gradient-boosted trees (GBTs) are a popular classification and regression method using ensembles of decision trees. More information about the spark.ml implementation can be found further in the section on [GBTs.](http://spark.apache.org/docs/latest/ml-classification-regression.html#gradient-boosted-trees-gbts). For more information on the algorithm itself, please see the [spark.mllib documentation on GBTs.](http://spark.apache.org/docs/latest/mllib-ensembles.html#gradient-boosted-trees-gbts)\n",
    "\n",
    "Luckily Spark makes very easy to use, basically just an import switch:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+----------+-----+--------------------+\n",
      "|prediction|label|            features|\n",
      "+----------+-----+--------------------+\n",
      "|       0.0|  0.0|(692,[95,96,97,12...|\n",
      "|       0.0|  0.0|(692,[98,99,100,1...|\n",
      "|       0.0|  0.0|(692,[121,122,123...|\n",
      "|       0.0|  0.0|(692,[124,125,126...|\n",
      "|       0.0|  0.0|(692,[124,125,126...|\n",
      "+----------+-----+--------------------+\n",
      "only showing top 5 rows\n",
      "\n"
     ]
    }
   ],
   "source": [
    "from pyspark.ml.classification import GBTClassifier\n",
    "\n",
    "# Load and parse the data file, converting it to a DataFrame.\n",
    "data = spark.read.format(\"libsvm\").load(\"data/sample_libsvm_data.txt\")\n",
    "\n",
    "\n",
    "# Split the data into training and test sets (30% held out for testing)\n",
    "(trainingData, testData) = data.randomSplit([0.7, 0.3])\n",
    "\n",
    "# Train a GBT model.\n",
    "gbt = GBTClassifier(labelCol=\"label\", featuresCol=\"features\", maxIter=10)\n",
    "\n",
    "# Train model.  This also runs the indexers.\n",
    "model = gbt.fit(trainingData)\n",
    "\n",
    "# Make predictions.\n",
    "predictions = model.transform(testData)\n",
    "\n",
    "# Select example rows to display.\n",
    "predictions.select(\"prediction\", \"label\", \"features\").show(5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Test Error = 0.0322581\n"
     ]
    }
   ],
   "source": [
    "# Select (prediction, true label) and compute test error\n",
    "evaluator = MulticlassClassificationEvaluator(labelCol=\"label\", predictionCol=\"prediction\", metricName=\"accuracy\")\n",
    "accuracy = evaluator.evaluate(predictions)\n",
    "print(\"Test Error = %g\" % (1.0 - accuracy))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "So this data isn't really realistic enough to really judge to effectiveness of GBT models, this data makes it seem like they are perfection, instead of just an improvement on normal Random Forests."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Let's move on to a more realistic example!"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Tree Methods Code Along\n",
    "\n",
    "In this part we will code along with some data and test out 3 different tree methods:\n",
    "\n",
    "* A single decision tree\n",
    "* A random forest\n",
    "* A gradient boosted tree classifier\n",
    "    \n",
    "We will be using a college dataset to try to classify colleges as Private or Public based off these features:\n",
    "\n",
    "    Private A factor with levels No and Yes indicating private or public university\n",
    "    Apps Number of applications received\n",
    "    Accept Number of applications accepted\n",
    "    Enroll Number of new students enrolled\n",
    "    Top10perc Pct. new students from top 10% of H.S. class\n",
    "    Top25perc Pct. new students from top 25% of H.S. class\n",
    "    F.Undergrad Number of fulltime undergraduates\n",
    "    P.Undergrad Number of parttime undergraduates\n",
    "    Outstate Out-of-state tuition\n",
    "    Room.Board Room and board costs\n",
    "    Books Estimated book costs\n",
    "    Personal Estimated personal spending\n",
    "    PhD Pct. of faculty with Ph.D.’s\n",
    "    Terminal Pct. of faculty with terminal degree\n",
    "    S.F.Ratio Student/faculty ratio\n",
    "    perc.alumni Pct. alumni who donate\n",
    "    Expend Instructional expenditure per student\n",
    "    Grad.Rate Graduation rate"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Load training data\n",
    "data = spark.read.csv('data/College.csv',inferSchema=True,header=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "root\n",
      " |-- School: string (nullable = true)\n",
      " |-- Private: string (nullable = true)\n",
      " |-- Apps: integer (nullable = true)\n",
      " |-- Accept: integer (nullable = true)\n",
      " |-- Enroll: integer (nullable = true)\n",
      " |-- Top10perc: integer (nullable = true)\n",
      " |-- Top25perc: integer (nullable = true)\n",
      " |-- F_Undergrad: integer (nullable = true)\n",
      " |-- P_Undergrad: integer (nullable = true)\n",
      " |-- Outstate: integer (nullable = true)\n",
      " |-- Room_Board: integer (nullable = true)\n",
      " |-- Books: integer (nullable = true)\n",
      " |-- Personal: integer (nullable = true)\n",
      " |-- PhD: integer (nullable = true)\n",
      " |-- Terminal: integer (nullable = true)\n",
      " |-- S_F_Ratio: double (nullable = true)\n",
      " |-- perc_alumni: integer (nullable = true)\n",
      " |-- Expend: integer (nullable = true)\n",
      " |-- Grad_Rate: integer (nullable = true)\n",
      "\n"
     ]
    }
   ],
   "source": [
    "data.printSchema()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Row(School='Abilene Christian University', Private='Yes', Apps=1660, Accept=1232, Enroll=721, Top10perc=23, Top25perc=52, F_Undergrad=2885, P_Undergrad=537, Outstate=7440, Room_Board=3300, Books=450, Personal=2200, PhD=70, Terminal=78, S_F_Ratio=18.1, perc_alumni=12, Expend=7041, Grad_Rate=60)"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Spark Formatting of Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "# A few things we need to do before Spark can accept the data!\n",
    "# It needs to be in the form of two columns\n",
    "# (\"label\",\"features\")\n",
    "\n",
    "# Import VectorAssembler and Vectors\n",
    "from pyspark.ml.linalg import Vectors\n",
    "from pyspark.ml.feature import VectorAssembler"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['School',\n",
       " 'Private',\n",
       " 'Apps',\n",
       " 'Accept',\n",
       " 'Enroll',\n",
       " 'Top10perc',\n",
       " 'Top25perc',\n",
       " 'F_Undergrad',\n",
       " 'P_Undergrad',\n",
       " 'Outstate',\n",
       " 'Room_Board',\n",
       " 'Books',\n",
       " 'Personal',\n",
       " 'PhD',\n",
       " 'Terminal',\n",
       " 'S_F_Ratio',\n",
       " 'perc_alumni',\n",
       " 'Expend',\n",
       " 'Grad_Rate']"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "assembler = VectorAssembler(\n",
    "  inputCols=['Apps',\n",
    "             'Accept',\n",
    "             'Enroll',\n",
    "             'Top10perc',\n",
    "             'Top25perc',\n",
    "             'F_Undergrad',\n",
    "             'P_Undergrad',\n",
    "             'Outstate',\n",
    "             'Room_Board',\n",
    "             'Books',\n",
    "             'Personal',\n",
    "             'PhD',\n",
    "             'Terminal',\n",
    "             'S_F_Ratio',\n",
    "             'perc_alumni',\n",
    "             'Expend',\n",
    "             'Grad_Rate'],\n",
    "              outputCol=\"features\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "output = assembler.transform(data)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Deal with Private column being \"yes\" or \"no\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "from pyspark.ml.feature import StringIndexer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "indexer = StringIndexer(inputCol=\"Private\", outputCol=\"PrivateIndex\")\n",
    "output_fixed = indexer.fit(output).transform(output)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "final_data = output_fixed.select(\"features\",'PrivateIndex')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_data,test_data = final_data.randomSplit([0.7,0.3])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### The Classifiers"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "from pyspark.ml.classification import DecisionTreeClassifier,GBTClassifier,RandomForestClassifier\n",
    "from pyspark.ml import Pipeline"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Create all three models:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Use mostly defaults to make this comparison \"fair\"\n",
    "\n",
    "dtc = DecisionTreeClassifier(labelCol='PrivateIndex',featuresCol='features')\n",
    "rfc = RandomForestClassifier(labelCol='PrivateIndex',featuresCol='features')\n",
    "gbt = GBTClassifier(labelCol='PrivateIndex',featuresCol='features')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Train all three models:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Train the models (its three models, so it might take some time)\n",
    "dtc_model = dtc.fit(train_data)\n",
    "rfc_model = rfc.fit(train_data)\n",
    "gbt_model = gbt.fit(train_data)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Model Comparison\n",
    "\n",
    "Let's compare each of these models!"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 118,
   "metadata": {},
   "outputs": [],
   "source": [
    "dtc_predictions = dtc_model.transform(test_data)\n",
    "rfc_predictions = rfc_model.transform(test_data)\n",
    "gbt_predictions = gbt_model.transform(test_data)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Evaluation Metrics:**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 119,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from pyspark.ml.evaluation import MulticlassClassificationEvaluator"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 120,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Select (prediction, true label) and compute test error\n",
    "acc_evaluator = MulticlassClassificationEvaluator(labelCol=\"PrivateIndex\", predictionCol=\"prediction\", metricName=\"accuracy\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 121,
   "metadata": {},
   "outputs": [],
   "source": [
    "dtc_acc = acc_evaluator.evaluate(dtc_predictions)\n",
    "rfc_acc = acc_evaluator.evaluate(rfc_predictions)\n",
    "gbt_acc = acc_evaluator.evaluate(gbt_predictions)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 122,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Here are the results!\n",
      "--------------------------------------------------------------------------------\n",
      "A single decision tree had an accuracy of: 91.53%\n",
      "--------------------------------------------------------------------------------\n",
      "A random forest ensemble had an accuracy of: 91.53%\n",
      "--------------------------------------------------------------------------------\n",
      "A ensemble using GBT had an accuracy of: 91.95%\n"
     ]
    }
   ],
   "source": [
    "print(\"Here are the results!\")\n",
    "print('-'*80)\n",
    "print('A single decision tree had an accuracy of: {0:2.2f}%'.format(dtc_acc*100))\n",
    "print('-'*80)\n",
    "print('A random forest ensemble had an accuracy of: {0:2.2f}%'.format(rfc_acc*100))\n",
    "print('-'*80)\n",
    "print('A ensemble using GBT had an accuracy of: {0:2.2f}%'.format(gbt_acc*100))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Interesting! Optional Assignment - play around with the parameters of each of these models, can you squeeze some more accuracy out of them? Or is the data the limiting factor?"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Tree Methods Consulting Project "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "You've been hired by a dog food company to try to predict why some batches of their dog food are spoiling much quicker than intended! Unfortunately this Dog Food company hasn't upgraded to the latest machinery, meaning that the amounts of the five preservative chemicals they are using can vary a lot, but which is the chemical that has the strongest effect? The dog food company first mixes up a batch of preservative that contains 4 different preservative chemicals (A,B,C,D) and then is completed with a \"filler\" chemical. The food scientists beelive one of the A,B,C, or D preservatives is causing the problem, but need your help to figure out which one!\n",
    "Use Machine Learning with RF to find out which parameter had the most predicitive power, thus finding out which chemical causes the early spoiling! So create a model and then find out how you can decide which chemical is the problem!\n",
    "\n",
    "* Pres_A : Percentage of preservative A in the mix\n",
    "* Pres_B : Percentage of preservative B in the mix\n",
    "* Pres_C : Percentage of preservative C in the mix\n",
    "* Pres_D : Percentage of preservative D in the mix\n",
    "* Spoiled: Label indicating whether or not the dog food batch was spoiled.\n",
    "___\n",
    "\n",
    "**Think carefully about what this problem is really asking you to solve. While we will use Machine Learning to solve this, it won't be with your typical train/test split workflow. If this confuses you, skip ahead to the solution code along walk-through!**\n",
    "____"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Good Luck!"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Tree Methods Consulting Project - SOLUTION"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "You've been hired by a dog food company to try to predict why some batches of their dog food are spoiling much quicker than intended! Unfortunately this Dog Food company hasn't upgraded to the latest machinery, meaning that the amounts of the five preservative chemicals they are using can vary a lot, but which is the chemical that has the strongest effect? The dog food company first mixes up a batch of preservative that contains 4 different preservative chemicals (A,B,C,D) and then is completed with a \"filler\" chemical. The food scientists beelive one of the A,B,C, or D preservatives is causing the problem, but need your help to figure out which one!\n",
    "Use Machine Learning with RF to find out which parameter had the most predicitive power, thus finding out which chemical causes the early spoiling! So create a model and then find out how you can decide which chemical is the problem!\n",
    "\n",
    "* Pres_A : Percentage of preservative A in the mix\n",
    "* Pres_B : Percentage of preservative B in the mix\n",
    "* Pres_C : Percentage of preservative C in the mix\n",
    "* Pres_D : Percentage of preservative D in the mix\n",
    "* Spoiled: Label indicating whether or not the dog food batch was spoiled.\n",
    "___\n",
    "\n",
    "**Think carefully about what this problem is really asking you to solve. While we will use Machine Learning to solve this, it won't be with your typical train/test split workflow. If this confuses you, skip ahead to the solution code along walk-through!**\n",
    "____"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#Tree methods Example\n",
    "from pyspark.sql import SparkSession\n",
    "spark = SparkSession.builder.appName('dogfood').getOrCreate()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Load training data\n",
    "data = spark.read.csv('dog_food.csv',inferSchema=True,header=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "root\n",
      " |-- A: integer (nullable = true)\n",
      " |-- B: integer (nullable = true)\n",
      " |-- C: double (nullable = true)\n",
      " |-- D: integer (nullable = true)\n",
      " |-- Spoiled: double (nullable = true)\n",
      "\n"
     ]
    }
   ],
   "source": [
    "data.printSchema()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Row(A=4, B=2, C=12.0, D=3, Spoiled=1.0)"
      ]
     },
     "execution_count": 49,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+-------+------------------+------------------+------------------+------------------+-------------------+\n",
      "|summary|                 A|                 B|                 C|                 D|            Spoiled|\n",
      "+-------+------------------+------------------+------------------+------------------+-------------------+\n",
      "|  count|               490|               490|               490|               490|                490|\n",
      "|   mean|  5.53469387755102| 5.504081632653061| 9.126530612244897| 5.579591836734694| 0.2857142857142857|\n",
      "| stddev|2.9515204234399057|2.8537966089662063|2.0555451971054275|2.8548369309982857|0.45221563164613465|\n",
      "|    min|                 1|                 1|               5.0|                 1|                0.0|\n",
      "|    max|                10|                10|              14.0|                10|                1.0|\n",
      "+-------+------------------+------------------+------------------+------------------+-------------------+\n",
      "\n"
     ]
    }
   ],
   "source": [
    "data.describe().show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Import VectorAssembler and Vectors\n",
    "from pyspark.ml.linalg import Vectors\n",
    "from pyspark.ml.feature import VectorAssembler"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['A', 'B', 'C', 'D', 'Spoiled']"
      ]
     },
     "execution_count": 52,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [],
   "source": [
    "assembler = VectorAssembler(inputCols=['A', 'B', 'C', 'D'],outputCol=\"features\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "output = assembler.transform(data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from pyspark.ml.classification import RandomForestClassifier,DecisionTreeClassifier"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "rfc = DecisionTreeClassifier(labelCol='Spoiled',featuresCol='features')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "root\n",
      " |-- A: integer (nullable = true)\n",
      " |-- B: integer (nullable = true)\n",
      " |-- C: double (nullable = true)\n",
      " |-- D: integer (nullable = true)\n",
      " |-- Spoiled: double (nullable = true)\n",
      " |-- features: vector (nullable = true)\n",
      "\n"
     ]
    }
   ],
   "source": [
    "output.printSchema()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Row(features=DenseVector([4.0, 2.0, 12.0, 3.0]), Spoiled=1.0)"
      ]
     },
     "execution_count": 58,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "final_data = output.select('features','Spoiled')\n",
    "final_data.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [],
   "source": [
    "rfc_model = rfc.fit(final_data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "SparseVector(4, {0: 0.0026, 1: 0.0089, 2: 0.9686, 3: 0.0199})"
      ]
     },
     "execution_count": 60,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "rfc_model.featureImportances"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Bingo! Feature at index 2 (Chemical C) is by far the most important feature, meaning it is causing the early spoilage! This is a pretty interesting use of a machine learning model in an alternative way!\n",
    "\n",
    "# Great Job"
   ]
  }
 ],
 "metadata": {
  "anaconda-cloud": {},
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}