spark2_classification.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 5,
  "metadata": {
    "kernelspec": {
      "display_name": "Python 3",
      "language": "python",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.7.3"
    },
    "colab": {
      "provenance": []
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "tags": [],
        "id": "2yhNxjtG5426"
      },
      "source": [
        "# Classification using pyspark"
      ],
      "id": "2yhNxjtG5426"
    },
    {
      "cell_type": "code",
      "source": [
        "!apt-get install openjdk-8-jdk-headless -qq > /dev/null # install Java8\n",
        "!wget -q https://archive.apache.org/dist/spark/spark-3.1.1/spark-3.1.1-bin-hadoop2.7.tgz # download spark 3.1.1\n",
        "!tar xf spark-3.1.1-bin-hadoop2.7.tgz # unzip it\n",
        "!pip install -q findspark # install findspark\n",
        "!pip install pyspark==3.1.1 # install pyspark\n",
        "!ls\n",
        "\n",
        "import os\n",
        "os.environ[\"JAVA_HOME\"] = \"/usr/lib/jvm/java-8-openjdk-amd64\"\n",
        "os.environ[\"SPARK_HOME\"] = \"/content/spark-3.1.1-bin-hadoop2.7\"\n",
        "\n",
        "import findspark\n",
        "findspark.init()\n",
        "\n",
        "import pyspark.sql.functions as F\n",
        "from pyspark.ml.linalg import Vectors\n",
        "from pyspark.ml import feature\n",
        "from pyspark.sql import SparkSession\n",
        "spark = SparkSession.builder.master(\"local[*]\").getOrCreate()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "HT30_AOBaBPi",
        "outputId": "85d76157-69ad-4de6-8735-000d7f8a3000"
      },
      "id": "HT30_AOBaBPi",
      "execution_count": 1,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Requirement already satisfied: pyspark==3.1.1 in /usr/local/lib/python3.10/dist-packages (3.1.1)\n",
            "Requirement already satisfied: py4j==0.10.9 in /usr/local/lib/python3.10/dist-packages (from pyspark==3.1.1) (0.10.9)\n",
            "clustering_dataset.csv\tiris.data    spark-3.1.1-bin-hadoop2.7\t    spark-3.1.1-bin-hadoop2.7.tgz.1\n",
            "gas_emissions.csv\tsample_data  spark-3.1.1-bin-hadoop2.7.tgz\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "X6rdk0Vr543D",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "bf3b7e79-752e-4619-b3ae-3bbe46f9f027"
      },
      "source": [
        "df = spark.read.csv(\"/content/iris.data\")\n",
        "df.show()"
      ],
      "id": "X6rdk0Vr543D",
      "execution_count": 2,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "+---+---+---+---+-----------+\n",
            "|_c0|_c1|_c2|_c3|        _c4|\n",
            "+---+---+---+---+-----------+\n",
            "|5.1|3.5|1.4|0.2|Iris-setosa|\n",
            "|4.9|3.0|1.4|0.2|Iris-setosa|\n",
            "|4.7|3.2|1.3|0.2|Iris-setosa|\n",
            "|4.6|3.1|1.5|0.2|Iris-setosa|\n",
            "|5.0|3.6|1.4|0.2|Iris-setosa|\n",
            "|5.4|3.9|1.7|0.4|Iris-setosa|\n",
            "|4.6|3.4|1.4|0.3|Iris-setosa|\n",
            "|5.0|3.4|1.5|0.2|Iris-setosa|\n",
            "|4.4|2.9|1.4|0.2|Iris-setosa|\n",
            "|4.9|3.1|1.5|0.1|Iris-setosa|\n",
            "|5.4|3.7|1.5|0.2|Iris-setosa|\n",
            "|4.8|3.4|1.6|0.2|Iris-setosa|\n",
            "|4.8|3.0|1.4|0.1|Iris-setosa|\n",
            "|4.3|3.0|1.1|0.1|Iris-setosa|\n",
            "|5.8|4.0|1.2|0.2|Iris-setosa|\n",
            "|5.7|4.4|1.5|0.4|Iris-setosa|\n",
            "|5.4|3.9|1.3|0.4|Iris-setosa|\n",
            "|5.1|3.5|1.4|0.3|Iris-setosa|\n",
            "|5.7|3.8|1.7|0.3|Iris-setosa|\n",
            "|5.1|3.8|1.5|0.3|Iris-setosa|\n",
            "+---+---+---+---+-----------+\n",
            "only showing top 20 rows\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "X9QJfHAR543E",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "9250ae88-38ff-4016-a162-8526652f508e"
      },
      "source": [
        "df = df.select(F.col('_c0').alias('sepal_length'),\n",
        "               F.col('_c1').alias('sepal_width'),\n",
        "               F.col('_c2').alias('petal_length'),\n",
        "               F.col('_c3').alias('petal_width'),\n",
        "               F.col('_c4').alias('species'),)\n",
        "df.show()"
      ],
      "id": "X9QJfHAR543E",
      "execution_count": 3,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "+------------+-----------+------------+-----------+-----------+\n",
            "|sepal_length|sepal_width|petal_length|petal_width|    species|\n",
            "+------------+-----------+------------+-----------+-----------+\n",
            "|         5.1|        3.5|         1.4|        0.2|Iris-setosa|\n",
            "|         4.9|        3.0|         1.4|        0.2|Iris-setosa|\n",
            "|         4.7|        3.2|         1.3|        0.2|Iris-setosa|\n",
            "|         4.6|        3.1|         1.5|        0.2|Iris-setosa|\n",
            "|         5.0|        3.6|         1.4|        0.2|Iris-setosa|\n",
            "|         5.4|        3.9|         1.7|        0.4|Iris-setosa|\n",
            "|         4.6|        3.4|         1.4|        0.3|Iris-setosa|\n",
            "|         5.0|        3.4|         1.5|        0.2|Iris-setosa|\n",
            "|         4.4|        2.9|         1.4|        0.2|Iris-setosa|\n",
            "|         4.9|        3.1|         1.5|        0.1|Iris-setosa|\n",
            "|         5.4|        3.7|         1.5|        0.2|Iris-setosa|\n",
            "|         4.8|        3.4|         1.6|        0.2|Iris-setosa|\n",
            "|         4.8|        3.0|         1.4|        0.1|Iris-setosa|\n",
            "|         4.3|        3.0|         1.1|        0.1|Iris-setosa|\n",
            "|         5.8|        4.0|         1.2|        0.2|Iris-setosa|\n",
            "|         5.7|        4.4|         1.5|        0.4|Iris-setosa|\n",
            "|         5.4|        3.9|         1.3|        0.4|Iris-setosa|\n",
            "|         5.1|        3.5|         1.4|        0.3|Iris-setosa|\n",
            "|         5.7|        3.8|         1.7|        0.3|Iris-setosa|\n",
            "|         5.1|        3.8|         1.5|        0.3|Iris-setosa|\n",
            "+------------+-----------+------------+-----------+-----------+\n",
            "only showing top 20 rows\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Nw-C5aSs543E",
        "outputId": "90127bc7-32b3-4fec-ae4a-81a8ca976ad7"
      },
      "source": [
        "df.dtypes"
      ],
      "id": "Nw-C5aSs543E",
      "execution_count": 4,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[('sepal_length', 'string'),\n",
              " ('sepal_width', 'string'),\n",
              " ('petal_length', 'string'),\n",
              " ('petal_width', 'string'),\n",
              " ('species', 'string')]"
            ]
          },
          "metadata": {},
          "execution_count": 4
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ClSKlKNJ543F",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "e2a58260-0493-4d32-84f0-6ec2c9d31e0a"
      },
      "source": [
        "df2 = df.selectExpr(\"cast(sepal_length as float)\",\n",
        "                    \"cast(sepal_width as float)\",\n",
        "                    \"cast(petal_length as float)\",\n",
        "                    \"cast(petal_width as float)\",\n",
        "                    \"species\")\n",
        "df2.show()"
      ],
      "id": "ClSKlKNJ543F",
      "execution_count": 5,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "+------------+-----------+------------+-----------+-----------+\n",
            "|sepal_length|sepal_width|petal_length|petal_width|    species|\n",
            "+------------+-----------+------------+-----------+-----------+\n",
            "|         5.1|        3.5|         1.4|        0.2|Iris-setosa|\n",
            "|         4.9|        3.0|         1.4|        0.2|Iris-setosa|\n",
            "|         4.7|        3.2|         1.3|        0.2|Iris-setosa|\n",
            "|         4.6|        3.1|         1.5|        0.2|Iris-setosa|\n",
            "|         5.0|        3.6|         1.4|        0.2|Iris-setosa|\n",
            "|         5.4|        3.9|         1.7|        0.4|Iris-setosa|\n",
            "|         4.6|        3.4|         1.4|        0.3|Iris-setosa|\n",
            "|         5.0|        3.4|         1.5|        0.2|Iris-setosa|\n",
            "|         4.4|        2.9|         1.4|        0.2|Iris-setosa|\n",
            "|         4.9|        3.1|         1.5|        0.1|Iris-setosa|\n",
            "|         5.4|        3.7|         1.5|        0.2|Iris-setosa|\n",
            "|         4.8|        3.4|         1.6|        0.2|Iris-setosa|\n",
            "|         4.8|        3.0|         1.4|        0.1|Iris-setosa|\n",
            "|         4.3|        3.0|         1.1|        0.1|Iris-setosa|\n",
            "|         5.8|        4.0|         1.2|        0.2|Iris-setosa|\n",
            "|         5.7|        4.4|         1.5|        0.4|Iris-setosa|\n",
            "|         5.4|        3.9|         1.3|        0.4|Iris-setosa|\n",
            "|         5.1|        3.5|         1.4|        0.3|Iris-setosa|\n",
            "|         5.7|        3.8|         1.7|        0.3|Iris-setosa|\n",
            "|         5.1|        3.8|         1.5|        0.3|Iris-setosa|\n",
            "+------------+-----------+------------+-----------+-----------+\n",
            "only showing top 20 rows\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "dpYHvjAp543F",
        "outputId": "740ee1c0-752e-46f9-f507-d438ab44f398"
      },
      "source": [
        "df2.dtypes"
      ],
      "id": "dpYHvjAp543F",
      "execution_count": 6,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[('sepal_length', 'float'),\n",
              " ('sepal_width', 'float'),\n",
              " ('petal_length', 'float'),\n",
              " ('petal_width', 'float'),\n",
              " ('species', 'string')]"
            ]
          },
          "metadata": {},
          "execution_count": 6
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "13ZTGYgZ543G",
        "outputId": "323d481d-cc11-4f69-9944-d9faf70742e5"
      },
      "source": [
        "df.columns"
      ],
      "id": "13ZTGYgZ543G",
      "execution_count": 7,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "['sepal_length', 'sepal_width', 'petal_length', 'petal_width', 'species']"
            ]
          },
          "metadata": {},
          "execution_count": 7
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ua6gWlbR543G"
      },
      "source": [
        "vector_assembler = feature.VectorAssembler(inputCols = ['sepal_length', 'sepal_width', 'petal_length', 'petal_width'],\n",
        "                                           outputCol = 'features')\n",
        "df2 = vector_assembler.transform(df2)"
      ],
      "id": "ua6gWlbR543G",
      "execution_count": 8,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "9G-lfs1H543G",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "b315a4ea-52f4-4d62-a056-bdc075f404a5"
      },
      "source": [
        "indexer = feature.StringIndexer(inputCol = 'species', outputCol = 'label')\n",
        "df2 = indexer.fit(df2).transform(df2)\n",
        "df2.show(n = 2, truncate = False)"
      ],
      "id": "9G-lfs1H543G",
      "execution_count": 9,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "+------------+-----------+------------+-----------+-----------+-------------------------------------------------------------+-----+\n",
            "|sepal_length|sepal_width|petal_length|petal_width|species    |features                                                     |label|\n",
            "+------------+-----------+------------+-----------+-----------+-------------------------------------------------------------+-----+\n",
            "|5.1         |3.5        |1.4         |0.2        |Iris-setosa|[5.099999904632568,3.5,1.399999976158142,0.20000000298023224]|0.0  |\n",
            "|4.9         |3.0        |1.4         |0.2        |Iris-setosa|[4.900000095367432,3.0,1.399999976158142,0.20000000298023224]|0.0  |\n",
            "+------------+-----------+------------+-----------+-----------+-------------------------------------------------------------+-----+\n",
            "only showing top 2 rows\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "-R8k35Fh543H"
      },
      "source": [
        "### Dataset split"
      ],
      "id": "-R8k35Fh543H"
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "SM92a7wG543H"
      },
      "source": [
        "splits = df2.randomSplit([0.7, 0.3], seed = 101)\n",
        "df_tr = splits[0]\n",
        "df_tst = splits[1]"
      ],
      "id": "SM92a7wG543H",
      "execution_count": 10,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "vSo8e08o543I",
        "outputId": "6d734cae-6bd2-489d-ffbf-c7d2f59a7cc2"
      },
      "source": [
        "df_tr.show()"
      ],
      "id": "vSo8e08o543I",
      "execution_count": 11,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "+------------+-----------+------------+-----------+---------------+--------------------+-----+\n",
            "|sepal_length|sepal_width|petal_length|petal_width|        species|            features|label|\n",
            "+------------+-----------+------------+-----------+---------------+--------------------+-----+\n",
            "|         4.3|        3.0|         1.1|        0.1|    Iris-setosa|[4.30000019073486...|  0.0|\n",
            "|         4.4|        2.9|         1.4|        0.2|    Iris-setosa|[4.40000009536743...|  0.0|\n",
            "|         4.4|        3.0|         1.3|        0.2|    Iris-setosa|[4.40000009536743...|  0.0|\n",
            "|         4.4|        3.2|         1.3|        0.2|    Iris-setosa|[4.40000009536743...|  0.0|\n",
            "|         4.6|        3.2|         1.4|        0.2|    Iris-setosa|[4.59999990463256...|  0.0|\n",
            "|         4.6|        3.4|         1.4|        0.3|    Iris-setosa|[4.59999990463256...|  0.0|\n",
            "|         4.6|        3.6|         1.0|        0.2|    Iris-setosa|[4.59999990463256...|  0.0|\n",
            "|         4.7|        3.2|         1.3|        0.2|    Iris-setosa|[4.69999980926513...|  0.0|\n",
            "|         4.7|        3.2|         1.6|        0.2|    Iris-setosa|[4.69999980926513...|  0.0|\n",
            "|         4.8|        3.0|         1.4|        0.1|    Iris-setosa|[4.80000019073486...|  0.0|\n",
            "|         4.8|        3.0|         1.4|        0.3|    Iris-setosa|[4.80000019073486...|  0.0|\n",
            "|         4.8|        3.1|         1.6|        0.2|    Iris-setosa|[4.80000019073486...|  0.0|\n",
            "|         4.9|        2.4|         3.3|        1.0|Iris-versicolor|[4.90000009536743...|  1.0|\n",
            "|         4.9|        2.5|         4.5|        1.7| Iris-virginica|[4.90000009536743...|  2.0|\n",
            "|         4.9|        3.1|         1.5|        0.1|    Iris-setosa|[4.90000009536743...|  0.0|\n",
            "|         4.9|        3.1|         1.5|        0.1|    Iris-setosa|[4.90000009536743...|  0.0|\n",
            "|         5.0|        3.0|         1.6|        0.2|    Iris-setosa|[5.0,3.0,1.600000...|  0.0|\n",
            "|         5.0|        3.2|         1.2|        0.2|    Iris-setosa|[5.0,3.2000000476...|  0.0|\n",
            "|         5.0|        3.3|         1.4|        0.2|    Iris-setosa|[5.0,3.2999999523...|  0.0|\n",
            "|         5.0|        3.4|         1.5|        0.2|    Iris-setosa|[5.0,3.4000000953...|  0.0|\n",
            "+------------+-----------+------------+-----------+---------------+--------------------+-----+\n",
            "only showing top 20 rows\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "df_tr.count()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "mMvGHG-6d2kV",
        "outputId": "e46696a8-5d23-442f-a131-47d15c38b7ea"
      },
      "id": "mMvGHG-6d2kV",
      "execution_count": 12,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "102"
            ]
          },
          "metadata": {},
          "execution_count": 12
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "df_tst.show()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "mX0WlPYwdxxk",
        "outputId": "3e594ddb-bc7a-484e-cad2-65a4331b3f52"
      },
      "id": "mX0WlPYwdxxk",
      "execution_count": 13,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "+------------+-----------+------------+-----------+---------------+--------------------+-----+\n",
            "|sepal_length|sepal_width|petal_length|petal_width|        species|            features|label|\n",
            "+------------+-----------+------------+-----------+---------------+--------------------+-----+\n",
            "|         4.5|        2.3|         1.3|        0.3|    Iris-setosa|[4.5,2.2999999523...|  0.0|\n",
            "|         4.6|        3.1|         1.5|        0.2|    Iris-setosa|[4.59999990463256...|  0.0|\n",
            "|         4.8|        3.4|         1.6|        0.2|    Iris-setosa|[4.80000019073486...|  0.0|\n",
            "|         4.8|        3.4|         1.9|        0.2|    Iris-setosa|[4.80000019073486...|  0.0|\n",
            "|         4.9|        3.0|         1.4|        0.2|    Iris-setosa|[4.90000009536743...|  0.0|\n",
            "|         4.9|        3.1|         1.5|        0.1|    Iris-setosa|[4.90000009536743...|  0.0|\n",
            "|         5.0|        2.0|         3.5|        1.0|Iris-versicolor|   [5.0,2.0,3.5,1.0]|  1.0|\n",
            "|         5.0|        2.3|         3.3|        1.0|Iris-versicolor|[5.0,2.2999999523...|  1.0|\n",
            "|         5.1|        3.3|         1.7|        0.5|    Iris-setosa|[5.09999990463256...|  0.0|\n",
            "|         5.1|        3.5|         1.4|        0.2|    Iris-setosa|[5.09999990463256...|  0.0|\n",
            "|         5.1|        3.5|         1.4|        0.3|    Iris-setosa|[5.09999990463256...|  0.0|\n",
            "|         5.2|        3.4|         1.4|        0.2|    Iris-setosa|[5.19999980926513...|  0.0|\n",
            "|         5.4|        3.4|         1.5|        0.4|    Iris-setosa|[5.40000009536743...|  0.0|\n",
            "|         5.4|        3.9|         1.3|        0.4|    Iris-setosa|[5.40000009536743...|  0.0|\n",
            "|         5.4|        3.9|         1.7|        0.4|    Iris-setosa|[5.40000009536743...|  0.0|\n",
            "|         5.5|        2.3|         4.0|        1.3|Iris-versicolor|[5.5,2.2999999523...|  1.0|\n",
            "|         5.5|        2.6|         4.4|        1.2|Iris-versicolor|[5.5,2.5999999046...|  1.0|\n",
            "|         5.5|        4.2|         1.4|        0.2|    Iris-setosa|[5.5,4.1999998092...|  0.0|\n",
            "|         5.6|        2.8|         4.9|        2.0| Iris-virginica|[5.59999990463256...|  2.0|\n",
            "|         5.6|        3.0|         4.1|        1.3|Iris-versicolor|[5.59999990463256...|  1.0|\n",
            "+------------+-----------+------------+-----------+---------------+--------------------+-----+\n",
            "only showing top 20 rows\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "ntP5vbsO543J",
        "outputId": "3814eaa7-8da7-48a5-86c8-12decf1c0110"
      },
      "source": [
        "df_tst.count()"
      ],
      "id": "ntP5vbsO543J",
      "execution_count": 14,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "48"
            ]
          },
          "metadata": {},
          "execution_count": 14
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "PhjUf4Kv543J"
      },
      "source": [
        "### Models"
      ],
      "id": "PhjUf4Kv543J"
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "xSSOqhjx543K"
      },
      "source": [
        "from pyspark.ml.classification import NaiveBayes\n",
        "from pyspark.ml.classification import DecisionTreeClassifier, RandomForestClassifier\n",
        "from pyspark.ml.evaluation import MulticlassClassificationEvaluator"
      ],
      "id": "xSSOqhjx543K",
      "execution_count": 15,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "models = ['naive bayes', 'decision tree']\n",
        "accuracy = []\n",
        "f1 = []"
      ],
      "metadata": {
        "id": "l3fJytmCd-z2"
      },
      "id": "l3fJytmCd-z2",
      "execution_count": 16,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Na1Nt0d0543K"
      },
      "source": [
        "#### Naive Bayes"
      ],
      "id": "Na1Nt0d0543K"
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "lFB25CZP543K"
      },
      "source": [
        "nb = NaiveBayes(modelType = 'multinomial')\n",
        "nb_model = nb.fit(df_tr)\n",
        "pred = nb_model.transform(df_tst)"
      ],
      "id": "lFB25CZP543K",
      "execution_count": 17,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "jByL563E543L",
        "outputId": "25bcdcbd-3c88-48f8-821f-76fd39702e95"
      },
      "source": [
        "pred.columns"
      ],
      "id": "jByL563E543L",
      "execution_count": 18,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "['sepal_length',\n",
              " 'sepal_width',\n",
              " 'petal_length',\n",
              " 'petal_width',\n",
              " 'species',\n",
              " 'features',\n",
              " 'label',\n",
              " 'rawPrediction',\n",
              " 'probability',\n",
              " 'prediction']"
            ]
          },
          "metadata": {},
          "execution_count": 18
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "h6neBUg8543M",
        "outputId": "e1171c85-e115-45cd-a423-94982fb03347"
      },
      "source": [
        "pred.select('label', 'probability', 'prediction').show(truncate = False)"
      ],
      "id": "h6neBUg8543M",
      "execution_count": 19,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "+-----+--------------------------------------------------------------+----------+\n",
            "|label|probability                                                   |prediction|\n",
            "+-----+--------------------------------------------------------------+----------+\n",
            "|0.0  |[0.5750219584755774,0.25057822528261237,0.1743998162418101]   |0.0       |\n",
            "|0.0  |[0.6864575093386172,0.19265338304282892,0.12088910761855388]  |0.0       |\n",
            "|0.0  |[0.7161675808925836,0.1771046575420379,0.10672776156537848]   |0.0       |\n",
            "|0.0  |[0.6682399101371,0.20500435727907615,0.1267557325838238]      |0.0       |\n",
            "|0.0  |[0.7030077977232515,0.18390043525156033,0.11309176702518807]  |0.0       |\n",
            "|0.0  |[0.7280960194734826,0.17001235008514878,0.10189163044136856]  |0.0       |\n",
            "|1.0  |[0.08110989922547217,0.46331366566630805,0.45557643510821977] |1.0       |\n",
            "|1.0  |[0.1095242305646629,0.4603095643688824,0.4301662050664548]    |1.0       |\n",
            "|0.0  |[0.6067242108824067,0.23923972426575532,0.15403606485183807]  |0.0       |\n",
            "|0.0  |[0.7677104234726349,0.14771792837428024,0.0845716481530848]   |0.0       |\n",
            "|0.0  |[0.7419069868843214,0.16274367613404933,0.09534933698162933]  |0.0       |\n",
            "|0.0  |[0.7608491238495994,0.15188000933538343,0.08727086681501718]  |0.0       |\n",
            "|0.0  |[0.699310039646653,0.18819721979682025,0.11249274055652678]   |0.0       |\n",
            "|0.0  |[0.7835460429662846,0.1391593370431939,0.07729461999052147]   |0.0       |\n",
            "|0.0  |[0.7284307105851603,0.17244738076798463,0.09912190864685505]  |0.0       |\n",
            "|1.0  |[0.04979785578338851,0.4695579135883085,0.4806442306283031]   |2.0       |\n",
            "|1.0  |[0.050663772640615355,0.47448943316229025,0.47484679419709447]|2.0       |\n",
            "|0.0  |[0.8435612385887371,0.10337842632235118,0.05306033508891185]  |0.0       |\n",
            "|2.0  |[0.013184170420881814,0.44470662335970135,0.5421092062194169] |2.0       |\n",
            "|1.0  |[0.07019722926007424,0.47756593369433953,0.4522368370455862]  |1.0       |\n",
            "+-----+--------------------------------------------------------------+----------+\n",
            "only showing top 20 rows\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "uc5OwrJ2543M"
      },
      "source": [
        "evaluator = MulticlassClassificationEvaluator(metricName = 'accuracy')\n",
        "accuracy.append(evaluator.evaluate(pred))"
      ],
      "id": "uc5OwrJ2543M",
      "execution_count": 20,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "_25nyxod543O"
      },
      "source": [
        "evaluator = MulticlassClassificationEvaluator(metricName = 'f1')\n",
        "f1.append(evaluator.evaluate(pred))"
      ],
      "id": "_25nyxod543O",
      "execution_count": 21,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "CwSZV2Ff543O"
      },
      "source": [
        "#### Decision Tree"
      ],
      "id": "CwSZV2Ff543O"
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "1aoJjxmg543O"
      },
      "source": [
        "dt = DecisionTreeClassifier()\n",
        "dt_model = dt.fit(df_tr)\n",
        "pred = dt_model.transform(df_tst)"
      ],
      "id": "1aoJjxmg543O",
      "execution_count": 22,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "xeiVI7kX_995",
        "outputId": "9cb456a7-7907-49c3-966b-016a6455185a"
      },
      "source": [
        "pred.show()"
      ],
      "id": "xeiVI7kX_995",
      "execution_count": 23,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "+------------+-----------+------------+-----------+---------------+--------------------+-----+--------------+-------------+----------+\n",
            "|sepal_length|sepal_width|petal_length|petal_width|        species|            features|label| rawPrediction|  probability|prediction|\n",
            "+------------+-----------+------------+-----------+---------------+--------------------+-----+--------------+-------------+----------+\n",
            "|         4.5|        2.3|         1.3|        0.3|    Iris-setosa|[4.5,2.2999999523...|  0.0|[36.0,0.0,0.0]|[1.0,0.0,0.0]|       0.0|\n",
            "|         4.6|        3.1|         1.5|        0.2|    Iris-setosa|[4.59999990463256...|  0.0|[36.0,0.0,0.0]|[1.0,0.0,0.0]|       0.0|\n",
            "|         4.8|        3.4|         1.6|        0.2|    Iris-setosa|[4.80000019073486...|  0.0|[36.0,0.0,0.0]|[1.0,0.0,0.0]|       0.0|\n",
            "|         4.8|        3.4|         1.9|        0.2|    Iris-setosa|[4.80000019073486...|  0.0|[36.0,0.0,0.0]|[1.0,0.0,0.0]|       0.0|\n",
            "|         4.9|        3.0|         1.4|        0.2|    Iris-setosa|[4.90000009536743...|  0.0|[36.0,0.0,0.0]|[1.0,0.0,0.0]|       0.0|\n",
            "|         4.9|        3.1|         1.5|        0.1|    Iris-setosa|[4.90000009536743...|  0.0|[36.0,0.0,0.0]|[1.0,0.0,0.0]|       0.0|\n",
            "|         5.0|        2.0|         3.5|        1.0|Iris-versicolor|   [5.0,2.0,3.5,1.0]|  1.0|[0.0,29.0,0.0]|[0.0,1.0,0.0]|       1.0|\n",
            "|         5.0|        2.3|         3.3|        1.0|Iris-versicolor|[5.0,2.2999999523...|  1.0|[0.0,29.0,0.0]|[0.0,1.0,0.0]|       1.0|\n",
            "|         5.1|        3.3|         1.7|        0.5|    Iris-setosa|[5.09999990463256...|  0.0|[36.0,0.0,0.0]|[1.0,0.0,0.0]|       0.0|\n",
            "|         5.1|        3.5|         1.4|        0.2|    Iris-setosa|[5.09999990463256...|  0.0|[36.0,0.0,0.0]|[1.0,0.0,0.0]|       0.0|\n",
            "|         5.1|        3.5|         1.4|        0.3|    Iris-setosa|[5.09999990463256...|  0.0|[36.0,0.0,0.0]|[1.0,0.0,0.0]|       0.0|\n",
            "|         5.2|        3.4|         1.4|        0.2|    Iris-setosa|[5.19999980926513...|  0.0|[36.0,0.0,0.0]|[1.0,0.0,0.0]|       0.0|\n",
            "|         5.4|        3.4|         1.5|        0.4|    Iris-setosa|[5.40000009536743...|  0.0|[36.0,0.0,0.0]|[1.0,0.0,0.0]|       0.0|\n",
            "|         5.4|        3.9|         1.3|        0.4|    Iris-setosa|[5.40000009536743...|  0.0|[36.0,0.0,0.0]|[1.0,0.0,0.0]|       0.0|\n",
            "|         5.4|        3.9|         1.7|        0.4|    Iris-setosa|[5.40000009536743...|  0.0|[36.0,0.0,0.0]|[1.0,0.0,0.0]|       0.0|\n",
            "|         5.5|        2.3|         4.0|        1.3|Iris-versicolor|[5.5,2.2999999523...|  1.0|[0.0,29.0,0.0]|[0.0,1.0,0.0]|       1.0|\n",
            "|         5.5|        2.6|         4.4|        1.2|Iris-versicolor|[5.5,2.5999999046...|  1.0|[0.0,29.0,0.0]|[0.0,1.0,0.0]|       1.0|\n",
            "|         5.5|        4.2|         1.4|        0.2|    Iris-setosa|[5.5,4.1999998092...|  0.0|[36.0,0.0,0.0]|[1.0,0.0,0.0]|       0.0|\n",
            "|         5.6|        2.8|         4.9|        2.0| Iris-virginica|[5.59999990463256...|  2.0|[0.0,0.0,30.0]|[0.0,0.0,1.0]|       2.0|\n",
            "|         5.6|        3.0|         4.1|        1.3|Iris-versicolor|[5.59999990463256...|  1.0|[0.0,29.0,0.0]|[0.0,1.0,0.0]|       1.0|\n",
            "+------------+-----------+------------+-----------+---------------+--------------------+-----+--------------+-------------+----------+\n",
            "only showing top 20 rows\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "_fIX5KMi543P"
      },
      "source": [
        "evaluator = MulticlassClassificationEvaluator(metricName = 'accuracy')\n",
        "accuracy.append(evaluator.evaluate(pred))"
      ],
      "id": "_fIX5KMi543P",
      "execution_count": 24,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "0vO6wqGr543Q"
      },
      "source": [
        "evaluator = MulticlassClassificationEvaluator(metricName = 'f1')\n",
        "f1.append(evaluator.evaluate(pred))"
      ],
      "id": "0vO6wqGr543Q",
      "execution_count": 25,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "-sDP17VQ543Q"
      },
      "source": [
        "### Model selection"
      ],
      "id": "-sDP17VQ543Q"
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "ASM5MxMZ543Q",
        "outputId": "52f0c19a-afcc-490d-c13b-6b5d7d633969"
      },
      "source": [
        "models"
      ],
      "id": "ASM5MxMZ543Q",
      "execution_count": 26,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "['naive bayes', 'decision tree']"
            ]
          },
          "metadata": {},
          "execution_count": 26
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "qf4k7d0P543R",
        "outputId": "1c9c032c-4dc3-4807-f154-481e195c23e5"
      },
      "source": [
        "accuracy"
      ],
      "id": "qf4k7d0P543R",
      "execution_count": 27,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[0.9375, 0.9791666666666666]"
            ]
          },
          "metadata": {},
          "execution_count": 27
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "EbxkHSo1543R",
        "outputId": "ca123733-f560-4940-ec34-e28366cc35b5"
      },
      "source": [
        "f1"
      ],
      "id": "EbxkHSo1543R",
      "execution_count": 28,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[0.9373376623376624, 0.9791121766928219]"
            ]
          },
          "metadata": {},
          "execution_count": 28
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "pYBHRxNA543R"
      },
      "source": [
        "df_results = spark.createDataFrame(data = zip(models, accuracy, f1),\n",
        "                                   schema = ['models', 'accuracy', 'f1'])"
      ],
      "id": "pYBHRxNA543R",
      "execution_count": 29,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "ZQhmL_u_543R",
        "outputId": "2d04c8f8-4351-4724-b30e-82d6d108ffce"
      },
      "source": [
        "df_results.dtypes"
      ],
      "id": "ZQhmL_u_543R",
      "execution_count": 30,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[('models', 'string'), ('accuracy', 'double'), ('f1', 'double')]"
            ]
          },
          "metadata": {},
          "execution_count": 30
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "P3OXdoAG543S",
        "outputId": "262dd56d-b0ca-4c6c-8798-625ff813def7"
      },
      "source": [
        "df_results.printSchema()"
      ],
      "id": "P3OXdoAG543S",
      "execution_count": 31,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "root\n",
            " |-- models: string (nullable = true)\n",
            " |-- accuracy: double (nullable = true)\n",
            " |-- f1: double (nullable = true)\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "KLv8Nrwu543S",
        "outputId": "fb593e13-ff76-41b7-e801-bb8a546a8e66"
      },
      "source": [
        "df_results.show()"
      ],
      "id": "KLv8Nrwu543S",
      "execution_count": 32,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "+-------------+------------------+------------------+\n",
            "|       models|          accuracy|                f1|\n",
            "+-------------+------------------+------------------+\n",
            "|  naive bayes|            0.9375|0.9373376623376624|\n",
            "|decision tree|0.9791666666666666|0.9791121766928219|\n",
            "+-------------+------------------+------------------+\n",
            "\n"
          ]
        }
      ]
    }
  ]
}