Farsi_News_Topic_Detection.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "Farsi News Topic Detection.ipynb",
      "provenance": [],
      "collapsed_sections": [
        "gHtYSDIkhIj4",
        "1oIrm8AaiR86",
        "TmeYzsDKhQEz",
        "Td0Dheaby0Z_",
        "1a5onl1EPRBf",
        "nDlwHVvljw0_",
        "cBoDjUyqkNs_",
        "3p1C5WG5vnQy",
        "JA5nNOrJ_ACy",
        "tFW7NkwT_EOC",
        "WrYnKyjCjYSu",
        "Ox03scXsxJSJ",
        "fuoUULNalFgo",
        "5ES7DnYgqh47"
      ],
      "authorship_tag": "ABX9TyOVKzLIyE3LYfUIHfrNp3yu",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/arman-aminian/farsi_news_topic_detection/blob/master/Farsi_News_Topic_Detection.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "gHtYSDIkhIj4"
      },
      "source": [
        "# Download dataset"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "991aGgchazhZ",
        "outputId": "91bf4e33-d75d-4c7b-9a32-ef138db46688"
      },
      "source": [
        "from google.colab import drive\n",
        "drive.mount('/content/gdrive')"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Mounted at /content/gdrive\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "yzYjz5ZvfDOj"
      },
      "source": [
        "import os\n",
        "os.environ['KAGGLE_CONFIG_DIR'] = \"/content/gdrive/My Drive/Kaggle\"\n",
        "# /content/gdrive/My Drive/Kaggle is the path where kaggle.json is present in the Google Drive"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "0vNNAkYIfeMH",
        "outputId": "69f25fd8-798b-4710-b59d-16c47fae19c5"
      },
      "source": [
        "%cd /content/gdrive/My Drive/Kaggle"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "/content/gdrive/My Drive/Kaggle\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "fr21UBaBfgZw",
        "outputId": "53e8c30c-8c5d-4ef9-9ad2-d1441d9b44ca"
      },
      "source": [
        "!kaggle competitions download -c computational-intelligence-course-final-project"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Warning: Looks like you're using an outdated API Version, please consider updating (server 1.5.10 / client 1.5.4)\n",
            "Downloading test.csv.zip to /content/gdrive/My Drive/Kaggle\n",
            " 30% 5.00M/16.8M [00:00<00:01, 10.7MB/s]\n",
            "100% 16.8M/16.8M [00:00<00:00, 31.1MB/s]\n",
            "Downloading sampleSubmission.csv to /content/gdrive/My Drive/Kaggle\n",
            "  0% 0.00/327k [00:00<?, ?B/s]\n",
            "100% 327k/327k [00:00<00:00, 45.2MB/s]\n",
            "Downloading train.csv.zip to /content/gdrive/My Drive/Kaggle\n",
            " 92% 138M/151M [00:02<00:00, 35.4MB/s]\n",
            "100% 151M/151M [00:03<00:00, 52.2MB/s]\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "XSCLdT88glFu",
        "outputId": "d9b5dfad-0af4-4bcb-c7e8-b0d81e20e276"
      },
      "source": [
        "!unzip \\*.zip  && rm *.zip"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Archive:  test.csv.zip\n",
            "  inflating: test.csv                \n",
            "\n",
            "Archive:  train.csv.zip\n",
            "  inflating: train.csv               \n",
            "\n",
            "2 archives were successfully processed.\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Gd_bvO6kg7Z9",
        "outputId": "4b97a9be-1166-48e5-81f6-219d204b19ed"
      },
      "source": [
        "cd ./gdrive/MyDrive/Kaggle"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "[Errno 2] No such file or directory: './gdrive/MyDrive/Kaggle'\n",
            "/\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "iRyzeHoPnxD-",
        "outputId": "01f6dd47-2b41-465a-b302-a8ac4aa00594"
      },
      "source": [
        "ls"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "category.csv        result_12.csv  result_18.csv  sampleSubmission.csv\n",
            "cleaned_data.txt    result_13.csv  result_19.csv  test_cleaned_data.txt\n",
            "kaggle.json         result_14.csv  result_20.csv  test.csv\n",
            "\u001b[0m\u001b[01;34mpersian-stopwords\u001b[0m/  result_15.csv  result_21.csv  train.csv\n",
            "result_10.csv       result_16.csv  result_22.csv  train_data.npy\n",
            "result_11.csv       result_17.csv  result_9.csv\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "1oIrm8AaiR86"
      },
      "source": [
        "# Download packages"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "HoNVkhT9i7dt",
        "outputId": "ac278718-5d46-4756-a20a-c7aadd75d955"
      },
      "source": [
        "!git clone https://github.com/kharazi/persian-stopwords.git"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Cloning into 'persian-stopwords'...\n",
            "remote: Enumerating objects: 3, done.\u001b[K\n",
            "remote: Counting objects: 100% (3/3), done.\u001b[K\n",
            "remote: Compressing objects: 100% (3/3), done.\u001b[K\n",
            "remote: Total 43 (delta 0), reused 0 (delta 0), pack-reused 40\u001b[K\n",
            "Unpacking objects: 100% (43/43), done.\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "frhCQJX9iVsM",
        "outputId": "6d55a9e3-46cd-4aa0-b5bc-93c85eae3f65"
      },
      "source": [
        "pip install hazm"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Collecting hazm\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/22/13/5a7074bc11d20dbbb46239349ac3f85f7edc148b4cf68e9b8c2f8263830c/hazm-0.7.0-py3-none-any.whl (316kB)\n",
            "\u001b[K     |████████████████████████████████| 317kB 5.9MB/s \n",
            "\u001b[?25hCollecting libwapiti>=0.2.1; platform_system != \"Windows\"\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/bc/0f/1c9b49bb49821b5856a64ea6fac8d96a619b9f291d1f06999ea98a32c89c/libwapiti-0.2.1.tar.gz (233kB)\n",
            "\u001b[K     |████████████████████████████████| 235kB 32.0MB/s \n",
            "\u001b[?25hCollecting nltk==3.3\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/50/09/3b1755d528ad9156ee7243d52aa5cd2b809ef053a0f31b53d92853dd653a/nltk-3.3.0.zip (1.4MB)\n",
            "\u001b[K     |████████████████████████████████| 1.4MB 28.5MB/s \n",
            "\u001b[?25hRequirement already satisfied: six in /usr/local/lib/python3.6/dist-packages (from libwapiti>=0.2.1; platform_system != \"Windows\"->hazm) (1.15.0)\n",
            "Building wheels for collected packages: libwapiti, nltk\n",
            "  Building wheel for libwapiti (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for libwapiti: filename=libwapiti-0.2.1-cp36-cp36m-linux_x86_64.whl size=154544 sha256=3ec0b42e0292abea25f18006f7409affa8ee2f1cef900ec490e16e57870432d4\n",
            "  Stored in directory: /root/.cache/pip/wheels/66/15/54/4510dce8bb958b1cdd2c47425cbd1e1eecc0480ac9bb1fb9ab\n",
            "  Building wheel for nltk (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for nltk: filename=nltk-3.3-cp36-none-any.whl size=1394472 sha256=572ea4ca6c08b75454b202b454792591f34dce960b57556a9eb360db74d59886\n",
            "  Stored in directory: /root/.cache/pip/wheels/d1/ab/40/3bceea46922767e42986aef7606a600538ca80de6062dc266c\n",
            "Successfully built libwapiti nltk\n",
            "Installing collected packages: libwapiti, nltk, hazm\n",
            "  Found existing installation: nltk 3.2.5\n",
            "    Uninstalling nltk-3.2.5:\n",
            "      Successfully uninstalled nltk-3.2.5\n",
            "Successfully installed hazm-0.7.0 libwapiti-0.2.1 nltk-3.3\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "YXyqUgWeig6C",
        "outputId": "55a3f318-bdcd-4a3d-e174-f1978e6d2f91"
      },
      "source": [
        "pip install https://github.com/sobhe/hazm/archive/master.zip --upgrade"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Collecting https://github.com/sobhe/hazm/archive/master.zip\n",
            "\u001b[?25l  Downloading https://github.com/sobhe/hazm/archive/master.zip\n",
            "\u001b[K     \\ 1.4MB 8.0MB/s\n",
            "\u001b[?25hCollecting nltk==3.4\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/6f/ed/9c755d357d33bc1931e157f537721efb5b88d2c583fe593cc09603076cc3/nltk-3.4.zip (1.4MB)\n",
            "\u001b[K     |████████████████████████████████| 1.4MB 5.9MB/s \n",
            "\u001b[?25hRequirement already satisfied, skipping upgrade: libwapiti>=0.2.1 in /usr/local/lib/python3.6/dist-packages (from hazm==0.7.1) (0.2.1)\n",
            "Requirement already satisfied, skipping upgrade: six in /usr/local/lib/python3.6/dist-packages (from nltk==3.4->hazm==0.7.1) (1.15.0)\n",
            "Collecting singledispatch\n",
            "  Downloading https://files.pythonhosted.org/packages/c5/10/369f50bcd4621b263927b0a1519987a04383d4a98fb10438042ad410cf88/singledispatch-3.4.0.3-py2.py3-none-any.whl\n",
            "Building wheels for collected packages: hazm, nltk\n",
            "  Building wheel for hazm (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for hazm: filename=hazm-0.7.1-cp36-none-any.whl size=318794 sha256=ce2259f10bbac73f4d7d8d0ea2568a097e38b257e65a6e251378b8a087c5cca0\n",
            "  Stored in directory: /tmp/pip-ephem-wheel-cache-fyqe32x0/wheels/32/97/cc/640e3b2f3e7b53c5ae336824a31184e86072b8ded31dc0151e\n",
            "  Building wheel for nltk (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for nltk: filename=nltk-3.4-cp36-none-any.whl size=1436381 sha256=59e807e69bc0b917ab3e964d05e6df3d9685e0720738c037f72330a272651255\n",
            "  Stored in directory: /root/.cache/pip/wheels/4b/c8/24/b2343664bcceb7147efeb21c0b23703a05b23fcfeaceaa2a1e\n",
            "Successfully built hazm nltk\n",
            "Installing collected packages: singledispatch, nltk, hazm\n",
            "  Found existing installation: nltk 3.3\n",
            "    Uninstalling nltk-3.3:\n",
            "      Successfully uninstalled nltk-3.3\n",
            "  Found existing installation: hazm 0.7.0\n",
            "    Uninstalling hazm-0.7.0:\n",
            "      Successfully uninstalled hazm-0.7.0\n",
            "Successfully installed hazm-0.7.1 nltk-3.4 singledispatch-3.4.0.3\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "TmeYzsDKhQEz"
      },
      "source": [
        "# Import packages"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "X_AAo9-jhFFw"
      },
      "source": [
        "import numpy as np\n",
        "import pandas as pd\n",
        "import seaborn as sns\n",
        "from keras.preprocessing.text import Tokenizer\n",
        "from keras.preprocessing.sequence import pad_sequences\n",
        "from keras.models import Sequential\n",
        "from keras.layers import Dense, Embedding, LSTM, SpatialDropout1D, Bidirectional, GlobalMaxPooling1D, Conv1D, MaxPooling1D\n",
        "from sklearn.model_selection import train_test_split\n",
        "from keras.utils.np_utils import to_categorical\n",
        "from keras.callbacks import EarlyStopping\n",
        "from keras.layers import Dropout\n",
        "from keras import utils\n",
        "import pickle\n",
        "from numpy import asarray\n",
        "from sklearn.preprocessing import OrdinalEncoder"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Td0Dheaby0Z_"
      },
      "source": [
        "# Preprocess"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "NpvCDhjNy3ks"
      },
      "source": [
        "cd persian-stopwords"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "USLdbu7VzMmQ"
      },
      "source": [
        "my_file = open(\"persian\", \"r\")\n",
        "content = my_file.read()\n",
        "stopwords = content.split(\"\\n\")\n",
        "stopwords = stopwords[:-1]\n",
        "my_file.close()\n",
        "my_file = open(\"short\", \"r\")\n",
        "content = my_file.read()\n",
        "stopwords2 = content.split(\"\\n\")\n",
        "stopwords2 = stopwords2[:-1]\n",
        "my_file.close()\n",
        "my_file = open(\"verbal\", \"r\")\n",
        "content = my_file.read()\n",
        "stopwords3 = content.split(\"\\n\")\n",
        "stopwords3 = stopwords3[:-1]\n",
        "my_file.close()\n",
        "my_file = open(\"nonverbal\", \"r\")\n",
        "content = my_file.read()\n",
        "stopwords4 = content.split(\"\\n\")\n",
        "stopwords4 = stopwords4[:-1]\n",
        "my_file.close()"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "buXmxTpCzTR5"
      },
      "source": [
        "cd .."
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "GjMAX_7uzVnc"
      },
      "source": [
        "my_file = open(\"stopwords.txt\", \"r\")\n",
        "content = my_file.read()\n",
        "stopwords5 = content.split(\"\\n\")\n",
        "stopwords5 = stopwords5[:-1]\n",
        "my_file.close()\n",
        "my_file = open(\"stopwords2.txt\", \"r\")\n",
        "content = my_file.read()\n",
        "stopwords6 = content.split(\"\\n\")\n",
        "stopwords6 = stopwords6[:-1]\n",
        "my_file.close()"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "LGAs39UNzWhb"
      },
      "source": [
        "def hasNumbers(inputString):\n",
        "    return any(char.isdigit() for char in inputString)\n",
        "\n",
        "def isEnglish(s):\n",
        "    return s.lower().islower()"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Xs6XPYIyzYzO"
      },
      "source": [
        "def preProcess(set):\n",
        "    cleanedData = []\n",
        "    tokenizer = WordTokenizer(join_verb_parts=False, replace_links=True, replace_numbers=True)\n",
        "    normalizer = Normalizer()\n",
        "    stemmer = Stemmer()\n",
        "    lemmatizer = Lemmatizer()\n",
        "    i = 0\n",
        "    for text in set[\"Text\"]:\n",
        "        i = i+1\n",
        "        if i % 20000 == 0:\n",
        "            print(i)\n",
        "    #     text = re.sub(r'http\\S+', '', text)\n",
        "\n",
        "        words = tokenizer.tokenize(text)\n",
        "        words = [word for word in words if (word.startswith(\"NUM\") != True and word.startswith(\"LINK\") != True)]\n",
        "\n",
        "        #replace ي with ی and remove arabic symbols\n",
        "        words = [normalizer.character_refinement(word) for word in words]\n",
        "        \n",
        "        words = [word for word in words if word not in stopwords]\n",
        "        words = [word for word in words if word not in stopwords2]\n",
        "        words = [word for word in words if word not in stopwords3]\n",
        "        words = [word for word in words if word not in stopwords4]\n",
        "        words = [word for word in words if word not in stopwords5]\n",
        "        words = [word for word in words if word not in stopwords6]\n",
        "        words = [word for word in words if not isEnglish(word)]\n",
        "        words = [word for word in words if not hasNumbers(word)]\n",
        "        \n",
        "        \n",
        "\n",
        "#         words = [lemmatizer.lemmatize(word) for word in words]\n",
        "\n",
        "    #     text = \" \".join([stemmer.stem(word) for word in tokenizer.tokenize(text)])\n",
        "        text = \" \".join(words)\n",
        "        cleanedData.append(text)\n",
        "    return cleanedData"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "9-tMGEt5zdI2"
      },
      "source": [
        "cleanedData = preProcess(train_set)\n",
        "test_cleanedData = preProcess(test_set)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "tWmMQr-9znGO"
      },
      "source": [
        "with open(\"cleaned_data.txt\", \"wb\") as fp:\n",
        "    pickle.dump(cleanedData, fp)\n",
        "    \n",
        "with open(\"test_cleaned_data.txt\", \"wb\") as fp:\n",
        "    pickle.dump(test_cleanedData, fp)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "tc_-3fL_BBta"
      },
      "source": [
        "# Load preprocessed data"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "MhVDu693hO8V"
      },
      "source": [
        "with open(\"test_cleaned_data.txt\", \"rb\") as fp:   # Unpickling\n",
        "    test_cleanedData = pickle.load(fp)\n",
        "\n",
        "with open(\"cleaned_data.txt\", \"rb\") as fp:   # Unpickling\n",
        "    cleanedData = pickle.load(fp)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "UGCEZl13ivPl"
      },
      "source": [
        "train_categories = pd.read_csv('./category.csv', index_col = [0])"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "OAK8JGbQiwC5",
        "outputId": "1bb019ec-5044-4a54-c30f-5c6b26423713"
      },
      "source": [
        "# define ordinal encoding\n",
        "encoder = OrdinalEncoder()\n",
        "# transform data\n",
        "train_categories = encoder.fit_transform(train_categories)\n",
        "print(train_categories)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "[[23.]\n",
            " [31.]\n",
            " [ 0.]\n",
            " ...\n",
            " [18.]\n",
            " [23.]\n",
            " [21.]]\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "1a5onl1EPRBf"
      },
      "source": [
        "# Save model predictions"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "8Zj4Fs5uPQRQ"
      },
      "source": [
        "def saveModelResult(model, test_set, file_name):\n",
        "    predicted = model.predict_classes(test_set)\n",
        "    predicted_transformed = encoder.inverse_transform(predicted.reshape(-1, 1))\n",
        "    index = [str(i) for i in range(len(predicted_transformed))]\n",
        "    res = pd.DataFrame({'Id': index, 'Category': predicted_transformed[:,0]})\n",
        "    print(res.head())\n",
        "    res.to_csv(file_name, index=False)\n",
        "    print(file_name, ' saved!')"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "nDlwHVvljw0_"
      },
      "source": [
        "# Vectorize"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "LMk5AWfbjzQL"
      },
      "source": [
        "## tfidf"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "76BLds2djuGK"
      },
      "source": [
        "from sklearn.feature_extraction.text import TfidfVectorizer\n",
        "from sklearn.feature_selection import SelectPercentile, f_classif\n",
        "\n",
        "tfIdfVectorizer=TfidfVectorizer(sublinear_tf=True, max_df=0.3)\n",
        "tfIdf = tfIdfVectorizer.fit_transform(cleanedData)\n",
        "test_tfIdf = tfIdfVectorizer.transform(test_cleanedData)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Vydr6Flyjt_m"
      },
      "source": [
        "selector = SelectPercentile(f_classif, percentile=0.1)\n",
        "selector.fit(tfIdf, train_categories[:,0])\n",
        "tfIdf = selector.transform(tfIdf)\n",
        "test_tfIdf = selector.transform(test_tfIdf)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Uw9xS1gOjt3L"
      },
      "source": [
        "tfIdf = tfIdf.toarray()\n",
        "test_tfIdf = test_tfIdf.toarray()"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "zLM6_i5XjtsC"
      },
      "source": [
        "print(type(test_tfIdf))\n",
        "print(test_tfIdf.shape)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "cBoDjUyqkNs_"
      },
      "source": [
        "# LSTM"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "VI2wK_lQkKIi"
      },
      "source": [
        "MAX_NB_WORDS = 45000\n",
        "MAX_SEQUENCE_LENGTH = 300\n",
        "EMBEDDING_DIM = 100\n",
        "\n",
        "tokenizer = Tokenizer(num_words=MAX_NB_WORDS, filters='!\"#$%&()*+,-./:;<=>?@[\\]^_`{|}~', lower=True)\n",
        "tokenizer.fit_on_texts(cleanedData)\n",
        "word_index = tokenizer.word_index\n",
        "\n",
        "X = tokenizer.texts_to_sequences(cleanedData)\n",
        "X = pad_sequences(X, maxlen=MAX_SEQUENCE_LENGTH)\n",
        "\n",
        "Y = pd.get_dummies(train_categories[:,0]).values\n",
        "\n",
        "x_test = tokenizer.texts_to_sequences(test_cleanedData)\n",
        "x_test = pad_sequences(x_test, maxlen=MAX_SEQUENCE_LENGTH)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "3p1C5WG5vnQy"
      },
      "source": [
        "# Score : 0.851\n",
        "\n",
        "*   MAX_NB_WORDS = 45000\n",
        "*   MAX_SEQUENCE_LENGTH = 300\n",
        "*   EMBEDDING_DIM = 100\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "cPxvpwfvvnJF"
      },
      "source": [
        ""
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "dCK4f62mkeV-",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "64e56540-d01e-428a-b4bb-c2c92f5b4f2f"
      },
      "source": [
        "model = Sequential()\n",
        "model.add(Embedding(MAX_NB_WORDS, EMBEDDING_DIM, input_length=X.shape[1]))\n",
        "model.add(SpatialDropout1D(0.4))\n",
        "model.add(LSTM(512, dropout=0.5))\n",
        "model.add(Dense(34, activation='softmax'))\n",
        "model.compile(loss='categorical_crossentropy', optimizer='rmsprop', metrics=['accuracy'])\n",
        "print(model.summary())"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Model: \"sequential_3\"\n",
            "_________________________________________________________________\n",
            "Layer (type)                 Output Shape              Param #   \n",
            "=================================================================\n",
            "embedding_3 (Embedding)      (None, 300, 100)          4500000   \n",
            "_________________________________________________________________\n",
            "spatial_dropout1d_3 (Spatial (None, 300, 100)          0         \n",
            "_________________________________________________________________\n",
            "lstm_3 (LSTM)                (None, 512)               1255424   \n",
            "_________________________________________________________________\n",
            "dense_3 (Dense)              (None, 34)                17442     \n",
            "=================================================================\n",
            "Total params: 5,772,866\n",
            "Trainable params: 5,772,866\n",
            "Non-trainable params: 0\n",
            "_________________________________________________________________\n",
            "None\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "OvjgFibXkeMm",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "f6b81c92-0aac-49e1-d2b5-4050efe0b561"
      },
      "source": [
        "epochs = 6\n",
        "batch_size = 64\n",
        "\n",
        "history = model.fit(X, Y, epochs=epochs, batch_size=batch_size,validation_split=0.1)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Epoch 1/6\n",
            "2111/2111 [==============================] - 168s 78ms/step - loss: 1.9468 - accuracy: 0.4057 - val_loss: 0.9469 - val_accuracy: 0.7248\n",
            "Epoch 2/6\n",
            "2111/2111 [==============================] - 171s 81ms/step - loss: 0.8686 - accuracy: 0.7465 - val_loss: 0.7032 - val_accuracy: 0.7856\n",
            "Epoch 3/6\n",
            "2111/2111 [==============================] - 171s 81ms/step - loss: 0.6870 - accuracy: 0.7914 - val_loss: 0.6422 - val_accuracy: 0.8043\n",
            "Epoch 4/6\n",
            "2111/2111 [==============================] - 171s 81ms/step - loss: 0.6103 - accuracy: 0.8114 - val_loss: 0.6008 - val_accuracy: 0.8161\n",
            "Epoch 5/6\n",
            "2111/2111 [==============================] - 171s 81ms/step - loss: 0.5664 - accuracy: 0.8241 - val_loss: 0.5892 - val_accuracy: 0.8207\n",
            "Epoch 6/6\n",
            "2111/2111 [==============================] - 171s 81ms/step - loss: 0.5393 - accuracy: 0.8341 - val_loss: 0.5828 - val_accuracy: 0.8227\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "LnQiSFuHJ9n1",
        "outputId": "f861e8dd-63cc-41a4-e865-d4863ef26d5b"
      },
      "source": [
        "epochs = 3\n",
        "batch_size = 64\n",
        "\n",
        "history = model.fit(X, Y, epochs=epochs, batch_size=batch_size,validation_split=0.1)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Epoch 1/3\n",
            "2111/2111 [==============================] - 170s 81ms/step - loss: 0.5126 - accuracy: 0.8431 - val_loss: 0.5801 - val_accuracy: 0.8250\n",
            "Epoch 2/3\n",
            "2111/2111 [==============================] - 169s 80ms/step - loss: 0.4906 - accuracy: 0.8490 - val_loss: 0.5848 - val_accuracy: 0.8270\n",
            "Epoch 3/3\n",
            "2111/2111 [==============================] - 169s 80ms/step - loss: 0.4741 - accuracy: 0.8554 - val_loss: 0.5900 - val_accuracy: 0.8219\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "oz8hxWCYOcy9",
        "outputId": "84135888-6cf4-4aaa-84b8-9d76796f6f83"
      },
      "source": [
        "epochs = 1\n",
        "batch_size = 64\n",
        "\n",
        "history = model.fit(X, Y, epochs=epochs, batch_size=batch_size,validation_split=0.1)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "2111/2111 [==============================] - 163s 77ms/step - loss: 0.4551 - accuracy: 0.8610 - val_loss: 0.5747 - val_accuracy: 0.8314\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "RiXsEANokeFT",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "e3fdea5f-bffc-44b5-9ed3-ed6d1179f53f"
      },
      "source": [
        "saveModelResult(model, x_test, 'result_17.csv')"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.6/dist-packages/tensorflow/python/keras/engine/sequential.py:450: UserWarning: `model.predict_classes()` is deprecated and will be removed after 2021-01-01. Please use instead:* `np.argmax(model.predict(x), axis=-1)`,   if your model does multi-class classification   (e.g. if it uses a `softmax` last-layer activation).* `(model.predict(x) > 0.5).astype(\"int32\")`,   if your model does binary classification   (e.g. if it uses a `sigmoid` last-layer activation).\n",
            "  warnings.warn('`model.predict_classes()` is deprecated and '\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "  Id             Category\n",
            "0  0  Miscellaneous.Urban\n",
            "1  1        Miscellaneous\n",
            "2  2              Economy\n",
            "3  3  Science and Culture\n",
            "4  4             Politics\n",
            "result_17.csv  saved!\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "WMvBuzPoTmj8",
        "outputId": "b2944540-32ed-4551-9569-5ca8d8d38a4d"
      },
      "source": [
        "epochs = 1\n",
        "batch_size = 64\n",
        "\n",
        "history = model.fit(X, Y, epochs=epochs, batch_size=batch_size,validation_split=0.1)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "2111/2111 [==============================] - 164s 78ms/step - loss: 0.4446 - accuracy: 0.8645 - val_loss: 0.5896 - val_accuracy: 0.8330\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "ebpVF3seUT45",
        "outputId": "19098e70-ab1b-405b-fa50-95d3f1cec07f"
      },
      "source": [
        "saveModelResult(model, x_test, 'result_18.csv')"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.6/dist-packages/tensorflow/python/keras/engine/sequential.py:450: UserWarning: `model.predict_classes()` is deprecated and will be removed after 2021-01-01. Please use instead:* `np.argmax(model.predict(x), axis=-1)`,   if your model does multi-class classification   (e.g. if it uses a `softmax` last-layer activation).* `(model.predict(x) > 0.5).astype(\"int32\")`,   if your model does binary classification   (e.g. if it uses a `sigmoid` last-layer activation).\n",
            "  warnings.warn('`model.predict_classes()` is deprecated and '\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "  Id             Category\n",
            "0  0  Miscellaneous.Urban\n",
            "1  1        Miscellaneous\n",
            "2  2              Economy\n",
            "3  3  Science and Culture\n",
            "4  4             Politics\n",
            "result_18.csv  saved!\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "I8JvdEM4UYew",
        "outputId": "b3f51711-477e-486f-c99f-c4cca656dfda"
      },
      "source": [
        "epochs = 1\n",
        "batch_size = 64\n",
        "\n",
        "history = model.fit(X, Y, epochs=epochs, batch_size=batch_size,validation_split=0.1)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "2111/2111 [==============================] - 165s 78ms/step - loss: 0.4185 - accuracy: 0.8738 - val_loss: 0.5968 - val_accuracy: 0.8345\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "22wslDZqbhQ8",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "da9ed35a-e505-4a99-a5d4-a9c7fe9715db"
      },
      "source": [
        "saveModelResult(model, x_test, 'result_19.csv')"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.6/dist-packages/tensorflow/python/keras/engine/sequential.py:450: UserWarning: `model.predict_classes()` is deprecated and will be removed after 2021-01-01. Please use instead:* `np.argmax(model.predict(x), axis=-1)`,   if your model does multi-class classification   (e.g. if it uses a `softmax` last-layer activation).* `(model.predict(x) > 0.5).astype(\"int32\")`,   if your model does binary classification   (e.g. if it uses a `sigmoid` last-layer activation).\n",
            "  warnings.warn('`model.predict_classes()` is deprecated and '\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "  Id             Category\n",
            "0  0  Miscellaneous.Urban\n",
            "1  1        Miscellaneous\n",
            "2  2              Economy\n",
            "3  3  Science and Culture\n",
            "4  4             Politics\n",
            "result_19.csv  saved!\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "XIkEt5ZWSAy4",
        "outputId": "71ce7860-7ea9-4e3e-c193-82de277ae131"
      },
      "source": [
        "predicted = model.predict_classes(x_test)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.6/dist-packages/tensorflow/python/keras/engine/sequential.py:450: UserWarning: `model.predict_classes()` is deprecated and will be removed after 2021-01-01. Please use instead:* `np.argmax(model.predict(x), axis=-1)`,   if your model does multi-class classification   (e.g. if it uses a `softmax` last-layer activation).* `(model.predict(x) > 0.5).astype(\"int32\")`,   if your model does binary classification   (e.g. if it uses a `sigmoid` last-layer activation).\n",
            "  warnings.warn('`model.predict_classes()` is deprecated and '\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "qAOGGWhLzdmC"
      },
      "source": [
        "# **Score : 86.2 (BEST MODEL)**"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "JA5nNOrJ_ACy"
      },
      "source": [
        "## calculate class weight"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "N1-Rmo-O-_rE"
      },
      "source": [
        "from sklearn.utils import class_weight\n",
        "\n",
        "yy = np.argmax(Y, axis=1)\n",
        "weights = class_weight.compute_class_weight('balanced',\n",
        "                                            np.unique(yy),\n",
        "                                            yy)\n",
        "class_weights = {c : w for c, w in enumerate(weights)}"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "E8gOPFNz_1nd",
        "outputId": "176adc5a-a91b-41d8-c98b-d84a2eb25080"
      },
      "source": [
        "class_weights"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "{0: 0.24692852865500153,\n",
              " 1: 10.125202374527793,\n",
              " 2: 6.370257193786606,\n",
              " 3: 14.379766238743054,\n",
              " 4: 73.5764705882353,\n",
              " 5: 5.039484286865431,\n",
              " 6: 4.027908973808501,\n",
              " 7: 0.9520354184373772,\n",
              " 8: 4.445708192642616,\n",
              " 9: 27.764705882352942,\n",
              " 10: 43.280276816608996,\n",
              " 11: 147.1529411764706,\n",
              " 12: 157.6638655462185,\n",
              " 13: 0.12979883670853892,\n",
              " 14: 1.3053188158764393,\n",
              " 15: 7.559226430298146,\n",
              " 16: 10.63756201275691,\n",
              " 17: 30.031212484993997,\n",
              " 18: 0.26213337897358335,\n",
              " 19: 0.33060647309923746,\n",
              " 20: 5.187530241238681,\n",
              " 21: 0.25843509163412465,\n",
              " 22: 6.1228685649016885,\n",
              " 23: 0.41935862404237845,\n",
              " 24: 4.777692895339954,\n",
              " 25: 20.823529411764707,\n",
              " 26: 16.596196373286155,\n",
              " 27: 147.1529411764706,\n",
              " 28: 0.5543179602328123,\n",
              " 29: 8.6560553633218,\n",
              " 30: 14.19481747682996,\n",
              " 31: 0.31233820824211955,\n",
              " 32: 22.755609460278958,\n",
              " 33: 12.908152734778122}"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 53
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "tFW7NkwT_EOC"
      },
      "source": [
        "## model"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "A70PH3l_2B30",
        "outputId": "40e2f638-3bdd-4a81-a7c9-f1f2ef7512fb"
      },
      "source": [
        "model = Sequential()\n",
        "model.add(Embedding(MAX_NB_WORDS, EMBEDDING_DIM, input_length=X.shape[1]))\n",
        "model.add(SpatialDropout1D(0.4))\n",
        "model.add(LSTM(512, dropout=0.5))\n",
        "model.add(Dense(34, activation='softmax'))\n",
        "model.compile(loss='categorical_crossentropy', optimizer='rmsprop', metrics=['accuracy'])\n",
        "print(model.summary())"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Model: \"sequential_10\"\n",
            "_________________________________________________________________\n",
            "Layer (type)                 Output Shape              Param #   \n",
            "=================================================================\n",
            "embedding_10 (Embedding)     (None, 300, 100)          4500000   \n",
            "_________________________________________________________________\n",
            "spatial_dropout1d_10 (Spatia (None, 300, 100)          0         \n",
            "_________________________________________________________________\n",
            "lstm_10 (LSTM)               (None, 512)               1255424   \n",
            "_________________________________________________________________\n",
            "dense_10 (Dense)             (None, 34)                17442     \n",
            "=================================================================\n",
            "Total params: 5,772,866\n",
            "Trainable params: 5,772,866\n",
            "Non-trainable params: 0\n",
            "_________________________________________________________________\n",
            "None\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "ZObq4xDa2YtD",
        "outputId": "7814bc59-ab44-4b5a-dd14-39c9e4b99edf"
      },
      "source": [
        "epochs = 10\n",
        "batch_size = 64\n",
        "\n",
        "history = model.fit(X, Y, shuffle=True, epochs=epochs, batch_size=batch_size,validation_split=0.1)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Epoch 1/10\n",
            "2111/2111 [==============================] - 170s 80ms/step - loss: 1.8738 - accuracy: 0.4229 - val_loss: 1.0345 - val_accuracy: 0.6946\n",
            "Epoch 2/10\n",
            "2111/2111 [==============================] - 167s 79ms/step - loss: 0.9183 - accuracy: 0.7285 - val_loss: 0.7305 - val_accuracy: 0.7787\n",
            "Epoch 3/10\n",
            "2111/2111 [==============================] - 167s 79ms/step - loss: 0.6906 - accuracy: 0.7910 - val_loss: 0.6405 - val_accuracy: 0.8035\n",
            "Epoch 4/10\n",
            "2111/2111 [==============================] - 168s 79ms/step - loss: 0.6148 - accuracy: 0.8095 - val_loss: 0.6160 - val_accuracy: 0.8095\n",
            "Epoch 5/10\n",
            "2111/2111 [==============================] - 168s 79ms/step - loss: 0.5705 - accuracy: 0.8244 - val_loss: 0.6033 - val_accuracy: 0.8175\n",
            "Epoch 6/10\n",
            "2111/2111 [==============================] - 168s 80ms/step - loss: 0.5386 - accuracy: 0.8324 - val_loss: 0.5813 - val_accuracy: 0.8209\n",
            "Epoch 7/10\n",
            "2111/2111 [==============================] - 168s 80ms/step - loss: 0.5147 - accuracy: 0.8411 - val_loss: 0.5758 - val_accuracy: 0.8273\n",
            "Epoch 8/10\n",
            "2111/2111 [==============================] - 168s 79ms/step - loss: 0.4966 - accuracy: 0.8472 - val_loss: 0.5743 - val_accuracy: 0.8283\n",
            "Epoch 9/10\n",
            "2111/2111 [==============================] - 167s 79ms/step - loss: 0.4775 - accuracy: 0.8524 - val_loss: 0.5996 - val_accuracy: 0.8237\n",
            "Epoch 10/10\n",
            "2111/2111 [==============================] - 168s 80ms/step - loss: 0.4575 - accuracy: 0.8597 - val_loss: 0.5772 - val_accuracy: 0.8307\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "E1C_1C_f56Vv",
        "outputId": "4cd0152c-8766-42e0-9b70-a5dcbf6c72d7"
      },
      "source": [
        "epochs = 1\n",
        "batch_size = 164\n",
        "\n",
        "history = model.fit(X, Y, shuffle=True, epochs=epochs, batch_size=batch_size,validation_split=0.1)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "824/824 [==============================] - 143s 174ms/step - loss: 0.4232 - accuracy: 0.8691 - val_loss: 0.5755 - val_accuracy: 0.8325\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "SBDGnoTSghwz",
        "outputId": "cb9aa843-f31f-478c-82e4-19aeaa0fc007"
      },
      "source": [
        "saveModelResult(model, x_test, 'result_21.csv')"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.6/dist-packages/tensorflow/python/keras/engine/sequential.py:450: UserWarning: `model.predict_classes()` is deprecated and will be removed after 2021-01-01. Please use instead:* `np.argmax(model.predict(x), axis=-1)`,   if your model does multi-class classification   (e.g. if it uses a `softmax` last-layer activation).* `(model.predict(x) > 0.5).astype(\"int32\")`,   if your model does binary classification   (e.g. if it uses a `sigmoid` last-layer activation).\n",
            "  warnings.warn('`model.predict_classes()` is deprecated and '\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "  Id             Category\n",
            "0  0  Miscellaneous.Urban\n",
            "1  1        Miscellaneous\n",
            "2  2              Economy\n",
            "3  3  Science and Culture\n",
            "4  4             Politics\n",
            "result_21.csv  saved!\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "CmLhJkYZ7IFA",
        "outputId": "29c33a33-406a-421f-bd88-9a1626e09eee"
      },
      "source": [
        "epochs = 1\n",
        "batch_size = 164\n",
        "\n",
        "history = model.fit(X, Y, shuffle=True, epochs=epochs, batch_size=batch_size,validation_split=0.1)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "824/824 [==============================] - 143s 174ms/step - loss: 0.3978 - accuracy: 0.8753 - val_loss: 0.5678 - val_accuracy: 0.8350\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "OkTCXiiyYs9Q",
        "outputId": "6684ebdc-bb53-4d44-d74b-19db47764b11"
      },
      "source": [
        "saveModelResult(model, x_test, 'result_22.csv')"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.6/dist-packages/tensorflow/python/keras/engine/sequential.py:450: UserWarning: `model.predict_classes()` is deprecated and will be removed after 2021-01-01. Please use instead:* `np.argmax(model.predict(x), axis=-1)`,   if your model does multi-class classification   (e.g. if it uses a `softmax` last-layer activation).* `(model.predict(x) > 0.5).astype(\"int32\")`,   if your model does binary classification   (e.g. if it uses a `sigmoid` last-layer activation).\n",
            "  warnings.warn('`model.predict_classes()` is deprecated and '\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "  Id             Category\n",
            "0  0  Miscellaneous.Urban\n",
            "1  1        Miscellaneous\n",
            "2  2              Economy\n",
            "3  3  Science and Culture\n",
            "4  4             Politics\n",
            "result_22.csv  saved!\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "zVsELxrx58bZ",
        "outputId": "f524ceb7-178c-413a-b6e6-66964cb958ab"
      },
      "source": [
        "epochs = 1\n",
        "batch_size = 128\n",
        "\n",
        "history = model.fit(X, Y, shuffle=True, epochs=epochs, batch_size=batch_size,validation_split=0.1)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "1056/1056 [==============================] - 139s 131ms/step - loss: 0.3026 - accuracy: 0.9039 - val_loss: 0.6403 - val_accuracy: 0.8243\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "WrYnKyjCjYSu"
      },
      "source": [
        "# Custom Model"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Xei8bDmrjUra",
        "outputId": "5f282a2b-c2aa-4add-e65c-34b72974b1c6"
      },
      "source": [
        "model = Sequential()\n",
        "model.add(Embedding(MAX_NB_WORDS, EMBEDDING_DIM, input_length=X.shape[1]))\n",
        "model.add(SpatialDropout1D(0.4))\n",
        "model.add(LSTM(512, dropout=0.5))\n",
        "model.add(Dense(34, activation='softmax'))\n",
        "model.compile(loss='categorical_crossentropy', optimizer='rmsprop', metrics=['accuracy'])\n",
        "print(model.summary())"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Model: \"sequential_3\"\n",
            "_________________________________________________________________\n",
            "Layer (type)                 Output Shape              Param #   \n",
            "=================================================================\n",
            "embedding_3 (Embedding)      (None, 300, 100)          4000000   \n",
            "_________________________________________________________________\n",
            "spatial_dropout1d_3 (Spatial (None, 300, 100)          0         \n",
            "_________________________________________________________________\n",
            "lstm_3 (LSTM)                (None, 512)               1255424   \n",
            "_________________________________________________________________\n",
            "dense_3 (Dense)              (None, 34)                17442     \n",
            "=================================================================\n",
            "Total params: 5,272,866\n",
            "Trainable params: 5,272,866\n",
            "Non-trainable params: 0\n",
            "_________________________________________________________________\n",
            "None\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "V6ETdmB1jotc",
        "outputId": "8d657920-68b9-442d-d555-d442c46eb73f"
      },
      "source": [
        "epochs = 10\n",
        "batch_size = 128\n",
        "\n",
        "history = model.fit(X, Y, shuffle=True, epochs=epochs, batch_size=batch_size,validation_split=0.1)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Epoch 1/10\n",
            "1056/1056 [==============================] - 102s 96ms/step - loss: 2.0382 - accuracy: 0.3775 - val_loss: 1.1243 - val_accuracy: 0.6529\n",
            "Epoch 2/10\n",
            "1056/1056 [==============================] - 100s 95ms/step - loss: 1.0775 - accuracy: 0.6765 - val_loss: 0.7640 - val_accuracy: 0.7665\n",
            "Epoch 3/10\n",
            "1056/1056 [==============================] - 100s 95ms/step - loss: 0.7278 - accuracy: 0.7795 - val_loss: 0.6767 - val_accuracy: 0.7922\n",
            "Epoch 4/10\n",
            "1056/1056 [==============================] - 100s 95ms/step - loss: 0.6291 - accuracy: 0.8063 - val_loss: 0.6416 - val_accuracy: 0.8061\n",
            "Epoch 5/10\n",
            "1056/1056 [==============================] - 100s 95ms/step - loss: 0.5736 - accuracy: 0.8218 - val_loss: 0.6040 - val_accuracy: 0.8149\n",
            "Epoch 6/10\n",
            "1056/1056 [==============================] - 100s 95ms/step - loss: 0.5310 - accuracy: 0.8320 - val_loss: 0.5964 - val_accuracy: 0.8161\n",
            "Epoch 7/10\n",
            "1056/1056 [==============================] - 101s 96ms/step - loss: 0.5010 - accuracy: 0.8436 - val_loss: 0.5702 - val_accuracy: 0.8247\n",
            "Epoch 8/10\n",
            "1056/1056 [==============================] - 101s 96ms/step - loss: 0.4724 - accuracy: 0.8501 - val_loss: 0.5647 - val_accuracy: 0.8245\n",
            "Epoch 9/10\n",
            "1056/1056 [==============================] - 101s 95ms/step - loss: 0.4470 - accuracy: 0.8605 - val_loss: 0.5613 - val_accuracy: 0.8300\n",
            "Epoch 10/10\n",
            "1056/1056 [==============================] - 101s 95ms/step - loss: 0.4232 - accuracy: 0.8667 - val_loss: 0.5899 - val_accuracy: 0.8245\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "QU0duRGUqLg1",
        "outputId": "7e3e9d1b-c454-455b-a8e5-93b99ff5f314"
      },
      "source": [
        "epochs = 2\n",
        "batch_size = 164\n",
        "\n",
        "history = model.fit(X, Y, shuffle=True, epochs=epochs, batch_size=batch_size,validation_split=0.1)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Epoch 1/2\n",
            "824/824 [==============================] - 95s 115ms/step - loss: 0.4089 - accuracy: 0.8706 - val_loss: 0.5734 - val_accuracy: 0.8286\n",
            "Epoch 2/2\n",
            "824/824 [==============================] - 96s 116ms/step - loss: 0.3896 - accuracy: 0.8770 - val_loss: 0.5783 - val_accuracy: 0.8272\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Ox03scXsxJSJ"
      },
      "source": [
        "# CNN"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "mUE1CW8YxGMC"
      },
      "source": [
        "model = Sequential()\n",
        "model.add(Embedding(MAX_NB_WORDS, EMBEDDING_DIM, input_length=X.shape[1]))\n",
        "model.add(Conv1D(128, 7, activation='relu'))\n",
        "model.add(GlobalMaxPooling1D())\n",
        "model.add(Dense(512, activation='relu'))\n",
        "model.add(Dropout(0.5))\n",
        "model.add(Dense(128, activation='relu'))\n",
        "model.add(Dropout(0.3))\n",
        "model.add(Dense(34, activation='softmax'))\n",
        "model.compile(optimizer='rmsprop',\n",
        "              loss='categorical_crossentropy',\n",
        "              metrics=['acc'])"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "-ywHM52VxzEE",
        "outputId": "28eb7709-ed98-4702-de4d-edd6bcc61669"
      },
      "source": [
        "epochs = 5\n",
        "batch_size = 64\n",
        "\n",
        "history = model.fit(X, Y, epochs=epochs, batch_size=batch_size,validation_split=0.1)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Epoch 1/5\n",
            "2111/2111 [==============================] - 59s 28ms/step - loss: 1.3428 - acc: 0.6108 - val_loss: 0.7268 - val_acc: 0.7836\n",
            "Epoch 2/5\n",
            "2111/2111 [==============================] - 58s 28ms/step - loss: 0.6607 - acc: 0.8063 - val_loss: 0.7208 - val_acc: 0.7933\n",
            "Epoch 3/5\n",
            "2111/2111 [==============================] - 57s 27ms/step - loss: 0.5590 - acc: 0.8375 - val_loss: 0.7958 - val_acc: 0.7899\n",
            "Epoch 4/5\n",
            "2111/2111 [==============================] - 58s 27ms/step - loss: 0.4987 - acc: 0.8565 - val_loss: 0.9667 - val_acc: 0.7807\n",
            "Epoch 5/5\n",
            "2111/2111 [==============================] - 58s 28ms/step - loss: 0.4461 - acc: 0.8750 - val_loss: 1.0174 - val_acc: 0.7679\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "fuoUULNalFgo"
      },
      "source": [
        "# LSTM + CNN"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "LkkQd397yWd_",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "dac2b8d6-56d0-4af6-d699-384cab318670"
      },
      "source": [
        "model = Sequential()\n",
        "model.add(Embedding(MAX_NB_WORDS, EMBEDDING_DIM, input_length=X.shape[1]))\n",
        "model.add(Conv1D(filters=32, kernel_size=3, padding='same', activation='relu'))\n",
        "model.add(MaxPooling1D(pool_size=3))\n",
        "model.add(LSTM(300, dropout=0.5))\n",
        "model.add(Dense(34, activation='softmax'))\n",
        "model.compile(loss='categorical_crossentropy', optimizer='rmsprop', metrics=['accuracy'])\n",
        "print(model.summary())"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Model: \"sequential_8\"\n",
            "_________________________________________________________________\n",
            "Layer (type)                 Output Shape              Param #   \n",
            "=================================================================\n",
            "embedding_8 (Embedding)      (None, 300, 100)          4500000   \n",
            "_________________________________________________________________\n",
            "conv1d_5 (Conv1D)            (None, 300, 32)           9632      \n",
            "_________________________________________________________________\n",
            "max_pooling1d_5 (MaxPooling1 (None, 100, 32)           0         \n",
            "_________________________________________________________________\n",
            "lstm_8 (LSTM)                (None, 300)               399600    \n",
            "_________________________________________________________________\n",
            "dense_8 (Dense)              (None, 34)                10234     \n",
            "=================================================================\n",
            "Total params: 4,919,466\n",
            "Trainable params: 4,919,466\n",
            "Non-trainable params: 0\n",
            "_________________________________________________________________\n",
            "None\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "5Zi1eCq6lEJq",
        "outputId": "761b888f-cf33-435b-8d60-ef76392b470e"
      },
      "source": [
        "epochs = 5\n",
        "batch_size = 64\n",
        "\n",
        "history = model.fit(X, Y, shuffle=True, epochs=epochs, batch_size=batch_size,validation_split=0.1)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Epoch 1/5\n",
            "2111/2111 [==============================] - 66s 31ms/step - loss: 1.5940 - accuracy: 0.5007 - val_loss: 0.8190 - val_accuracy: 0.7576\n",
            "Epoch 2/5\n",
            "2111/2111 [==============================] - 64s 30ms/step - loss: 0.7854 - accuracy: 0.7637 - val_loss: 0.7044 - val_accuracy: 0.7879\n",
            "Epoch 3/5\n",
            "2111/2111 [==============================] - 64s 30ms/step - loss: 0.6688 - accuracy: 0.7977 - val_loss: 0.6653 - val_accuracy: 0.7950\n",
            "Epoch 4/5\n",
            "2111/2111 [==============================] - 64s 30ms/step - loss: 0.6136 - accuracy: 0.8120 - val_loss: 0.6410 - val_accuracy: 0.8045\n",
            "Epoch 5/5\n",
            "2111/2111 [==============================] - 64s 30ms/step - loss: 0.5614 - accuracy: 0.8305 - val_loss: 0.6530 - val_accuracy: 0.8005\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "BH5NcpFiwCUg",
        "outputId": "a1bed65f-a1ea-4e02-8a32-0cbe142acb08"
      },
      "source": [
        "saveModelResult(model, x_test, 'result_30.csv')"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.6/dist-packages/tensorflow/python/keras/engine/sequential.py:450: UserWarning: `model.predict_classes()` is deprecated and will be removed after 2021-01-01. Please use instead:* `np.argmax(model.predict(x), axis=-1)`,   if your model does multi-class classification   (e.g. if it uses a `softmax` last-layer activation).* `(model.predict(x) > 0.5).astype(\"int32\")`,   if your model does binary classification   (e.g. if it uses a `sigmoid` last-layer activation).\n",
            "  warnings.warn('`model.predict_classes()` is deprecated and '\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "  Id             Category\n",
            "0  0  Miscellaneous.Urban\n",
            "1  1        Miscellaneous\n",
            "2  2              Economy\n",
            "3  3  Science and Culture\n",
            "4  4             Politics\n",
            "result_30.csv  saved!\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "xwd00YWauVlC",
        "outputId": "f9fd153f-566f-4f74-f753-22ccd80981c3"
      },
      "source": [
        "epochs = 2\n",
        "batch_size = 64\n",
        "\n",
        "history = model.fit(X, Y, shuffle=True, epochs=epochs, batch_size=batch_size,validation_split=0.1)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Epoch 1/2\n",
            "2111/2111 [==============================] - 65s 31ms/step - loss: 0.5049 - accuracy: 0.8462 - val_loss: 0.6413 - val_accuracy: 0.8054\n",
            "Epoch 2/2\n",
            "2111/2111 [==============================] - 65s 31ms/step - loss: 0.4783 - accuracy: 0.8552 - val_loss: 0.6736 - val_accuracy: 0.7942\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "GAvwrAjdwn-W",
        "outputId": "eebb1a4f-5b4e-44cd-a429-abfd91c4824d"
      },
      "source": [
        "saveModelResult(model, x_test, 'result_31.csv')"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.6/dist-packages/tensorflow/python/keras/engine/sequential.py:450: UserWarning: `model.predict_classes()` is deprecated and will be removed after 2021-01-01. Please use instead:* `np.argmax(model.predict(x), axis=-1)`,   if your model does multi-class classification   (e.g. if it uses a `softmax` last-layer activation).* `(model.predict(x) > 0.5).astype(\"int32\")`,   if your model does binary classification   (e.g. if it uses a `sigmoid` last-layer activation).\n",
            "  warnings.warn('`model.predict_classes()` is deprecated and '\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "  Id                  Category\n",
            "0  0       Miscellaneous.Urban\n",
            "1  1             Miscellaneous\n",
            "2  2                   Economy\n",
            "3  3       Science and Culture\n",
            "4  4  Miscellaneous.World News\n",
            "result_31.csv  saved!\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "5ES7DnYgqh47"
      },
      "source": [
        "# Bidirectional LSTM"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "PWABasEHqjpU",
        "outputId": "61c59418-d19c-4666-9524-41723f4aa057"
      },
      "source": [
        "model = Sequential()\n",
        "model.add(Embedding(MAX_NB_WORDS, EMBEDDING_DIM, input_length=X.shape[1]))\n",
        "model.add(SpatialDropout1D(0.4))\n",
        "model.add(Bidirectional(LSTM(512, dropout=0.5)))\n",
        "model.add(Dense(34, activation='softmax'))\n",
        "model.compile(loss='categorical_crossentropy', optimizer='rmsprop', metrics=['accuracy'])\n",
        "print(model.summary())"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Model: \"sequential_2\"\n",
            "_________________________________________________________________\n",
            "Layer (type)                 Output Shape              Param #   \n",
            "=================================================================\n",
            "embedding_2 (Embedding)      (None, 300, 100)          4500000   \n",
            "_________________________________________________________________\n",
            "spatial_dropout1d_2 (Spatial (None, 300, 100)          0         \n",
            "_________________________________________________________________\n",
            "bidirectional (Bidirectional (None, 1024)              2510848   \n",
            "_________________________________________________________________\n",
            "dense_2 (Dense)              (None, 34)                34850     \n",
            "=================================================================\n",
            "Total params: 7,045,698\n",
            "Trainable params: 7,045,698\n",
            "Non-trainable params: 0\n",
            "_________________________________________________________________\n",
            "None\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "SdhvWgxDq10y",
        "outputId": "a5cf2bd5-ce80-47c1-ca65-80ec05938091"
      },
      "source": [
        "epochs = 8\n",
        "batch_size = 64\n",
        "\n",
        "history = model.fit(X, Y, epochs=epochs, batch_size=batch_size,validation_split=0.1)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Epoch 1/8\n",
            "2111/2111 [==============================] - 306s 144ms/step - loss: 1.8590 - accuracy: 0.4221 - val_loss: 1.1032 - val_accuracy: 0.6854\n",
            "Epoch 2/8\n",
            "2111/2111 [==============================] - 303s 144ms/step - loss: 0.9755 - accuracy: 0.7187 - val_loss: 0.7200 - val_accuracy: 0.7833\n",
            "Epoch 3/8\n",
            "2111/2111 [==============================] - 304s 144ms/step - loss: 0.7101 - accuracy: 0.7863 - val_loss: 0.6644 - val_accuracy: 0.7987\n",
            "Epoch 4/8\n",
            "2111/2111 [==============================] - 304s 144ms/step - loss: 0.6502 - accuracy: 0.8035 - val_loss: 0.6314 - val_accuracy: 0.8103\n",
            "Epoch 5/8\n",
            "2111/2111 [==============================] - 304s 144ms/step - loss: 0.5814 - accuracy: 0.8207 - val_loss: 0.6029 - val_accuracy: 0.8159\n",
            "Epoch 6/8\n",
            "2111/2111 [==============================] - 304s 144ms/step - loss: 0.5435 - accuracy: 0.8321 - val_loss: 0.5867 - val_accuracy: 0.8233\n",
            "Epoch 7/8\n",
            "2111/2111 [==============================] - 305s 145ms/step - loss: 0.5152 - accuracy: 0.8411 - val_loss: 0.5795 - val_accuracy: 0.8228\n",
            "Epoch 8/8\n",
            "2111/2111 [==============================] - 305s 145ms/step - loss: 0.4918 - accuracy: 0.8494 - val_loss: 0.5764 - val_accuracy: 0.8214\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Sk2BpVI2uyGf",
        "outputId": "0a7afefa-37f1-4c97-9c58-8285883e8ab1"
      },
      "source": [
        "epochs = 1\n",
        "batch_size = 164\n",
        "\n",
        "history = model.fit(X, Y, shuffle=True, epochs=epochs, batch_size=batch_size,validation_split=0.1)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "824/824 [==============================] - 274s 332ms/step - loss: 0.3271 - accuracy: 0.8985 - val_loss: 0.6237 - val_accuracy: 0.8303\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Jc85HHejcLhz",
        "outputId": "85502d6f-ddd9-4efe-9f6a-e54912664c6d"
      },
      "source": [
        "saveModelResult(model, x_test, 'result_24.csv')"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.6/dist-packages/tensorflow/python/keras/engine/sequential.py:450: UserWarning: `model.predict_classes()` is deprecated and will be removed after 2021-01-01. Please use instead:* `np.argmax(model.predict(x), axis=-1)`,   if your model does multi-class classification   (e.g. if it uses a `softmax` last-layer activation).* `(model.predict(x) > 0.5).astype(\"int32\")`,   if your model does binary classification   (e.g. if it uses a `sigmoid` last-layer activation).\n",
            "  warnings.warn('`model.predict_classes()` is deprecated and '\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "  Id             Category\n",
            "0  0  Miscellaneous.Urban\n",
            "1  1        Miscellaneous\n",
            "2  2              Economy\n",
            "3  3  Science and Culture\n",
            "4  4             Politics\n",
            "result_24.csv  saved!\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Of1cakqou0iu",
        "outputId": "03448779-5e33-4e7e-d442-b5327796d284"
      },
      "source": [
        "model3 = Sequential()\n",
        "model3.add(Embedding(MAX_NB_WORDS, EMBEDDING_DIM, input_length=X.shape[1]))\n",
        "model3.add(SpatialDropout1D(0.4))\n",
        "model3.add(Bidirectional(LSTM(300, dropout=0.5)))\n",
        "model3.add(Dense(34, activation='softmax'))\n",
        "model3.compile(loss='categorical_crossentropy', optimizer='rmsprop', metrics=['accuracy'])\n",
        "print(model3.summary())"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Model: \"sequential\"\n",
            "_________________________________________________________________\n",
            "Layer (type)                 Output Shape              Param #   \n",
            "=================================================================\n",
            "embedding (Embedding)        (None, 300, 100)          4500000   \n",
            "_________________________________________________________________\n",
            "spatial_dropout1d (SpatialDr (None, 300, 100)          0         \n",
            "_________________________________________________________________\n",
            "bidirectional (Bidirectional (None, 600)               962400    \n",
            "_________________________________________________________________\n",
            "dense (Dense)                (None, 34)                20434     \n",
            "=================================================================\n",
            "Total params: 5,482,834\n",
            "Trainable params: 5,482,834\n",
            "Non-trainable params: 0\n",
            "_________________________________________________________________\n",
            "None\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "-9UH50NAu9pv",
        "outputId": "27c4bbff-4283-42c2-8100-c51a7dcc246e"
      },
      "source": [
        "epochs = 5\n",
        "batch_size = 128\n",
        "\n",
        "history = model3.fit(X, Y, shuffle=True, epochs=epochs, batch_size=batch_size,validation_split=0.1)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Epoch 1/5\n",
            "1056/1056 [==============================] - 131s 116ms/step - loss: 1.8432 - accuracy: 0.4373 - val_loss: 1.2198 - val_accuracy: 0.6287\n",
            "Epoch 2/5\n",
            "1056/1056 [==============================] - 121s 114ms/step - loss: 1.1148 - accuracy: 0.6708 - val_loss: 0.9978 - val_accuracy: 0.7150\n",
            "Epoch 3/5\n",
            "1056/1056 [==============================] - 121s 114ms/step - loss: 0.8738 - accuracy: 0.7433 - val_loss: 0.7288 - val_accuracy: 0.7816\n",
            "Epoch 4/5\n",
            "1056/1056 [==============================] - 120s 114ms/step - loss: 0.6993 - accuracy: 0.7886 - val_loss: 0.6739 - val_accuracy: 0.7961\n",
            "Epoch 5/5\n",
            "1056/1056 [==============================] - 120s 114ms/step - loss: 0.6219 - accuracy: 0.8101 - val_loss: 0.6220 - val_accuracy: 0.8091\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "GjrztwZ_Dvv4",
        "outputId": "4cba833f-c335-4587-c52c-f6c3c70fa5d0"
      },
      "source": [
        "epochs = 5\n",
        "batch_size = 128\n",
        "\n",
        "history = model3.fit(X, Y, shuffle=True, epochs=epochs, batch_size=batch_size,validation_split=0.1)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Epoch 1/5\n",
            "1056/1056 [==============================] - 120s 114ms/step - loss: 0.4298 - accuracy: 0.8658 - val_loss: 0.5896 - val_accuracy: 0.8227\n",
            "Epoch 2/5\n",
            "1056/1056 [==============================] - 120s 113ms/step - loss: 0.4109 - accuracy: 0.8722 - val_loss: 0.5750 - val_accuracy: 0.8267\n",
            "Epoch 3/5\n",
            "1056/1056 [==============================] - 120s 114ms/step - loss: 0.3933 - accuracy: 0.8780 - val_loss: 0.6020 - val_accuracy: 0.8262\n",
            "Epoch 4/5\n",
            "1056/1056 [==============================] - 120s 113ms/step - loss: 0.3760 - accuracy: 0.8824 - val_loss: 0.6212 - val_accuracy: 0.8201\n",
            "Epoch 5/5\n",
            "1056/1056 [==============================] - 120s 114ms/step - loss: 0.3630 - accuracy: 0.8867 - val_loss: 0.6010 - val_accuracy: 0.8253\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "A-2xuHlVvPtl"
      },
      "source": [
        "saveModelResult(model, x_test, 'result_model.csv')\n",
        "saveModelResult(model2, x_test, 'result_model2.csv')\n",
        "saveModelResult(model3, x_test, 'result_model3.csv')"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "qPDy2_ftE2VU"
      },
      "source": [
        ""
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "P50GUsthE4fY"
      },
      "source": [
        "# Custom Model"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "y8QGrMgpE6B6",
        "outputId": "3ede4a9f-90f8-4c36-bc69-9197e9df9680"
      },
      "source": [
        "model2 = Sequential()\n",
        "model2.add(Embedding(MAX_NB_WORDS, EMBEDDING_DIM, input_length=X.shape[1]))\n",
        "model2.add(SpatialDropout1D(0.4))\n",
        "model2.add(LSTM(512, dropout=0.5))\n",
        "model2.add(Dense(34, activation='softmax'))\n",
        "model2.compile(loss='categorical_crossentropy', optimizer='rmsprop', metrics=['accuracy'])\n",
        "print(model2.summary())"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Model: \"sequential_1\"\n",
            "_________________________________________________________________\n",
            "Layer (type)                 Output Shape              Param #   \n",
            "=================================================================\n",
            "embedding_1 (Embedding)      (None, 300, 100)          4500000   \n",
            "_________________________________________________________________\n",
            "spatial_dropout1d_1 (Spatial (None, 300, 100)          0         \n",
            "_________________________________________________________________\n",
            "lstm_1 (LSTM)                (None, 512)               1255424   \n",
            "_________________________________________________________________\n",
            "dense_1 (Dense)              (None, 34)                17442     \n",
            "=================================================================\n",
            "Total params: 5,772,866\n",
            "Trainable params: 5,772,866\n",
            "Non-trainable params: 0\n",
            "_________________________________________________________________\n",
            "None\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "hZen5c4QFAsM",
        "outputId": "12014b7e-d60b-48f2-8de3-6f9c940a6a23"
      },
      "source": [
        "epochs = 10\n",
        "batch_size = 64\n",
        "\n",
        "history = model2.fit(X, Y, shuffle=True, epochs=epochs, batch_size=batch_size,validation_split=0.1)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Epoch 1/10\n",
            "2111/2111 [==============================] - 167s 78ms/step - loss: 1.9033 - accuracy: 0.4170 - val_loss: 1.0014 - val_accuracy: 0.7039\n",
            "Epoch 2/10\n",
            "2111/2111 [==============================] - 164s 78ms/step - loss: 0.8688 - accuracy: 0.7424 - val_loss: 0.6986 - val_accuracy: 0.7869\n",
            "Epoch 3/10\n",
            "2111/2111 [==============================] - 165s 78ms/step - loss: 0.6982 - accuracy: 0.7892 - val_loss: 0.6736 - val_accuracy: 0.7927\n",
            "Epoch 4/10\n",
            "2111/2111 [==============================] - 165s 78ms/step - loss: 0.6916 - accuracy: 0.7921 - val_loss: 0.6560 - val_accuracy: 0.8009\n",
            "Epoch 5/10\n",
            "2111/2111 [==============================] - 165s 78ms/step - loss: 0.6175 - accuracy: 0.8113 - val_loss: 0.6127 - val_accuracy: 0.8146\n",
            "Epoch 6/10\n",
            "2111/2111 [==============================] - 164s 78ms/step - loss: 0.5616 - accuracy: 0.8279 - val_loss: 0.6097 - val_accuracy: 0.8163\n",
            "Epoch 7/10\n",
            "2111/2111 [==============================] - 166s 78ms/step - loss: 0.5271 - accuracy: 0.8360 - val_loss: 0.5811 - val_accuracy: 0.8207\n",
            "Epoch 8/10\n",
            "2111/2111 [==============================] - 166s 79ms/step - loss: 0.5043 - accuracy: 0.8455 - val_loss: 0.6064 - val_accuracy: 0.8191\n",
            "Epoch 9/10\n",
            "2111/2111 [==============================] - 166s 79ms/step - loss: 0.4853 - accuracy: 0.8504 - val_loss: 0.5880 - val_accuracy: 0.8251\n",
            "Epoch 10/10\n",
            "2111/2111 [==============================] - 166s 79ms/step - loss: 0.4639 - accuracy: 0.8569 - val_loss: 0.5868 - val_accuracy: 0.8243\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "wn8ecFkrFxZG"
      },
      "source": [
        ""
      ],
      "execution_count": null,
      "outputs": []
    }
  ]
}