[4] classification.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "90d0ab48-0abb-4ada-9d89-8f04407446e1",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "from nltk.tokenize import word_tokenize\n",
    "from sklearn.preprocessing import LabelEncoder\n",
    "from sklearn.feature_extraction.text import CountVectorizer, TfidfVectorizer\n",
    "from sklearn import model_selection, svm\n",
    "from sklearn.metrics import accuracy_score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "57f5fff9-9cc7-477a-a787-a4beb3fe9191",
   "metadata": {},
   "outputs": [],
   "source": [
    "np.random.seed(500)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "9cf7f66b-f8a9-48f1-be26-dfe4f95c55a1",
   "metadata": {},
   "source": [
    "## Definisikan Classifier"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "cc82edbd-9f40-4c30-b3d4-06bf7db68cd4",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Menggunakan SVM dengan kernel linear sebagai classifier\n",
    "# random_state merupakan pseudo randomization, supaya hasil yang didapat akan tetap sama setiap kali eksperimen diulang\n",
    "clf = svm.SVC(kernel='linear', random_state=42)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d25eff8f-1bc5-40d6-84db-ba094409aa14",
   "metadata": {},
   "source": [
    "#### opsi lain yang bisa dipertimbangkan:\n",
    "<code> clf = svm.SVC(kernel='linear', C=0.9, random_state=42) </code>\n",
    "<blockquote> C = regularization, default=1 </blockquote> \n",
    "<code> clf = svm.SVC(C=500.0, kernel='poly', degree=4, coef0=0, gamma=1.) </code>\n",
    "<blockquote> poly kernel for multiclass labeling </blockquote> "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "750efd1c-e715-4ba6-bd71-0659ede633e7",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "id": "ac8cdfc4-c53b-405f-96aa-8c9362efab18",
   "metadata": {},
   "source": [
    "## Persiapan Input\n",
    "**Corpus** memuat teks yang sudah dibersihkan (di tahap prapengolahan).\n",
    "**LabelInset** memuat label teks dengan leksikon `InSet`.\n",
    "**LabelSenti** memuat label teks dengan leksikon `sentiwords_id` dari sentistrength_id."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "8a215f4a-968c-473d-9c26-e6047e996013",
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "\n",
    "# Pastikan untuk mengganti path dengan absolute path direktorimu jika baris berikut dijalankan ulang, atau restart kernel.\n",
    "os.chdir('output')\n",
    "base = 'prastyo-sentiment_posneg-clean-slang-stop-dup.txt'\n",
    "lb_inset = 'prastyo-sentiment_posneg-clean-slang-stop-lb-inset.txt'\n",
    "lb_senti = 'prastyo-sentiment_posneg-clean-slang-stop-lb-senti.txt'\n",
    "\n",
    "Corpus = pd.read_csv(base, encoding='latin-1', header=None, sep='\\t', names=['text', 'label'], dtype=str)\n",
    "LabelInset = pd.read_csv(lb_inset, encoding='latin-1', header=None, names=['label'], dtype=str)\n",
    "LabelSenti = pd.read_csv(lb_senti, encoding='latin-1', header=None, names=['label'], dtype=str)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "8b2ebced-7d02-4f2e-9c49-1b11f633b355",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "neg: 900 ( 54.22 %) \t pos: 760 ( 45.78 %)  | actual label\n",
      "neg: 1197 ( 72.11 %) \t pos: 463 ( 27.89 %)  | inset\n",
      "neg: 1114 ( 67.11 %) \t pos: 546 ( 32.89 %)  | senti\n"
     ]
    }
   ],
   "source": [
    "# Jumlah positif dan negatif dari setiap jenis pelabelan\n",
    "neg0, pos0 = (Corpus['label'][Corpus['label']=='neg']).count(), (Corpus['label'][Corpus['label']=='pos']).count()\n",
    "neg1, pos1 = (LabelInset['label'][LabelInset['label']=='neg']).count(), (LabelInset['label'][LabelInset['label']=='pos']).count()\n",
    "neg2, pos2 = (LabelSenti['label'][LabelSenti['label']=='neg']).count(), (LabelSenti['label'][LabelSenti['label']=='pos']).count()\n",
    "print('neg:', neg0, '(', '{0:.2f}'.format(neg0/(neg0+pos0)*100), '%)','\\t', 'pos:', pos0, '(', '{0:.2f}'.format(pos0/(neg0+pos0)*100),'%)',' | actual label')\n",
    "print('neg:', neg1, '(', '{0:.2f}'.format(neg1/(neg1+pos1)*100), '%)','\\t', 'pos:', pos1, '(', '{0:.2f}'.format(pos1/(neg1+pos1)*100),'%)',' | inset')\n",
    "print('neg:', neg2, '(', '{0:.2f}'.format(neg2/(neg2+pos2)*100), '%)','\\t', 'pos:', pos2, '(', '{0:.2f}'.format(pos2/(neg2+pos2)*100),'%)',' | senti')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f8444630-b8ed-4222-9ffb-2030b8972d32",
   "metadata": {},
   "source": [
    "### **\\*Perhatian:** pilih salah satu jenis label sebagai *baseline* untuk proses selanjutnya\n",
    "`LLmark` akan digunakan nanti sebagai pembeda nama file saat menyimpan skor akurasi ke file"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "7d42e72e-3424-4c67-b420-363167eb8c7b",
   "metadata": {},
   "outputs": [],
   "source": [
    "## Menggunakan label aktual sebagai baseline terhadap dirinya sendiri\n",
    "# LL = Corpus[['label']]\n",
    "# LLmark = 0\n",
    "\n",
    "## Menggunakan pelabelan dari InSet\n",
    "LL = LabelInset\n",
    "LLmark = 1\n",
    "\n",
    "## Menggunakan pelabelan dari sentiwords_id\n",
    "# LL = LabelSenti\n",
    "# LLmark = 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "2a725dac-2339-4f35-9a87-3181bb2f4e04",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "                                                text label\n",
      "0  ya utang pemerintah utang bangsa indonesia hut...   neg\n",
      "1  yuk kawal kebijakan pemerintah disalah oknum b...   pos\n",
      "2  yuk bahu membahu membantuu pemerintah memutus ...   pos \n",
      "\n",
      "   label\n",
      "0   neg\n",
      "1   pos\n",
      "2   pos\n"
     ]
    }
   ],
   "source": [
    "print(Corpus[:3], '\\n\\n', LL[:3])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "5e582abd-8e38-4551-947a-faa27619d3ea",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "id": "5dcc8f60-34c9-4dd4-a53c-69dddc77d3b7",
   "metadata": {},
   "source": [
    "## Tokenisasi Teks"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "6ddd5ad0-8cd3-4d69-b0e6-dd291b7986c5",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Step - a : Menghapus baris kosong, jika ada.\n",
    "Corpus['text'].dropna(inplace=True)\n",
    "# # Step - b : Mengganti semua teks ke karakter kecil karena 'oke' dan 'OKE' diinterpretasikan berbeda\n",
    "# Corpus['text'] = [entry.lower() for entry in Corpus['text']] # we've done this in '[1] text cleaning.ipynb'\n",
    "# Step - c : Tokenisasi : Setiap kalimat di dalam korpus akan dipecah menjadi daftar kata/string\n",
    "Corpus['text']= [word_tokenize(entry) for entry in Corpus['text']]\n",
    "\n",
    "for index,entry in enumerate(Corpus['text']):\n",
    "    # Mendeklarasikan list kosong untuk menyimpan daftar kata yang sesuai dengan aturan yang dibuat\n",
    "    Final_words = []\n",
    "    for word in entry:\n",
    "        # Kondisi di bawah adalah untuk mengecek/mempertimbangkan alfabet saja\n",
    "        if word.isalpha():\n",
    "            word_Final = word\n",
    "            Final_words.append(word_Final)\n",
    "    Corpus.loc[index,'text_final'] = str(Final_words)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "0c2c1ede-2a32-45c4-b903-849e62b5cbb3",
   "metadata": {},
   "source": [
    "<blockquote>Ref: <i>https://medium.com/@bedigunjit/simple-guide-to-text-classification-nlp-using-svm-and-naive-bayes-with-python-421db3a72d34</i></blockquote>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "10a3e152-9069-49e2-ba4d-de372502365c",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "                                                text label  \\\n",
      "0  [ya, utang, pemerintah, utang, bangsa, indones...   neg   \n",
      "1  [yuk, kawal, kebijakan, pemerintah, disalah, o...   pos   \n",
      "2  [yuk, bahu, membahu, membantuu, pemerintah, me...   pos   \n",
      "\n",
      "                                          text_final  \n",
      "0  ['ya', 'utang', 'pemerintah', 'utang', 'bangsa...  \n",
      "1  ['yuk', 'kawal', 'kebijakan', 'pemerintah', 'd...  \n",
      "2  ['yuk', 'bahu', 'membahu', 'membantuu', 'pemer...  \n"
     ]
    }
   ],
   "source": [
    "print(Corpus[:3])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f92bed30-d127-42fb-b3f0-434077c6bfd0",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "id": "84896dff-5a49-49c8-a37d-5796ac413ba7",
   "metadata": {},
   "source": [
    "## Split Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "8c5cb0f8-ea35-4fe1-a2aa-a921608b97f4",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Membagi set data latih dan data uji dengan rasio 70:30\n",
    "Train_X, Test_X, Train_Y, Test_Y = model_selection.train_test_split(Corpus['text_final'],LL['label'],test_size=0.3, random_state=42)\n",
    "Train_Y_Actual, Test_Y_Actual = model_selection.train_test_split(Corpus['label'],test_size=0.3, random_state=42)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "706454cf-944c-4cc5-9f91-e879e6f3e061",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1162 0.7 % \n",
      " 498 0.3 %\n"
     ]
    }
   ],
   "source": [
    "print(Train_X.size, Train_X.size/(Test_X.size+Train_X.size),'%','\\n',\n",
    "      Test_X.size, Test_X.size/(Test_X.size+Train_X.size),'%')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "7227c351-e99d-4665-bec3-ef1d3a7af589",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Encoding label menjadi nilai antara 0 and kelas_n-1\n",
    "Encoder = LabelEncoder()\n",
    "Train_Y = Encoder.fit_transform(Train_Y)\n",
    "Test_Y = Encoder.fit_transform(Test_Y)\n",
    "Train_Y_Actual = Encoder.fit_transform(Train_Y_Actual)\n",
    "Test_Y_Actual = Encoder.fit_transform(Test_Y_Actual)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "c88b988e-12c5-4032-a2b7-dabf3bf5164d",
   "metadata": {},
   "outputs": [],
   "source": [
    "# print('TRAIN_X'+'\\n', Train_X, '\\n')\n",
    "# print('TEST_X'+'\\n', Test_X, '\\n')\n",
    "# print('TRAIN_Y'+'\\n', Train_Y, '\\n')\n",
    "# print('TEST_Y'+'\\n', Test_Y, '\\n')\n",
    "# # with np.printoptions():\n",
    "# #     print(Test_X[:17])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b5873111-e7d9-484a-ae32-3d4fd6929957",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "id": "a0e4373b-466e-4a86-8c2c-e4321cee603f",
   "metadata": {},
   "source": [
    "# EKSTRAKSI FITUR: *Term presence*"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "3ab7b05e-1ff5-425b-b27d-b87e3a2439de",
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.feature_extraction.text import CountVectorizer\n",
    "\n",
    "# binary=True artinya tidak mempertimbangkan frekuensi\n",
    "vectorizerTP = CountVectorizer(binary=True)\n",
    "X = vectorizerTP.fit_transform(Corpus['text_final'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "afa3a060-3dda-46f1-b7ce-7984a563a89c",
   "metadata": {},
   "outputs": [],
   "source": [
    "# # print 16 nama fitur pertama dan terakhir\n",
    "# print(vectorizerTP.get_feature_names()[:16],'...',\n",
    "#       vectorizerTP.get_feature_names()[-16:])\n",
    "# # print 16 vektor term presence pertama dan terakhir untuk 6 baris/kalimat\n",
    "# with np.printoptions(edgeitems=16):\n",
    "#     print(X.toarray()[:6])\n",
    "\n",
    "# # print(X.shape, type(X))\n",
    "# # # Jika kita ingin melihat vektor dari suatu kata:\n",
    "# # print('Vector abai: ')\n",
    "# # with np.printoptions(edgeitems=10):\n",
    "# #     print(X.transform(['abai']).toarray())\n",
    "\n",
    "# # print(vectorizer.vocabulary_)\n",
    "# import reprlib\n",
    "# print(reprlib.repr(vectorizerTP.vocabulary_))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "e3bd9795-e486-4cab-bd9c-008119df5a24",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Transform Train_X dan Test_X ke vektor term presence\n",
    "Train_X_TP = vectorizerTP.transform(Train_X)\n",
    "Test_X_TP = vectorizerTP.transform(Test_X)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "4ce434db-0764-4c88-8b26-13261ffc839a",
   "metadata": {},
   "outputs": [],
   "source": [
    "# print(Train_X_TP)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "67fad48c-410e-4220-a18b-edfb5885b403",
   "metadata": {},
   "source": [
    "### KLASIFIKASI dengan *term presence*"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "03a9f973-9a12-4788-97c9-4b83e0a624a7",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SVM Accuracy Score ->  60.44176706827309\n"
     ]
    }
   ],
   "source": [
    "# fitting data latih pada classifier\n",
    "clf.fit(Train_X_TP,Train_Y)\n",
    "# memprediksi label pada set data uji\n",
    "predictions_SVM_TP = clf.predict(Test_X_TP)\n",
    "\n",
    "# Menggunakan fungsi accuracy_score untuk mendapat nilai akurasi\n",
    "accuracy_tp = accuracy_score(Test_Y_Actual, predictions_SVM_TP)*100\n",
    "print('SVM Accuracy Score -> ', accuracy_tp)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "ad4020a7-dcda-413c-a4af-57fe1fe40ec6",
   "metadata": {},
   "outputs": [],
   "source": [
    "# # Membandingkan Nilai Leksikon dengan Nilai Prediksi\n",
    "# df = pd.DataFrame({'Lexicon Values':Test_Y, 'Predicted Values':predictions_SVM_TP})\n",
    "# df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "c82127e7-4b83-4f75-b7e9-64b5e36b8368",
   "metadata": {},
   "outputs": [],
   "source": [
    "# print(predictions_SVM_TP)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b6c016a6-490b-4a82-9083-ec0670261043",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "id": "21751408-2796-436f-981f-2c7249f5d9be",
   "metadata": {},
   "source": [
    "# EKSTRAKSI FITUR: *BoW*"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "8b06a28f-00b9-4a75-856b-893e397b0284",
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.feature_extraction.text import CountVectorizer\n",
    "\n",
    "vectorizer = CountVectorizer()\n",
    "X = vectorizer.fit_transform(Corpus['text_final'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "58f45548-ddba-496a-aa63-fd895eba7d6e",
   "metadata": {},
   "outputs": [],
   "source": [
    "# # print 16 nama fitur pertama dan terakhir\n",
    "# print(vectorizer.get_feature_names()[:16],'...',\n",
    "#       vectorizer.get_feature_names()[-16:])\n",
    "# # print 16 vektor BoW pertama dan terakhir untuk 6 baris/kalimat\n",
    "# with np.printoptions(edgeitems=16):\n",
    "#     print(X.toarray()[:6])\n",
    "\n",
    "# print(X.shape, type(X))\n",
    "# # # Jika kita ingin melihat vektor dari suatu kata:\n",
    "# # print('Vector abai: ')\n",
    "# # with np.printoptions(edgeitems=10):\n",
    "# #     print(X.transform(['abai']).toarray())\n",
    "\n",
    "# # print(vectorizer.vocabulary_)\n",
    "# import reprlib\n",
    "# print(reprlib.repr(vectorizer.vocabulary_))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "92b392c3-575f-416c-82bb-499751ea9694",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Transform Train_X dan Test_X ke vektor BoW\n",
    "Train_X_BoW = vectorizer.transform(Train_X)\n",
    "Test_X_BoW = vectorizer.transform(Test_X)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "5ae078a5-07b0-4ce4-b597-5647ca33a85f",
   "metadata": {},
   "outputs": [],
   "source": [
    "# print(Train_X_BoW)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a2ef7d11-9e9f-4644-9f84-6f5e18aa5e2f",
   "metadata": {},
   "source": [
    "### KLASIFIKASI dengan *BoW*"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "cb9b996f-a3e5-46cf-bbcb-8951d20368c5",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SVM Accuracy Score ->  59.63855421686747\n"
     ]
    }
   ],
   "source": [
    "# fitting data latih pada classifier\n",
    "clf.fit(Train_X_BoW,Train_Y)\n",
    "# memprediksi label pada set data uji\n",
    "predictions_SVM_BoW = clf.predict(Test_X_BoW)\n",
    "\n",
    "# Menggunakan fungsi accuracy_score untuk mendapat nilai akurasi\n",
    "accuracy_bow = accuracy_score(Test_Y_Actual, predictions_SVM_BoW)*100\n",
    "print('SVM Accuracy Score -> ',accuracy_bow)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "fdf238c8-5024-4019-8df8-c7232adff1aa",
   "metadata": {},
   "outputs": [],
   "source": [
    "# print(predictions_SVM_BoW)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "227107ef-0475-40c0-aa3c-926954e5f528",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "id": "8d6dc1c9-76d2-47d4-9f95-c4be9b3ad93b",
   "metadata": {},
   "source": [
    "# EKSTRAKSI FITUR: *TF-IDF*"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "id": "0dff5723-a40a-4e0d-ae4f-5b5292be2de6",
   "metadata": {},
   "outputs": [],
   "source": [
    "Tfidf_vect = TfidfVectorizer()\n",
    "Tfidf_vect.fit(Corpus['text_final'])\n",
    "\n",
    "X = Tfidf_vect.fit_transform(Corpus['text_final'])"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a5cdd665-7c8b-450c-9aa2-245c63cb0f9f",
   "metadata": {},
   "source": [
    "#### opsi lain yang bisa dipertimbangkan:\n",
    "\n",
    "<code> Tfidf_vect = TfidfVectorizer(max_features=None).fit(Corpus['text_final']) <code>\n",
    "<code> Tfidf_vect = TfidfVectorizer(max_features=5000).fit(Corpus['text_final']) <code>\n",
    "<code> Tfidf_vect = TfidfVectorizer(min_df=5, max_df=0.8, sublinear_tf=True, \\\n",
    "                    use_idf=True).fit(Corpus['text_final']) <code>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "id": "2b4fbe6d-9ced-428f-b0bd-1170cc6f1acc",
   "metadata": {},
   "outputs": [],
   "source": [
    "# # print 16 nama fitur pertama dan terakhir\n",
    "# print(Tfidf_vect.get_feature_names()[:16],'\\n',\n",
    "#       Tfidf_vect.get_feature_names()[-16:])\n",
    "# # print 16 vektor TF-IDF pertama dan terakhir untuk 6 baris/kalimat\n",
    "# with np.printoptions(edgeitems=16):\n",
    "#     print(X.toarray()[:6])\n",
    "\n",
    "# # print(Tfidf_vect.vocabulary_)\n",
    "# import reprlib\n",
    "# print(reprlib.repr(Tfidf_vect.vocabulary_))\n",
    "\n",
    "# # # Jika kita ingin melihat vektor dari suatu kata:\n",
    "# # # sebagai contoh kata di array[22]:\n",
    "# # val = list(Tfidf_vect.vocabulary_)[22]\n",
    "# # print(val)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "id": "2e5a1a22-b140-4afb-8b77-549da4c8ae6a",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Transform Train_X dan Test_X ke vektor TF-IDF\n",
    "Train_X_Tfidf = Tfidf_vect.transform(Train_X)\n",
    "Test_X_Tfidf = Tfidf_vect.transform(Test_X)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "id": "092afe8a-34ba-4e16-bb99-fff11746a528",
   "metadata": {},
   "outputs": [],
   "source": [
    "# print(Train_X_Tfidf)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "c411af4b-ea13-43e3-8bee-c7d1001bf456",
   "metadata": {},
   "source": [
    "### KLASIFIKASI dengan TF-IDF"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "id": "71b3db30-2956-4591-b231-484f185b4f91",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SVM Accuracy Score ->  62.65060240963856\n"
     ]
    }
   ],
   "source": [
    "# fitting data latih pada classifier\n",
    "clf.fit(Train_X_Tfidf,Train_Y)\n",
    "# memprediksi label pada set data uji\n",
    "predictions_SVM_Tfidf = clf.predict(Test_X_Tfidf)\n",
    "\n",
    "# Menggunakan fungsi accuracy_score untuk mendapat nilai akurasi\n",
    "accuracy_tfidf = accuracy_score(Test_Y_Actual, predictions_SVM_Tfidf)*100\n",
    "print('SVM Accuracy Score -> ',accuracy_tfidf)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "id": "695dadaa-dfe6-420c-9c6a-e4a10e7a2a01",
   "metadata": {},
   "outputs": [],
   "source": [
    "# # Simpan skor akurasi ke file\n",
    "# if LLmark == 1:\n",
    "#     output = 'svm_acc_lb1.txt'\n",
    "#     with open(output, 'w') as f:\n",
    "#         f.write(str(accuracy_tp)+str('\\n')+str(accuracy_bow)+str('\\n')+str(accuracy_tfidf))\n",
    "# elif LLmark == 2:\n",
    "#     output = 'svm_acc_lb2.txt'\n",
    "#     with open(output, 'w') as f:\n",
    "#         f.write(str(accuracy_tp)+str('\\n')+str(accuracy_bow)+str('\\n')+str(accuracy_tfidf))\n",
    "# else:\n",
    "#     output = 'svm_acc_lb0.txt'\n",
    "#     with open(output, 'w') as f:\n",
    "#         f.write(str(accuracy_tp)+str('\\n')+str(accuracy_bow)+str('\\n')+str(accuracy_tfidf))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "id": "1a1695c5-f8b8-427c-83f5-8ee79a3d785e",
   "metadata": {},
   "outputs": [],
   "source": [
    "# print(predictions_SVM_Tfidf)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f0670adb-83f7-41c1-b70d-1b59df7a8f20",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "id": "1e50e11c-0d07-4fe9-9db5-58b6922ad87a",
   "metadata": {},
   "source": [
    "#### **Gambaran Data:**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "id": "a342e1b8-75b2-4a89-a34e-0b01a2de8e9d",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "X: 1660 \n",
      "Train\tTest\t\t%train\t%test\n",
      " 1162 \t 498 \tX\t 70.00 \t 30.00 \n",
      " 19529 \t 8560 \tTP\t 69.53 \t 30.47 \n",
      " 19529 \t 8560 \tBoWs\t 69.53 \t 30.47 \n",
      " 19529 \t 8560 \tTF-IDF\t 69.53 \t 30.47\n",
      "\n",
      "neg\tpos\t\tsum\n",
      " 301 \t 197 \t 498 \tTest_Y_Actual\n",
      " 347 \t 151 \t 498 \tTest_Y\n",
      " 394 \t 104 \t 498 \tP_TP\n",
      " 384 \t 114 \t 498 \tP_BoWs\n",
      " 449 \t 49 \t 498 \tP_Tfidf\n"
     ]
    }
   ],
   "source": [
    "# Gambaran rasio data latih dan data uji dari teks awal dan dari vektor ekstraksi fitur.\n",
    "print('X:', Corpus['text'].size,\n",
    "      '\\nTrain\\tTest\\t\\t%train\\t%test\\n',\n",
    "#       Train_Y.size, '\\t', Test_Y.size, '\\ty\\t', '{:.2f}'.format(Train_Y.size/(Test_Y.size+Train_Y.size)*100), '\\t', '{:.2f}'.format(Test_Y.size/(Test_Y.size+Train_Y.size)*100), '\\n',\n",
    "      Train_X.size, '\\t', Test_X.size, '\\tX\\t', '{:.2f}'.format(Train_X.size/(Test_X.size+Train_X.size)*100), '\\t', '{:.2f}'.format(Test_X.size/(Test_X.size+Train_X.size)*100), '\\n',\n",
    "      Train_X_TP.size, '\\t', Test_X_TP.size, '\\tTP\\t', '{:.2f}'.format(Train_X_TP.size/(Test_X_TP.size+Train_X_TP.size)*100), '\\t', '{:.2f}'.format(Test_X_TP.size/(Test_X_TP.size+Train_X_TP.size)*100), '\\n',\n",
    "      Train_X_BoW.size, '\\t', Test_X_BoW.size, '\\tBoWs\\t', '{:.2f}'.format(Train_X_BoW.size/(Test_X_BoW.size+Train_X_BoW.size)*100), '\\t', '{:.2f}'.format(Test_X_BoW.size/(Test_X_BoW.size+Train_X_BoW.size)*100), '\\n',\n",
    "      Train_X_Tfidf.size, '\\t', Test_X_Tfidf.size, '\\tTF-IDF\\t', '{:.2f}'.format(Train_X_Tfidf.size/(Test_X_Tfidf.size+Train_X_Tfidf.size)*100), '\\t', '{:.2f}'.format(Test_X_Tfidf.size/(Test_X_Tfidf.size+Train_X_Tfidf.size)*100))\n",
    "\n",
    "# Gambaran jumlah kelas negatif dan positif dari data uji dengan label aktual, label leksikon, and label prediksi;\n",
    "# Label aktual dan label leksikon disimbolkan sebagai 'Test_Y_Actual' dan 'Test_Y';\n",
    "# Jika kamu memakai label aktual sebagai baseline, maka label leksikon di sini adalah label aktual itu sendiri.\n",
    "print('\\nneg\\tpos\\t\\tsum\\n',\n",
    "      (Test_Y_Actual==0).sum(), '\\t', (Test_Y_Actual==1).sum(), '\\t', Test_Y_Actual.size, '\\tTest_Y_Actual\\n',\n",
    "      (Test_Y==0).sum(), '\\t', (Test_Y==1).sum(), '\\t', Test_Y.size, '\\tTest_Y\\n',\n",
    "      (predictions_SVM_TP==0).sum(),    '\\t', (predictions_SVM_TP==1).sum(),    '\\t', predictions_SVM_TP.size,    '\\tP_TP\\n',\n",
    "      (predictions_SVM_BoW==0).sum(),   '\\t', (predictions_SVM_BoW==1).sum(),   '\\t', predictions_SVM_BoW.size,   '\\tP_BoWs\\n',\n",
    "      (predictions_SVM_Tfidf==0).sum(), '\\t', (predictions_SVM_Tfidf==1).sum(), '\\t', predictions_SVM_Tfidf.size, '\\tP_Tfidf')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4fb42497-8e1b-4fbe-bab5-24b85acb76fa",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "id": "bbe1c3b1-88ea-482a-8165-4d59db527a15",
   "metadata": {},
   "source": [
    "# EVALUASI / VALIDASI "
   ]
  },
  {
   "cell_type": "markdown",
   "id": "2bece049-c474-4e70-8e99-adf9ec735c4a",
   "metadata": {},
   "source": [
    "## Confusion Matrix"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "id": "de5ff636-ff35-42be-a397-5e7ed619e22d",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Confusion Matrix - Term presence\n",
      "[[ 52 145]\n",
      " [ 52 249]] 498\n",
      "\n",
      "Confusion Matrix - BoW\n",
      "[[ 55 142]\n",
      " [ 59 242]] 498\n",
      "\n",
      "Confusion Matrix - TF-IDF\n",
      "[[ 30 167]\n",
      " [ 19 282]] 498\n"
     ]
    }
   ],
   "source": [
    "# Membuat confusion matrix dari label prediksi terhadap label aktual\n",
    "from sklearn.metrics import confusion_matrix\n",
    "\n",
    "\n",
    "y_true = Test_Y_Actual\n",
    "\n",
    "## Term presence ##\n",
    "print('Confusion Matrix - Term presence')\n",
    "y_pred = predictions_SVM_TP\n",
    "conf_matrix = confusion_matrix(y_true, y_pred, labels=[1,0])\n",
    "print(conf_matrix, conf_matrix.sum())\n",
    "\n",
    "## BoW ##\n",
    "print('\\nConfusion Matrix - BoW')\n",
    "y_pred = predictions_SVM_BoW\n",
    "conf_matrix = confusion_matrix(y_true, y_pred, labels=[1,0])\n",
    "print(conf_matrix, conf_matrix.sum())\n",
    "\n",
    "## TF-IDF ##\n",
    "print('\\nConfusion Matrix - TF-IDF')\n",
    "y_pred = predictions_SVM_Tfidf\n",
    "conf_matrix = confusion_matrix(y_true, y_pred, labels=[1,0])\n",
    "print(conf_matrix, conf_matrix.sum())"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "9b542e72-8535-48c8-b47f-7e2c09b3d885",
   "metadata": {},
   "source": [
    "## Classification Report: dengan *imbalanced data*\n",
    "Karena kelas kata (positif & negatif) di semua jenis label—*actual label*, *label by InSet*, maupun *label by sentiwords_id*—tidak terdistribusi secara berimbang, data kita memuat *imbalanced class*. Ini bisa menyebabkan misklasifikasi pada model yang kita buat, menyebabkan penilaian yang tidak akurat. Kita akan coba melatih data *imbalance* ini dan mengevaluasinya nanti."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "id": "438d9269-ab45-4f76-bf2f-53dc0c2032b5",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Imbalanced data - Term presence\n",
      "               precision    recall  f1-score   support\n",
      "\n",
      "           0       0.63      0.83      0.72       301\n",
      "           1       0.50      0.26      0.35       197\n",
      "\n",
      "    accuracy                           0.60       498\n",
      "   macro avg       0.57      0.55      0.53       498\n",
      "weighted avg       0.58      0.60      0.57       498\n",
      "\n",
      "Imbalanced data - BoW\n",
      "               precision    recall  f1-score   support\n",
      "\n",
      "           0       0.63      0.80      0.71       301\n",
      "           1       0.48      0.28      0.35       197\n",
      "\n",
      "    accuracy                           0.60       498\n",
      "   macro avg       0.56      0.54      0.53       498\n",
      "weighted avg       0.57      0.60      0.57       498\n",
      "\n",
      "Imbalanced data - TF-IDF\n",
      "               precision    recall  f1-score   support\n",
      "\n",
      "           0       0.63      0.94      0.75       301\n",
      "           1       0.61      0.15      0.24       197\n",
      "\n",
      "    accuracy                           0.63       498\n",
      "   macro avg       0.62      0.54      0.50       498\n",
      "weighted avg       0.62      0.63      0.55       498\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# Membuat classification report dengan imbalanced data\n",
    "from sklearn.metrics import classification_report\n",
    "\n",
    "\n",
    "## Term presence ##\n",
    "print('Imbalanced data - Term presence\\n',\n",
    "      classification_report(Test_Y_Actual, predictions_SVM_TP))\n",
    "## BoW ##\n",
    "print('Imbalanced data - BoW\\n',\n",
    "      classification_report(Test_Y_Actual, predictions_SVM_BoW))\n",
    "## TF-IDF ##\n",
    "print('Imbalanced data - TF-IDF\\n',\n",
    "      classification_report(Test_Y_Actual, predictions_SVM_Tfidf))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "1ae0d080-b65f-479f-85ba-4f56ad4a8b6b",
   "metadata": {},
   "source": [
    "#### **\\*catatan:**\n",
    "Kode di bawah memiliki fungsi yang sama dengan kode di atas. Kode ini sengaja tetap disimpan untuk menunjukkan proses asli yang perlu dikerjakan jika kita tidak membuat variabel tersendiri dari hasil prediksi."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "id": "7e958a87-3219-40ab-86b5-8046b1861170",
   "metadata": {},
   "outputs": [],
   "source": [
    "# # Make classification report using 'imbalanced' data\n",
    "# from sklearn.metrics import classification_report\n",
    "\n",
    "\n",
    "# ## Term presence ##\n",
    "# X_train = Train_X_TP\n",
    "# X_test = Test_X_TP\n",
    "# clf.fit(X_train, Train_Y)\n",
    "# print('Imbalanced data - Term presence\\n',\n",
    "#       classification_report(Test_Y_Actual, clf.predict(X_test)))\n",
    "\n",
    "# ## BoW ##\n",
    "# X_train = Train_X_BoW\n",
    "# X_test = Test_X_BoW\n",
    "# clf.fit(X_train, Train_Y)\n",
    "# print('Imbalanced data - BoW\\n',\n",
    "#       classification_report(Test_Y_Actual, clf.predict(X_test)))\n",
    "\n",
    "# ## TF-IDF ##\n",
    "# X_train = Train_X_Tfidf\n",
    "# X_test = Test_X_Tfidf\n",
    "# clf.fit(X_train, Train_Y)\n",
    "# print('Imbalanced data - TF-IDF\\n',\n",
    "#       classification_report(Test_Y_Actual, clf.predict(X_test)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "24174bb7-ed97-4ca3-93ee-331839bda8cf",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "id": "43a2ee9e-821a-48dc-9b7f-fff645e7e9ec",
   "metadata": {},
   "source": [
    "## Classification Report: dengan *oversampled data*\n",
    "Di sini, kita akan melatih *imbalanced data* kita dengan metode **oversampling** dan mengevaluasinya."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "id": "fe418f6b-f0ed-4c45-bf44-70dcb66be970",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Fungsi untuk menyimpan skor akurasi ke file\n",
    "def acc_oversampled(LLmark, accuracy):\n",
    "    if LLmark == 1:\n",
    "        output = 'svm_acc_o_lb1.txt'\n",
    "        with open(output, 'a') as f:\n",
    "            f.write(str(accuracy)+str('\\n'))\n",
    "    elif LLmark == 2:\n",
    "        output = 'svm_acc_o_lb2.txt'\n",
    "        with open(output, 'a') as f:\n",
    "            f.write(str(accuracy)+str('\\n'))\n",
    "    else:\n",
    "        output = 'svm_acc_o_lb0.txt'\n",
    "        with open(output, 'a') as f:\n",
    "            f.write(str(accuracy)+str('\\n'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "id": "b47d4928-9078-4037-b35d-4315acca87c8",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Oversampled data - Term presence\n",
      "               precision    recall  f1-score   support\n",
      "\n",
      "           0       0.62      0.71      0.66       301\n",
      "           1       0.43      0.32      0.37       197\n",
      "\n",
      "    accuracy                           0.56       498\n",
      "   macro avg       0.52      0.52      0.52       498\n",
      "weighted avg       0.54      0.56      0.55       498\n",
      "\n",
      "Oversampled data - BoW\n",
      "               precision    recall  f1-score   support\n",
      "\n",
      "           0       0.62      0.70      0.66       301\n",
      "           1       0.42      0.33      0.37       197\n",
      "\n",
      "    accuracy                           0.55       498\n",
      "   macro avg       0.52      0.52      0.51       498\n",
      "weighted avg       0.54      0.55      0.54       498\n",
      "\n",
      "Oversampled data - TF-IDF\n",
      "               precision    recall  f1-score   support\n",
      "\n",
      "           0       0.63      0.88      0.74       301\n",
      "           1       0.55      0.22      0.31       197\n",
      "\n",
      "    accuracy                           0.62       498\n",
      "   macro avg       0.59      0.55      0.53       498\n",
      "weighted avg       0.60      0.62      0.57       498\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# Membuat classification report dengan 'oversampled' data\n",
    "from imblearn.over_sampling import SVMSMOTE\n",
    "\n",
    "\n",
    "# svmsmote = SVMSMOTE(random_state=None)\n",
    "svmsmote = SVMSMOTE(random_state = 500)\n",
    "\n",
    "y_train = Train_Y\n",
    "y_test = Test_Y_Actual\n",
    "\n",
    "## Term presence ##\n",
    "X_train = Train_X_TP\n",
    "X_test = Test_X_TP\n",
    "X_oversample_svm, y_oversample_svm = svmsmote.fit_resample(X_train, y_train)\n",
    "# melatih classifier dengan oversampled data menggunakan borderline-SMOTE SVM (SVM SMOTE)\n",
    "clf.fit(X_oversample_svm, y_oversample_svm)\n",
    "# acc_oversampled(LLmark, accuracy_score(y_test, clf.predict(X_test))*100) # print to file\n",
    "print('Oversampled data - Term presence\\n', classification_report(y_test, clf.predict(X_test)))\n",
    "\n",
    "## BoW ##\n",
    "X_train = Train_X_BoW\n",
    "X_test = Test_X_BoW\n",
    "X_oversample_svm, y_oversample_svm = svmsmote.fit_resample(X_train, y_train)\n",
    "# melatih classifier dengan oversampled data menggunakan borderline-SMOTE SVM (SVM SMOTE)\n",
    "clf.fit(X_oversample_svm, y_oversample_svm)\n",
    "# acc_oversampled(LLmark, accuracy_score(y_test, clf.predict(X_test))*100) # print to file\n",
    "print('Oversampled data - BoW\\n', classification_report(y_test, clf.predict(X_test)))\n",
    "\n",
    "## TF-IDF ##\n",
    "X_train = Train_X_Tfidf\n",
    "X_test = Test_X_Tfidf\n",
    "X_oversample_svm, y_oversample_svm = svmsmote.fit_resample(X_train, y_train)\n",
    "# melatih classifier dengan oversampled data menggunakan borderline-SMOTE SVM (SVM SMOTE)\n",
    "clf.fit(X_oversample_svm, y_oversample_svm)\n",
    "# acc_oversampled(LLmark, accuracy_score(y_test, clf.predict(X_test))*100) # print to file\n",
    "print('Oversampled data - TF-IDF\\n', classification_report(y_test, clf.predict(X_test)))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "06f8d166-1736-41fc-8b4a-30a6b731d851",
   "metadata": {},
   "source": [
    "<blockquote><i>\"The purpose of oversampling is ... to have a better prediction model. This technique was not created for any analysis purposes as every data created is synthetic, so that is a reminder.\"</i></blockquote>\n",
    "\n",
    "<blockquote><i>\"... <b>you should only oversample your training data and not the whole data</b> except if you would use the entire data as your training data. <b>In case you want to split the data, you should split the data first</b> before oversampled the training data.\"</i></blockquote>\n",
    "\n",
    "<blockquote>Ref: <i>https://towardsdatascience.com/5-smote-techniques-for-oversampling-your-imbalance-data-b8155bdbe2b5?gi=67231aa6fa80</i></blockquote>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d17ea5f5-0c69-4bbd-a2e7-22f8732c8eaa",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "id": "1dbc7278-03d9-42a1-9c47-22717c7b57fb",
   "metadata": {},
   "source": [
    "## Validasi dengan k-Fold cv\n",
    "Ubah nilai `n_splits` sesuai dengan kebutuhan. Sebagai contoh, 5 splits berarti bahwa data (X dan y oversample) dibagi menjadi *4 porsi* untuk set latih baru dan *1 porsi* untuk set uji baru. Jika `shuffle` bernilai True, maka validasi silang dilakukan dengan kombinasi data yang berbeda di setiap iterasi. Kemudian validasi silang dilakukan dalam 5 iterasi."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "id": "a6bff046-7be3-4c43-bdd1-d1306aa95b03",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Simpan skor akurasi ke file\n",
    "def acc_oversampled(LLmark, featExt, accuracy):\n",
    "    if LLmark == 1:\n",
    "        output = 'svm_acc_ov_lb1_'+str(featExt)+'_kfold.txt'\n",
    "        with open(output, 'a') as f:\n",
    "            f.write(str(accuracy)+'\\n')\n",
    "    elif LLmark == 2:\n",
    "        output = 'svm_acc_ov_lb2_'+str(featExt)+'_kfold.txt'\n",
    "        with open(output, 'a') as f:\n",
    "            f.write(str(accuracy)+'\\n')\n",
    "    else:\n",
    "        output = 'svm_acc_ov_lb0_'+str(featExt)+'_kfold.txt'\n",
    "        with open(output, 'a') as f:\n",
    "            f.write(str(accuracy)+'\\n')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "id": "0b061993-ac3e-4059-9544-dbfef46ee8b3",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Simpan nilai presisi, recall, dan f1-score ke file\n",
    "def cr_oversampled(LLmark, featExt, precision, recall, f1):\n",
    "    if LLmark == 1:\n",
    "        output = 'svm_cr_ov_lb1_'+str(featExt)+'_kfold.txt'\n",
    "        with open(output, 'a') as f:\n",
    "            f.write(str(precision)+'\\t'+str(recall)+'\\t'+str(f1)+'\\n')\n",
    "    elif LLmark == 2:\n",
    "        output = 'svm_cr_ov_lb2_'+str(featExt)+'_kfold.txt'\n",
    "        with open(output, 'a') as f:\n",
    "            f.write(str(precision)+'\\t'+str(recall)+'\\t'+str(f1)+'\\n')\n",
    "    else:\n",
    "        output = 'svm_cr_ov_lb0_'+str(featExt)+'_kfold.txt'\n",
    "        with open(output, 'a') as f:\n",
    "            f.write(str(precision)+'\\t'+str(recall)+'\\t'+str(f1)+'\\n')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "id": "5ef7bf11-416f-4ef5-ba4e-231c474944da",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "d:\\apps\\scoop\\apps\\python\\3.8.5\\lib\\site-packages\\sklearn\\utils\\validation.py:63: DataConversionWarning: A column-vector y was passed when a 1d array was expected. Please change the shape of y to (n_samples, ), for example using ravel().\n",
      "  return f(*args, **kwargs)\n"
     ]
    }
   ],
   "source": [
    "from sklearn.metrics import f1_score, precision_score, recall_score\n",
    "from sklearn.model_selection import KFold\n",
    "\n",
    "outer_cv = KFold(n_splits=5, shuffle=True, random_state=42)\n",
    "\n",
    "y_true = Corpus[['label']]\n",
    "y_true = Encoder.fit_transform(y_true)\n",
    "y = LL\n",
    "y = Encoder.fit_transform(y)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "8a8482a6-4442-4733-8d97-2777e9cb747e",
   "metadata": {},
   "source": [
    "#### **Step 1:** Validasi dengan `term presence` sebagai metode ekstraksi fitur. Lalu, meng-**oversample** model di setiap iterasi k-Fold dengan `SVM SMOTE`"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "id": "f84820d0-9540-4cd5-ae7b-e060c13fd451",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "# Fold 1:\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.60      0.67      0.63       204\n",
      "           1       0.36      0.30      0.33       128\n",
      "\n",
      "    accuracy                           0.53       332\n",
      "   macro avg       0.48      0.49      0.48       332\n",
      "weighted avg       0.51      0.53      0.52       332\n",
      " \n",
      "\n",
      "# Fold 2:\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.57      0.73      0.64       177\n",
      "           1       0.54      0.36      0.43       155\n",
      "\n",
      "    accuracy                           0.56       332\n",
      "   macro avg       0.56      0.55      0.54       332\n",
      "weighted avg       0.56      0.56      0.54       332\n",
      " \n",
      "\n",
      "# Fold 3:\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.58      0.70      0.64       178\n",
      "           1       0.55      0.42      0.48       154\n",
      "\n",
      "    accuracy                           0.57       332\n",
      "   macro avg       0.57      0.56      0.56       332\n",
      "weighted avg       0.57      0.57      0.56       332\n",
      " \n",
      "\n",
      "# Fold 4:\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.52      0.67      0.59       165\n",
      "           1       0.55      0.40      0.46       167\n",
      "\n",
      "    accuracy                           0.53       332\n",
      "   macro avg       0.54      0.53      0.53       332\n",
      "weighted avg       0.54      0.53      0.52       332\n",
      " \n",
      "\n",
      "# Fold 5:\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.53      0.66      0.59       176\n",
      "           1       0.48      0.35      0.41       156\n",
      "\n",
      "    accuracy                           0.52       332\n",
      "   macro avg       0.51      0.51      0.50       332\n",
      "weighted avg       0.51      0.52      0.50       332\n",
      " \n",
      "\n"
     ]
    }
   ],
   "source": [
    "featExt = 'tp'\n",
    "\n",
    "X = vectorizerTP.fit_transform(Corpus['text_final'])\n",
    "kf = KFold(n_splits=5, shuffle=True, random_state=42)\n",
    "\n",
    "for fold, (train_index, test_index) in enumerate(kf.split(X), 1):\n",
    "    X_train, X_test = X[train_index], X[test_index]\n",
    "    y_train, y_test = y[train_index], y[test_index]\n",
    "    y_true_train, y_true_test = y_true[train_index], y_true[test_index]\n",
    "    X_train_oversampled, y_train_oversampled = svmsmote.fit_resample(X_train, y_train.ravel())\n",
    "    \n",
    "    clf.fit(X_train_oversampled, y_train_oversampled)\n",
    "    y_pred = clf.predict(X_test)\n",
    "    \n",
    "    accuracy = accuracy_score(y_true_test, y_pred)\n",
    "    precision = precision_score(y_true_test, y_pred)\n",
    "    recall = recall_score(y_true_test, y_pred)\n",
    "    f1 = f1_score(y_true_test, y_pred)\n",
    "    \n",
    "    #print to file\n",
    "#     acc_oversampled(LLmark, featExt, accuracy)\n",
    "#     cr_oversampled(LLmark, featExt, precision, recall, f1)\n",
    "\n",
    "    print(f'# Fold {fold}:')\n",
    "    print(classification_report(y_true_test, y_pred), \"\\n\")\n",
    "#     print(f'accuracy: {accuracy}')\n",
    "#     print(f'precision: {precision}')\n",
    "#     print(f'recall: {recall}')\n",
    "#     print(f'f-score: {f1}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "dea62f3d-1c2f-436d-9444-ec7684866a84",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "id": "5f3023e1-5654-4909-aaaa-ebfe7eac677f",
   "metadata": {},
   "source": [
    "#### **Step 2:** Validasi dengan `BoW` sebagai metode ekstraksi fitur. Lalu, meng-**oversample** model di setiap iterasi k-Fold dengan `SVM SMOTE`"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "id": "bb33a2f0-04a5-4a80-84e1-961b2edfb843",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "# Fold 1:\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.61      0.67      0.64       204\n",
      "           1       0.39      0.33      0.35       128\n",
      "\n",
      "    accuracy                           0.54       332\n",
      "   macro avg       0.50      0.50      0.50       332\n",
      "weighted avg       0.53      0.54      0.53       332\n",
      " \n",
      "\n",
      "# Fold 2:\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.55      0.69      0.62       177\n",
      "           1       0.50      0.35      0.42       155\n",
      "\n",
      "    accuracy                           0.54       332\n",
      "   macro avg       0.53      0.52      0.52       332\n",
      "weighted avg       0.53      0.54      0.52       332\n",
      " \n",
      "\n",
      "# Fold 3:\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.57      0.69      0.62       178\n",
      "           1       0.53      0.40      0.46       154\n",
      "\n",
      "    accuracy                           0.55       332\n",
      "   macro avg       0.55      0.54      0.54       332\n",
      "weighted avg       0.55      0.55      0.55       332\n",
      " \n",
      "\n",
      "# Fold 4:\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.55      0.72      0.62       165\n",
      "           1       0.59      0.41      0.49       167\n",
      "\n",
      "    accuracy                           0.56       332\n",
      "   macro avg       0.57      0.56      0.55       332\n",
      "weighted avg       0.57      0.56      0.55       332\n",
      " \n",
      "\n",
      "# Fold 5:\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.53      0.66      0.59       176\n",
      "           1       0.47      0.34      0.39       156\n",
      "\n",
      "    accuracy                           0.51       332\n",
      "   macro avg       0.50      0.50      0.49       332\n",
      "weighted avg       0.50      0.51      0.50       332\n",
      " \n",
      "\n"
     ]
    }
   ],
   "source": [
    "featExt = 'bow'\n",
    "\n",
    "X = vectorizer.fit_transform(Corpus['text_final'])\n",
    "kf = KFold(n_splits=5, shuffle=True, random_state=42)\n",
    "\n",
    "for fold, (train_index, test_index) in enumerate(kf.split(X), 1):\n",
    "    X_train, X_test = X[train_index], X[test_index]\n",
    "    y_train, y_test = y[train_index], y[test_index]\n",
    "    y_true_train, y_true_test = y_true[train_index], y_true[test_index]\n",
    "    X_train_oversampled, y_train_oversampled = svmsmote.fit_resample(X_train, y_train.ravel())\n",
    "    \n",
    "    clf.fit(X_train_oversampled, y_train_oversampled)\n",
    "    y_pred = clf.predict(X_test)\n",
    "    \n",
    "    accuracy = accuracy_score(y_true_test, y_pred)\n",
    "    precision = precision_score(y_true_test, y_pred)\n",
    "    recall = recall_score(y_true_test, y_pred)\n",
    "    f1 = f1_score(y_true_test, y_pred)\n",
    "\n",
    "    #print to file\n",
    "#     acc_oversampled(LLmark, featExt, accuracy)\n",
    "#     cr_oversampled(LLmark, featExt, precision, recall, f1)\n",
    "\n",
    "    print(f'# Fold {fold}:')\n",
    "    print(classification_report(y_true_test, y_pred), \"\\n\")\n",
    "#     print(f'accuracy: {accuracy}')\n",
    "#     print(f'precision: {precision}')\n",
    "#     print(f'recall: {recall}')\n",
    "#     print(f'f-score: {f1}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a3f31ca0-6429-4cb8-bea7-26579aec4e33",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "id": "2283adfa-3b58-46a0-ab1a-b9e76cb64b7b",
   "metadata": {},
   "source": [
    "#### **Step 3:** Validasi dengan `TF-IDF` sebagai metode ekstraksi fitur. Lalu, meng-**oversample** model di setiap iterasi k-Fold dengan `SVM SMOTE`"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "id": "e1a7971f-d48d-4365-90cb-9023e8fa77c4",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "# Fold 1:\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.64      0.84      0.72       204\n",
      "           1       0.48      0.23      0.31       128\n",
      "\n",
      "    accuracy                           0.61       332\n",
      "   macro avg       0.56      0.54      0.52       332\n",
      "weighted avg       0.57      0.61      0.57       332\n",
      " \n",
      "\n",
      "# Fold 2:\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.57      0.89      0.69       177\n",
      "           1       0.64      0.22      0.33       155\n",
      "\n",
      "    accuracy                           0.58       332\n",
      "   macro avg       0.60      0.56      0.51       332\n",
      "weighted avg       0.60      0.58      0.52       332\n",
      " \n",
      "\n",
      "# Fold 3:\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.59      0.87      0.70       178\n",
      "           1       0.67      0.30      0.41       154\n",
      "\n",
      "    accuracy                           0.61       332\n",
      "   macro avg       0.63      0.58      0.56       332\n",
      "weighted avg       0.63      0.61      0.57       332\n",
      " \n",
      "\n",
      "# Fold 4:\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.54      0.87      0.67       165\n",
      "           1       0.68      0.28      0.39       167\n",
      "\n",
      "    accuracy                           0.57       332\n",
      "   macro avg       0.61      0.57      0.53       332\n",
      "weighted avg       0.61      0.57      0.53       332\n",
      " \n",
      "\n",
      "# Fold 5:\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.58      0.87      0.69       176\n",
      "           1       0.65      0.28      0.39       156\n",
      "\n",
      "    accuracy                           0.59       332\n",
      "   macro avg       0.61      0.57      0.54       332\n",
      "weighted avg       0.61      0.59      0.55       332\n",
      " \n",
      "\n"
     ]
    }
   ],
   "source": [
    "featExt = 'tfidf'\n",
    "\n",
    "X = Tfidf_vect.fit_transform(Corpus['text_final'])\n",
    "kf = KFold(n_splits=5, shuffle=True, random_state=42)\n",
    "\n",
    "for fold, (train_index, test_index) in enumerate(kf.split(X), 1):\n",
    "    X_train, X_test = X[train_index], X[test_index]\n",
    "    y_train, y_test = y[train_index], y[test_index]\n",
    "    y_true_train, y_true_test = y_true[train_index], y_true[test_index]\n",
    "    X_train_oversampled, y_train_oversampled = svmsmote.fit_resample(X_train, y_train.ravel())\n",
    "    \n",
    "    clf.fit(X_train_oversampled, y_train_oversampled)\n",
    "    y_pred = clf.predict(X_test)\n",
    "    \n",
    "    accuracy = accuracy_score(y_true_test, y_pred)\n",
    "    precision = precision_score(y_true_test, y_pred)\n",
    "    recall = recall_score(y_true_test, y_pred)\n",
    "    f1 = f1_score(y_true_test, y_pred)\n",
    "\n",
    "    #print to file\n",
    "#     acc_oversampled(LLmark, featExt, accuracy)\n",
    "#     cr_oversampled(LLmark, featExt, precision, recall, f1)\n",
    "\n",
    "    print(f'# Fold {fold}:')\n",
    "    print(classification_report(y_true_test, y_pred), \"\\n\")\n",
    "#     print(f'accuracy: {accuracy}')\n",
    "#     print(f'precision: {precision}')\n",
    "#     print(f'recall: {recall}')\n",
    "#     print(f'f-score: {f1}')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "10d43234-bf4a-4cd1-af35-d4be6409852b",
   "metadata": {},
   "source": [
    "<blockquote>Ref: <i>https://stackoverflow.com/questions/55591063/how-to-perform-smote-with-cross-validation-in-sklearn-in-python</i></blockquote>"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}