DL_Project_PreProcess.ipynb

{
  "cells": [
    {
      "cell_type": "code",
      "source": [
        "!git clone https://github.com/Arman-Rayan-Sharif/arman-text-emotion.git"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "OeaS86_YajA_",
        "outputId": "c40d31c9-d897-4316-82ac-222b3ffe2a27"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "fatal: destination path 'arman-text-emotion' already exists and is not an empty directory.\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "!pip install --quiet parsivar"
      ],
      "metadata": {
        "id": "3psMsK9eaesf"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "wfN48QA6aYso"
      },
      "outputs": [],
      "source": [
        "import csv\n",
        "import codecs\n",
        "import pandas as pd\n",
        "import seaborn as sns"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "A7rQQ0VMaYss"
      },
      "outputs": [],
      "source": [
        "# covert to csv\n",
        "def covert_to_csv(file_path,output_file_path):\n",
        "    with codecs.open(file_path, 'r', encoding='utf-8-sig') as tsv_file:\n",
        "        reader = csv.reader(tsv_file, delimiter='\\t')\n",
        "        rows = list(reader)\n",
        "\n",
        "    header = ['text', 'label']\n",
        "    rows.insert(0, header)\n",
        "\n",
        "    with open(output_file_path, 'w', encoding='utf-8-sig', newline='') as csv_file:\n",
        "        writer = csv.writer(csv_file)\n",
        "        writer.writerows(rows)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "Iu7CCdbkaYst"
      },
      "outputs": [],
      "source": [
        "covert_to_csv(r'arman-text-emotion/dataset/train.tsv',r'arman-text-emotion/dataset/train.csv')\n",
        "\n",
        "covert_to_csv(r'arman-text-emotion/dataset/test.tsv',r'arman-text-emotion/dataset/test.csv')"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "RKezkGKzaYst"
      },
      "outputs": [],
      "source": [
        "df = pd.read_csv(r'arman-text-emotion/dataset/train.csv')"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 206
        },
        "id": "dUAT-SV8aYsu",
        "outputId": "0713edc1-c09a-48c0-c140-636545c3d558"
      },
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "                                                text  label\n",
              "0  خیلی کوچیک هستن و سایزشون بدرد نمیخوره میخوام ...    SAD\n",
              "1     از صدای پرنده دم دمای صبح متنفرم متنفرم متنفرم   HATE\n",
              "2   \"کیفیتش خیلی خوبه با شک خریدم ولی واقعا راضیم...    SAD\n",
              "3  چون همش با دوربین ثبت شده ، ایا میشه اعتراض زد...  OTHER\n",
              "4              اين وضع ب طرز خنده داري گريه داره ...    SAD"
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-2fe11853-a16c-4379-8c24-a48666749608\" class=\"colab-df-container\">\n",
              "    <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>text</th>\n",
              "      <th>label</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>خیلی کوچیک هستن و سایزشون بدرد نمیخوره میخوام ...</td>\n",
              "      <td>SAD</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>از صدای پرنده دم دمای صبح متنفرم متنفرم متنفرم</td>\n",
              "      <td>HATE</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>\"کیفیتش خیلی خوبه با شک خریدم ولی واقعا راضیم...</td>\n",
              "      <td>SAD</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>چون همش با دوربین ثبت شده ، ایا میشه اعتراض زد...</td>\n",
              "      <td>OTHER</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>اين وضع ب طرز خنده داري گريه داره ...</td>\n",
              "      <td>SAD</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "    <div class=\"colab-df-buttons\">\n",
              "\n",
              "  <div class=\"colab-df-container\">\n",
              "    <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-2fe11853-a16c-4379-8c24-a48666749608')\"\n",
              "            title=\"Convert this dataframe to an interactive table.\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\" viewBox=\"0 -960 960 960\">\n",
              "    <path d=\"M120-120v-720h720v720H120Zm60-500h600v-160H180v160Zm220 220h160v-160H400v160Zm0 220h160v-160H400v160ZM180-400h160v-160H180v160Zm440 0h160v-160H620v160ZM180-180h160v-160H180v160Zm440 0h160v-160H620v160Z\"/>\n",
              "  </svg>\n",
              "    </button>\n",
              "\n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    .colab-df-buttons div {\n",
              "      margin-bottom: 4px;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "    <script>\n",
              "      const buttonEl =\n",
              "        document.querySelector('#df-2fe11853-a16c-4379-8c24-a48666749608 button.colab-df-convert');\n",
              "      buttonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "      async function convertToInteractive(key) {\n",
              "        const element = document.querySelector('#df-2fe11853-a16c-4379-8c24-a48666749608');\n",
              "        const dataTable =\n",
              "          await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                    [key], {});\n",
              "        if (!dataTable) return;\n",
              "\n",
              "        const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "          '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "          + ' to learn more about interactive tables.';\n",
              "        element.innerHTML = '';\n",
              "        dataTable['output_type'] = 'display_data';\n",
              "        await google.colab.output.renderOutput(dataTable, element);\n",
              "        const docLink = document.createElement('div');\n",
              "        docLink.innerHTML = docLinkHtml;\n",
              "        element.appendChild(docLink);\n",
              "      }\n",
              "    </script>\n",
              "  </div>\n",
              "\n",
              "\n",
              "<div id=\"df-b173c6dc-221b-41d9-86e1-777856f7afa2\">\n",
              "  <button class=\"colab-df-quickchart\" onclick=\"quickchart('df-b173c6dc-221b-41d9-86e1-777856f7afa2')\"\n",
              "            title=\"Suggest charts\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "<svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "     width=\"24px\">\n",
              "    <g>\n",
              "        <path d=\"M19 3H5c-1.1 0-2 .9-2 2v14c0 1.1.9 2 2 2h14c1.1 0 2-.9 2-2V5c0-1.1-.9-2-2-2zM9 17H7v-7h2v7zm4 0h-2V7h2v10zm4 0h-2v-4h2v4z\"/>\n",
              "    </g>\n",
              "</svg>\n",
              "  </button>\n",
              "\n",
              "<style>\n",
              "  .colab-df-quickchart {\n",
              "      --bg-color: #E8F0FE;\n",
              "      --fill-color: #1967D2;\n",
              "      --hover-bg-color: #E2EBFA;\n",
              "      --hover-fill-color: #174EA6;\n",
              "      --disabled-fill-color: #AAA;\n",
              "      --disabled-bg-color: #DDD;\n",
              "  }\n",
              "\n",
              "  [theme=dark] .colab-df-quickchart {\n",
              "      --bg-color: #3B4455;\n",
              "      --fill-color: #D2E3FC;\n",
              "      --hover-bg-color: #434B5C;\n",
              "      --hover-fill-color: #FFFFFF;\n",
              "      --disabled-bg-color: #3B4455;\n",
              "      --disabled-fill-color: #666;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart {\n",
              "    background-color: var(--bg-color);\n",
              "    border: none;\n",
              "    border-radius: 50%;\n",
              "    cursor: pointer;\n",
              "    display: none;\n",
              "    fill: var(--fill-color);\n",
              "    height: 32px;\n",
              "    padding: 0;\n",
              "    width: 32px;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart:hover {\n",
              "    background-color: var(--hover-bg-color);\n",
              "    box-shadow: 0 1px 2px rgba(60, 64, 67, 0.3), 0 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "    fill: var(--button-hover-fill-color);\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart-complete:disabled,\n",
              "  .colab-df-quickchart-complete:disabled:hover {\n",
              "    background-color: var(--disabled-bg-color);\n",
              "    fill: var(--disabled-fill-color);\n",
              "    box-shadow: none;\n",
              "  }\n",
              "\n",
              "  .colab-df-spinner {\n",
              "    border: 2px solid var(--fill-color);\n",
              "    border-color: transparent;\n",
              "    border-bottom-color: var(--fill-color);\n",
              "    animation:\n",
              "      spin 1s steps(1) infinite;\n",
              "  }\n",
              "\n",
              "  @keyframes spin {\n",
              "    0% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "      border-left-color: var(--fill-color);\n",
              "    }\n",
              "    20% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    30% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    40% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    60% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    80% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "    90% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "  }\n",
              "</style>\n",
              "\n",
              "  <script>\n",
              "    async function quickchart(key) {\n",
              "      const quickchartButtonEl =\n",
              "        document.querySelector('#' + key + ' button');\n",
              "      quickchartButtonEl.disabled = true;  // To prevent multiple clicks.\n",
              "      quickchartButtonEl.classList.add('colab-df-spinner');\n",
              "      try {\n",
              "        const charts = await google.colab.kernel.invokeFunction(\n",
              "            'suggestCharts', [key], {});\n",
              "      } catch (error) {\n",
              "        console.error('Error during call to suggestCharts:', error);\n",
              "      }\n",
              "      quickchartButtonEl.classList.remove('colab-df-spinner');\n",
              "      quickchartButtonEl.classList.add('colab-df-quickchart-complete');\n",
              "    }\n",
              "    (() => {\n",
              "      let quickchartButtonEl =\n",
              "        document.querySelector('#df-b173c6dc-221b-41d9-86e1-777856f7afa2 button');\n",
              "      quickchartButtonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "    })();\n",
              "  </script>\n",
              "</div>\n",
              "    </div>\n",
              "  </div>\n"
            ]
          },
          "metadata": {},
          "execution_count": 23
        }
      ],
      "source": [
        "df.head()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "vQ1mf5yvaYsv",
        "outputId": "698a16ad-a808-444f-a834-884c46c039da"
      },
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "OTHER       1681\n",
              "ANGRY        923\n",
              "SAD          896\n",
              "FEAR         757\n",
              "SURPRISE     739\n",
              "HAPPY        618\n",
              "HATE         511\n",
              "Name: label, dtype: int64"
            ]
          },
          "metadata": {},
          "execution_count": 24
        }
      ],
      "source": [
        "df['label'].value_counts()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 467
        },
        "id": "UrdSOpbhaYsw",
        "outputId": "1d91368e-2638-408f-d82f-7314456c9a7b"
      },
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<Axes: xlabel='label', ylabel='count'>"
            ]
          },
          "metadata": {},
          "execution_count": 25
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<Figure size 640x480 with 1 Axes>"
            ],
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAkQAAAGwCAYAAABIC3rIAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/bCgiHAAAACXBIWXMAAA9hAAAPYQGoP6dpAAA64ElEQVR4nO3de1hVZf7//9dGBJSjqIAUHkpTNNPCEx1ME8VDThalFpWOpo2DmtKg+R1FRzPMc5jp2KhoQ2VNH61sMh1LKcUThSaSh3LCSTd0pYCH5CD794cX69cWUEJgb13Px3Wt62Lf93uvdd+L7dov11p7Y7HZbDYBAACYmIujBwAAAOBoBCIAAGB6BCIAAGB6BCIAAGB6BCIAAGB6BCIAAGB6BCIAAGB6ro4ewI2ipKREJ0+elLe3tywWi6OHAwAAKsFms+ns2bMKDg6Wi0vF54EIRJV08uRJhYSEOHoYAACgCk6cOKFbb721wn4CUSV5e3tLurxDfXx8HDwaAABQGfn5+QoJCTHexytCIKqk0stkPj4+BCIAAG4w17rdhZuqAQCA6RGIAACA6RGIAACA6RGIAACA6RGIAACA6RGIAACA6RGIAACA6RGIAACA6RGIAACA6RGIAACA6RGIAACA6RGIAACA6RGIAACA6RGIAACA6RGIAACA6bk6egCAWYXFrXX0EGpc2rxnHT0EAKgUzhABAADTIxABAADTIxABAADTIxABAADTIxABAADTIxABAADTIxABAADTIxABAADTIxABAADTIxABAADTIxABAADTIxABAADTc2ggSklJ0cCBAxUcHCyLxaINGzaUqcnMzNQf/vAH+fr6ytPTU507d1ZWVpbRf/HiRcXExKhhw4by8vJSVFSUsrOz7daRlZWlAQMGqH79+goICFBcXJyKi4trenoAAOAG4dBAdP78eXXo0EFLly4tt//777/X/fffrzZt2mjbtm06cOCApk2bJg8PD6Nm4sSJ+vjjj/X+++9r+/btOnnypB577DGj/9KlSxowYIAKCwu1c+dOrVmzRklJSYqPj6/x+QEAgBuDxWaz2Rw9CEmyWCxav369Bg0aZLQNHTpUdevW1VtvvVXuc/Ly8tS4cWO9/fbbevzxxyVJ3333nUJDQ5Wamqpu3brp008/1cMPP6yTJ08qMDBQkrR8+XJNnjxZP//8s9zc3Co1vvz8fPn6+iovL08+Pj7XN1lAUljcWkcPocalzXvW0UMAYHKVff922nuISkpK9Mknn+iOO+5QZGSkAgIC1LVrV7vLamlpaSoqKlJERITR1qZNGzVt2lSpqamSpNTUVLVv394IQ5IUGRmp/Px8ZWRkVLj9goIC5efn2y0AAODm5LSBKCcnR+fOndOcOXPUt29fbd68WY8++qgee+wxbd++XZJktVrl5uYmPz8/u+cGBgbKarUaNb8NQ6X9pX0VSUhIkK+vr7GEhIRU4+wAAIAzcdpAVFJSIkl65JFHNHHiRHXs2FEvvfSSHn74YS1fvrzGtz9lyhTl5eUZy4kTJ2p8mwAAwDGcNhA1atRIrq6uatu2rV17aGio8SmzoKAgFRYWKjc3164mOztbQUFBRs2VnzorfVxaUx53d3f5+PjYLQAA4ObktIHIzc1NnTt31uHDh+3ajxw5ombNmkmSwsLCVLduXW3dutXoP3z4sLKyshQeHi5JCg8P17fffqucnByjZsuWLfLx8SkTtgAAgDm5OnLj586d07Fjx4zHx48fV3p6uvz9/dW0aVPFxcVpyJAh6t69u3r27KlNmzbp448/1rZt2yRJvr6+GjlypGJjY+Xv7y8fHx+NGzdO4eHh6tatmySpT58+atu2rZ555hnNnTtXVqtVU6dOVUxMjNzd3R0xbQAA4GQcGoj27dunnj17Go9jY2MlScOGDVNSUpIeffRRLV++XAkJCRo/frxat26tDz74QPfff7/xnEWLFsnFxUVRUVEqKChQZGSk3njjDaO/Tp062rhxo8aMGaPw8HB5enpq2LBhmjlzZu1NFAAAODWn+R4iZ8f3EKG68T1EAFDzbvjvIQIAAKgtBCIAAGB6BCIAAGB6BCIAAGB6BCIAAGB6BCIAAGB6BCIAAGB6BCIAAGB6BCIAAGB6BCIAAGB6BCIAAGB6BCIAAGB6BCIAAGB6BCIAAGB6BCIAAGB6BCIAAGB6BCIAAGB6BCIAAGB6BCIAAGB6BCIAAGB6BCIAAGB6BCIAAGB6BCIAAGB6BCIAAGB6BCIAAGB6BCIAAGB6BCIAAGB6BCIAAGB6BCIAAGB6BCIAAGB6BCIAAGB6Dg1EKSkpGjhwoIKDg2WxWLRhw4YKa//0pz/JYrFo8eLFdu2nT59WdHS0fHx85Ofnp5EjR+rcuXN2NQcOHNADDzwgDw8PhYSEaO7cuTUwGwAAcKNyaCA6f/68OnTooKVLl161bv369dq1a5eCg4PL9EVHRysjI0NbtmzRxo0blZKSotGjRxv9+fn56tOnj5o1a6a0tDTNmzdPM2bM0IoVK6p9PgAA4Mbk6siN9+vXT/369btqzU8//aRx48bps88+04ABA+z6MjMztWnTJu3du1edOnWSJC1ZskT9+/fX/PnzFRwcrOTkZBUWFmrVqlVyc3NTu3btlJ6eroULF9oFJwAAYF5OfQ9RSUmJnnnmGcXFxaldu3Zl+lNTU+Xn52eEIUmKiIiQi4uLdu/ebdR0795dbm5uRk1kZKQOHz6sM2fOVLjtgoIC5efn2y0AAODm5NSB6NVXX5Wrq6vGjx9fbr/ValVAQIBdm6urq/z9/WW1Wo2awMBAu5rSx6U15UlISJCvr6+xhISEXM9UAACAE3PaQJSWlqbXXntNSUlJslgstb79KVOmKC8vz1hOnDhR62MAAAC1w2kD0ZdffqmcnBw1bdpUrq6ucnV11Y8//qgXX3xRzZs3lyQFBQUpJyfH7nnFxcU6ffq0goKCjJrs7Gy7mtLHpTXlcXd3l4+Pj90CAABuTk4biJ555hkdOHBA6enpxhIcHKy4uDh99tlnkqTw8HDl5uYqLS3NeN7nn3+ukpISde3a1ahJSUlRUVGRUbNlyxa1bt1aDRo0qN1JAQAAp+TQT5mdO3dOx44dMx4fP35c6enp8vf3V9OmTdWwYUO7+rp16yooKEitW7eWJIWGhqpv374aNWqUli9frqKiIo0dO1ZDhw41PqL/1FNP6W9/+5tGjhypyZMn6+DBg3rttde0aNGi2psoAABwag4NRPv27VPPnj2Nx7GxsZKkYcOGKSkpqVLrSE5O1tixY9WrVy+5uLgoKipKiYmJRr+vr682b96smJgYhYWFqVGjRoqPj+cj9wAAwGCx2Ww2Rw/iRpCfny9fX1/l5eVxPxGqRVjcWkcPocalzXvW0UMAYHKVff922nuIAAAAaguBCAAAmB6BCAAAmB6BCAAAmB6BCAAAmB6BCAAAmB6BCAAAmB6BCAAAmB6BCAAAmB6BCAAAmB6BCAAAmB6BCAAAmB6BCAAAmB6BCAAAmB6BCAAAmB6BCAAAmB6BCAAAmB6BCAAAmB6BCAAAmB6BCAAAmB6BCAAAmB6BCAAAmB6BCAAAmB6BCAAAmB6BCAAAmB6BCAAAmB6BCAAAmB6BCAAAmB6BCAAAmB6BCAAAmB6BCAAAmJ5DA1FKSooGDhyo4OBgWSwWbdiwwegrKirS5MmT1b59e3l6eio4OFjPPvusTp48abeO06dPKzo6Wj4+PvLz89PIkSN17tw5u5oDBw7ogQcekIeHh0JCQjR37tzamB4AALhBODQQnT9/Xh06dNDSpUvL9F24cEFff/21pk2bpq+//lr/93//p8OHD+sPf/iDXV10dLQyMjK0ZcsWbdy4USkpKRo9erTRn5+frz59+qhZs2ZKS0vTvHnzNGPGDK1YsaLG5wcAAG4Mro7ceL9+/dSvX79y+3x9fbVlyxa7ttdff11dunRRVlaWmjZtqszMTG3atEl79+5Vp06dJElLlixR//79NX/+fAUHBys5OVmFhYVatWqV3Nzc1K5dO6Wnp2vhwoV2welKBQUFKigoMB7n5+dXw4wBAIAzuqHuIcrLy5PFYpGfn58kKTU1VX5+fkYYkqSIiAi5uLho9+7dRk337t3l5uZm1ERGRurw4cM6c+ZMhdtKSEiQr6+vsYSEhNTMpAAAgMPdMIHo4sWLmjx5sp588kn5+PhIkqxWqwICAuzqXF1d5e/vL6vVatQEBgba1ZQ+Lq0pz5QpU5SXl2csJ06cqM7pAAAAJ+LQS2aVVVRUpMGDB8tms2nZsmW1sk13d3e5u7vXyrYAAIBjOX0gKg1DP/74oz7//HPj7JAkBQUFKScnx66+uLhYp0+fVlBQkFGTnZ1tV1P6uLQGAACYm1NfMisNQ0ePHtV//vMfNWzY0K4/PDxcubm5SktLM9o+//xzlZSUqGvXrkZNSkqKioqKjJotW7aodevWatCgQe1MBAAAODWHBqJz584pPT1d6enpkqTjx48rPT1dWVlZKioq0uOPP659+/YpOTlZly5dktVqldVqVWFhoSQpNDRUffv21ahRo7Rnzx7t2LFDY8eO1dChQxUcHCxJeuqpp+Tm5qaRI0cqIyND69at02uvvabY2FhHTRsAADgZh14y27dvn3r27Gk8Lg0pw4YN04wZM/TRRx9Jkjp27Gj3vC+++EI9evSQJCUnJ2vs2LHq1auXXFxcFBUVpcTERKPW19dXmzdvVkxMjMLCwtSoUSPFx8df9SP3AADAXBwaiHr06CGbzVZh/9X6Svn7++vtt9++as1dd92lL7/88nePDwAAmINT30MEAABQGwhEAADA9AhEAADA9AhEAADA9AhEAADA9AhEAADA9AhEAADA9AhEAADA9AhEAADA9AhEAADA9AhEAADA9AhEAADA9AhEAADA9AhEAADA9AhEAADA9AhEAADA9AhEAADA9AhEAADA9AhEAADA9AhEAADA9AhEAADA9AhEAADA9AhEAADA9AhEAADA9AhEAADA9AhEAADA9AhEAADA9AhEAADA9AhEAADA9AhEAADA9AhEAADA9BwaiFJSUjRw4EAFBwfLYrFow4YNdv02m03x8fFq0qSJ6tWrp4iICB09etSu5vTp04qOjpaPj4/8/Pw0cuRInTt3zq7mwIEDeuCBB+Th4aGQkBDNnTu3pqcGAABuIA4NROfPn1eHDh20dOnScvvnzp2rxMRELV++XLt375anp6ciIyN18eJFoyY6OloZGRnasmWLNm7cqJSUFI0ePdroz8/PV58+fdSsWTOlpaVp3rx5mjFjhlasWFHj8wMAADcGV0duvF+/furXr1+5fTabTYsXL9bUqVP1yCOPSJLWrl2rwMBAbdiwQUOHDlVmZqY2bdqkvXv3qlOnTpKkJUuWqH///po/f76Cg4OVnJyswsJCrVq1Sm5ubmrXrp3S09O1cOFCu+B0pYKCAhUUFBiP8/Pzq3HmAADAmTjtPUTHjx+X1WpVRESE0ebr66uuXbsqNTVVkpSamio/Pz8jDElSRESEXFxctHv3bqOme/fucnNzM2oiIyN1+PBhnTlzpsLtJyQkyNfX11hCQkKqe4oAAMBJOG0gslqtkqTAwEC79sDAQKPParUqICDArt/V1VX+/v52NeWt47fbKM+UKVOUl5dnLCdOnLi+CQEAAKfl0Etmzszd3V3u7u6OHgYAAKgFTnuGKCgoSJKUnZ1t156dnW30BQUFKScnx66/uLhYp0+ftqspbx2/3QYAADA3pw1ELVq0UFBQkLZu3Wq05efna/fu3QoPD5ckhYeHKzc3V2lpaUbN559/rpKSEnXt2tWoSUlJUVFRkVGzZcsWtW7dWg0aNKil2QAAAGfm0EB07tw5paenKz09XdLlG6nT09OVlZUli8WiCRMm6OWXX9ZHH32kb7/9Vs8++6yCg4M1aNAgSVJoaKj69u2rUaNGac+ePdqxY4fGjh2roUOHKjg4WJL01FNPyc3NTSNHjlRGRobWrVun1157TbGxsQ6aNQAAcDYOvYdo37596tmzp/G4NKQMGzZMSUlJmjRpks6fP6/Ro0crNzdX999/vzZt2iQPDw/jOcnJyRo7dqx69eolFxcXRUVFKTEx0ej39fXV5s2bFRMTo7CwMDVq1Ejx8fFX/cg9AAAwF4vNZrM5ehA3gvz8fPn6+iovL08+Pj6OHg5uAmFxax09hBqXNu9ZRw8BgMlV9v27SpfMHnroIeXm5pa70YceeqgqqwQAAHCYKgWibdu2qbCwsEz7xYsX9eWXX173oAAAAGrT77qH6MCBA8bPhw4dsvtiw0uXLmnTpk265ZZbqm90AAAAteB3BaKOHTvKYrHIYrGUe2msXr16WrJkSbUNDgAAoDb8rkB0/Phx2Ww23XbbbdqzZ48aN25s9Lm5uSkgIEB16tSp9kECAADUpN8ViJo1ayZJKikpqZHBAAAAOEKVv4fo6NGj+uKLL5STk1MmIMXHx1/3wAAAAGpLlQLRm2++qTFjxqhRo0YKCgqSxWIx+iwWC4EIAADcUKoUiF5++WXNnj1bkydPru7xAACugi/0BGpGlb6H6MyZM3riiSeqeywAAAAOUaVA9MQTT2jz5s3VPRYAAACHqNIls5YtW2ratGnatWuX2rdvr7p169r1jx8/vloGBwAAUBuqFIhWrFghLy8vbd++Xdu3b7frs1gsBCIAAHBDqVIgOn78eHWPAwAAwGGq/D1EsHezf/KDT30AAG5mVQpEI0aMuGr/qlWrqjQYAAAAR6hSIDpz5ozd46KiIh08eFC5ubnl/tFXAAAAZ1alQLR+/foybSUlJRozZoxuv/326x4UAABAbarS9xCVuyIXF8XGxmrRokXVtUoAAIBaUW2BSJK+//57FRcXV+cqAQAAalyVLpnFxsbaPbbZbDp16pQ++eQTDRs2rFoGBgAAUFuqFIi++eYbu8cuLi5q3LixFixYcM1PoAEAADibKgWiL774orrHAQAA4DDX9cWMP//8sw4fPixJat26tRo3blwtgwIAAKhNVbqp+vz58xoxYoSaNGmi7t27q3v37goODtbIkSN14cKF6h4jAABAjapSIIqNjdX27dv18ccfKzc3V7m5ufrwww+1fft2vfjii9U9RgAAgBpVpUtmH3zwgf71r3+pR48eRlv//v1Vr149DR48WMuWLauu8QEAANS4Kp0hunDhggIDA8u0BwQEcMkMAADccKoUiMLDwzV9+nRdvHjRaPv111/1t7/9TeHh4dU2OAAAgNpQpUtmixcvVt++fXXrrbeqQ4cOkqT9+/fL3d1dmzdvrtYBAgAA1LQqBaL27dvr6NGjSk5O1nfffSdJevLJJxUdHa169epV6wABAABqWpUumSUkJOjdd9/VqFGjtGDBAi1YsEDPPfec3nnnHb366qvVNrhLly5p2rRpatGiherVq6fbb79ds2bNks1mM2psNpvi4+PVpEkT1atXTxERETp69Kjdek6fPq3o6Gj5+PjIz89PI0eO1Llz56ptnAAA4MZWpUD097//XW3atCnT3q5dOy1fvvy6B1Xq1Vdf1bJly/T6668rMzNTr776qubOnaslS5YYNXPnzlViYqKWL1+u3bt3y9PTU5GRkXb3N0VHRysjI0NbtmzRxo0blZKSotGjR1fbOAEAwI2tSpfMrFarmjRpUqa9cePGOnXq1HUPqtTOnTv1yCOPaMCAAZKk5s2b65133tGePXskXT47tHjxYk2dOlWPPPKIJGnt2rUKDAzUhg0bNHToUGVmZmrTpk3au3evOnXqJElasmSJ+vfvr/nz5ys4OLjcbRcUFKigoMB4nJ+fX23zAgAAzqVKZ4hCQkK0Y8eOMu07duyoMGBUxb333qutW7fqyJEjki7fuP3VV1+pX79+kqTjx4/LarUqIiLCeI6vr6+6du2q1NRUSVJqaqr8/PyMMCRJERERcnFx0e7duyvcdkJCgnx9fY0lJCSk2uYFAACcS5XOEI0aNUoTJkxQUVGRHnroIUnS1q1bNWnSpGr9puqXXnpJ+fn5atOmjerUqaNLly5p9uzZio6OlnT5TJWkMt+JFBgYaPRZrVYFBATY9bu6usrf39+oKc+UKVMUGxtrPM7PzycUAQBwk6pSIIqLi9Mvv/yiP//5zyosLJQkeXh4aPLkyZoyZUq1De69995TcnKy3n77bbVr107p6emaMGGCgoODNWzYsGrbTnnc3d3l7u5eo9sAAADOoUqByGKx6NVXX9W0adOUmZmpevXqqVWrVtUeIOLi4vTSSy9p6NChki5/3P/HH39UQkKChg0bpqCgIElSdna23T1N2dnZ6tixoyQpKChIOTk5dustLi7W6dOnjecDAABzq9I9RKW8vLzUuXNn3XnnnTVyNuXChQtycbEfYp06dVRSUiJJatGihYKCgrR161ajPz8/X7t37za+MTs8PFy5ublKS0szaj7//HOVlJSoa9eu1T5mAABw46nSGaLaMnDgQM2ePVtNmzZVu3bt9M0332jhwoUaMWKEpMtnqiZMmKCXX35ZrVq1UosWLTRt2jQFBwdr0KBBkqTQ0FD17dtXo0aN0vLly1VUVKSxY8dq6NCh1XoDOAAAuHE5dSBasmSJpk2bpj//+c/KyclRcHCwnn/+ecXHxxs1kyZN0vnz5zV69Gjl5ubq/vvv16ZNm+Th4WHUJCcna+zYserVq5dcXFwUFRWlxMRER0wJAAA4IacORN7e3lq8eLEWL15cYY3FYtHMmTM1c+bMCmv8/f319ttv18AIAQDAzeC67iECAAC4GTj1GSIA5hQWt9bRQ6hxafOedfQQAPwGZ4gAAIDpEYgAAIDpEYgAAIDpcQ8RAAA3uZv9vrzquCePM0QAAMD0CEQAAMD0CEQAAMD0CEQAAMD0CEQAAMD0CEQAAMD0CEQAAMD0CEQAAMD0CEQAAMD0CEQAAMD0CEQAAMD0CEQAAMD0CEQAAMD0CEQAAMD0CEQAAMD0CEQAAMD0CEQAAMD0CEQAAMD0CEQAAMD0CEQAAMD0CEQAAMD0CEQAAMD0CEQAAMD0CEQAAMD0XB09gGv56aefNHnyZH366ae6cOGCWrZsqdWrV6tTp06SJJvNpunTp+vNN99Ubm6u7rvvPi1btkytWrUy1nH69GmNGzdOH3/8sVxcXBQVFaXXXntNXl5ejpoWAKCahcWtdfQQalzavGcdPYSbllOfITpz5ozuu+8+1a1bV59++qkOHTqkBQsWqEGDBkbN3LlzlZiYqOXLl2v37t3y9PRUZGSkLl68aNRER0crIyNDW7Zs0caNG5WSkqLRo0c7YkoAAMAJOfUZoldffVUhISFavXq10daiRQvjZ5vNpsWLF2vq1Kl65JFHJElr165VYGCgNmzYoKFDhyozM1ObNm3S3r17jbNKS5YsUf/+/TV//nwFBwfX7qQAAIDTceozRB999JE6deqkJ554QgEBAbr77rv15ptvGv3Hjx+X1WpVRESE0ebr66uuXbsqNTVVkpSamio/Pz8jDElSRESEXFxctHv37gq3XVBQoPz8fLsFAADcnJw6EP3www/G/UCfffaZxowZo/Hjx2vNmjWSJKvVKkkKDAy0e15gYKDRZ7VaFRAQYNfv6uoqf39/o6Y8CQkJ8vX1NZaQkJDqnBoAAHAiTh2ISkpKdM899+iVV17R3XffrdGjR2vUqFFavnx5jW97ypQpysvLM5YTJ07U+DYBAIBjOHUgatKkidq2bWvXFhoaqqysLElSUFCQJCk7O9uuJjs72+gLCgpSTk6OXX9xcbFOnz5t1JTH3d1dPj4+dgsAALg5OXUguu+++3T48GG7tiNHjqhZs2aSLt9gHRQUpK1btxr9+fn52r17t8LDwyVJ4eHhys3NVVpamlHz+eefq6SkRF27dq2FWQAAAGfn1J8ymzhxou6991698sorGjx4sPbs2aMVK1ZoxYoVkiSLxaIJEybo5ZdfVqtWrdSiRQtNmzZNwcHBGjRokKTLZ5T69u1rXGorKirS2LFjNXToUD5hBgAAJDl5IOrcubPWr1+vKVOmaObMmWrRooUWL16s6Ohoo2bSpEk6f/68Ro8erdzcXN1///3atGmTPDw8jJrk5GSNHTtWvXr1Mr6YMTEx0RFTAgAATsipA5EkPfzww3r44Ycr7LdYLJo5c6ZmzpxZYY2/v7/efvvtmhgeAAC4CTj1PUQAAAC1gUAEAABMj0AEAABMj0AEAABMj0AEAABMj0AEAABMj0AEAABMj0AEAABMj0AEAABMj0AEAABMj0AEAABMj0AEAABMz+n/uCtubGFxax09hBqXNu9ZRw8BAHCdOEMEAABMj0AEAABMj0AEAABMj0AEAABMj0AEAABMj0AEAABMj0AEAABMj0AEAABMj0AEAABMj0AEAABMj0AEAABMj0AEAABMj0AEAABMj0AEAABMj0AEAABMj0AEAABMj0AEAABMj0AEAABM74YKRHPmzJHFYtGECROMtosXLyomJkYNGzaUl5eXoqKilJ2dbfe8rKwsDRgwQPXr11dAQIDi4uJUXFxcy6MHAADO6oYJRHv37tXf//533XXXXXbtEydO1Mcff6z3339f27dv18mTJ/XYY48Z/ZcuXdKAAQNUWFionTt3as2aNUpKSlJ8fHxtTwEAADipGyIQnTt3TtHR0XrzzTfVoEEDoz0vL08rV67UwoUL9dBDDyksLEyrV6/Wzp07tWvXLknS5s2bdejQIf3zn/9Ux44d1a9fP82aNUtLly5VYWGho6YEAACcyA0RiGJiYjRgwABFRETYtaelpamoqMiuvU2bNmratKlSU1MlSampqWrfvr0CAwONmsjISOXn5ysjI6PCbRYUFCg/P99uAQAANydXRw/gWt599119/fXX2rt3b5k+q9UqNzc3+fn52bUHBgbKarUaNb8NQ6X9pX0VSUhI0N/+9rfrHD0AALgROPUZohMnTuiFF15QcnKyPDw8anXbU6ZMUV5enrGcOHGiVrcPAABqj1MHorS0NOXk5Oiee+6Rq6urXF1dtX37diUmJsrV1VWBgYEqLCxUbm6u3fOys7MVFBQkSQoKCirzqbPSx6U15XF3d5ePj4/dAgAAbk5OHYh69eqlb7/9Vunp6cbSqVMnRUdHGz/XrVtXW7duNZ5z+PBhZWVlKTw8XJIUHh6ub7/9Vjk5OUbNli1b5OPjo7Zt29b6nAAAgPNx6nuIvL29deedd9q1eXp6qmHDhkb7yJEjFRsbK39/f/n4+GjcuHEKDw9Xt27dJEl9+vRR27Zt9cwzz2ju3LmyWq2aOnWqYmJi5O7uXutzAgAAzsepA1FlLFq0SC4uLoqKilJBQYEiIyP1xhtvGP116tTRxo0bNWbMGIWHh8vT01PDhg3TzJkzHThqAADgTG64QLRt2za7xx4eHlq6dKmWLl1a4XOaNWumf//73zU8MgAAcKNy6nuIAAAAagOBCAAAmB6BCAAAmB6BCAAAmB6BCAAAmB6BCAAAmB6BCAAAmB6BCAAAmB6BCAAAmB6BCAAAmB6BCAAAmB6BCAAAmB6BCAAAmB6BCAAAmB6BCAAAmB6BCAAAmB6BCAAAmB6BCAAAmB6BCAAAmB6BCAAAmB6BCAAAmB6BCAAAmB6BCAAAmB6BCAAAmB6BCAAAmB6BCAAAmB6BCAAAmB6BCAAAmB6BCAAAmB6BCAAAmB6BCAAAmJ7TB6KEhAR17txZ3t7eCggI0KBBg3T48GG7mosXLyomJkYNGzaUl5eXoqKilJ2dbVeTlZWlAQMGqH79+goICFBcXJyKi4trcyoAAMBJOX0g2r59u2JiYrRr1y5t2bJFRUVF6tOnj86fP2/UTJw4UR9//LHef/99bd++XSdPntRjjz1m9F+6dEkDBgxQYWGhdu7cqTVr1igpKUnx8fGOmBIAAHAyro4ewLVs2rTJ7nFSUpICAgKUlpam7t27Ky8vTytXrtTbb7+thx56SJK0evVqhYaGateuXerWrZs2b96sQ4cO6T//+Y8CAwPVsWNHzZo1S5MnT9aMGTPk5ubmiKkBAAAn4fRniK6Ul5cnSfL395ckpaWlqaioSBEREUZNmzZt1LRpU6WmpkqSUlNT1b59ewUGBho1kZGRys/PV0ZGRrnbKSgoUH5+vt0CAABuTjdUICopKdGECRN033336c4775QkWa1Wubm5yc/Pz642MDBQVqvVqPltGCrtL+0rT0JCgnx9fY0lJCSkmmcDAACcxQ0ViGJiYnTw4EG9++67Nb6tKVOmKC8vz1hOnDhR49sEAACO4fT3EJUaO3asNm7cqJSUFN16661Ge1BQkAoLC5Wbm2t3lig7O1tBQUFGzZ49e+zWV/optNKaK7m7u8vd3b2aZwEAAJyR058hstlsGjt2rNavX6/PP/9cLVq0sOsPCwtT3bp1tXXrVqPt8OHDysrKUnh4uCQpPDxc3377rXJycoyaLVu2yMfHR23btq2diQAAAKfl9GeIYmJi9Pbbb+vDDz+Ut7e3cc+Pr6+v6tWrJ19fX40cOVKxsbHy9/eXj4+Pxo0bp/DwcHXr1k2S1KdPH7Vt21bPPPOM5s6dK6vVqqlTpyomJoazQAAAwPkD0bJlyyRJPXr0sGtfvXq1hg8fLklatGiRXFxcFBUVpYKCAkVGRuqNN94wauvUqaONGzdqzJgxCg8Pl6enp4YNG6aZM2fW1jQAAIATc/pAZLPZrlnj4eGhpUuXaunSpRXWNGvWTP/+97+rc2gAAOAm4fT3EAEAANQ0AhEAADA9AhEAADA9AhEAADA9AhEAADA9AhEAADA9AhEAADA9AhEAADA9AhEAADA9AhEAADA9AhEAADA9AhEAADA9AhEAADA9AhEAADA9AhEAADA9AhEAADA9AhEAADA9AhEAADA9AhEAADA9AhEAADA9AhEAADA9AhEAADA9AhEAADA9AhEAADA9AhEAADA9AhEAADA9AhEAADA9AhEAADA9AhEAADA9AhEAADA9AhEAADA9UwWipUuXqnnz5vLw8FDXrl21Z88eRw8JAAA4AdMEonXr1ik2NlbTp0/X119/rQ4dOigyMlI5OTmOHhoAAHAw0wSihQsXatSoUfrjH/+otm3bavny5apfv75WrVrl6KEBAAAHc3X0AGpDYWGh0tLSNGXKFKPNxcVFERERSk1NLfc5BQUFKigoMB7n5eVJkvLz88utv1TwazWO2PlUNO9rudn3i8S+uRr2TcXYNxVj31SMfVO+q+2X0j6bzXb1ldhM4KeffrJJsu3cudOuPS4uztalS5dynzN9+nSbJBYWFhYWFpabYDlx4sRVs4IpzhBVxZQpUxQbG2s8Likp0enTp9WwYUNZLBYHjuxy2g0JCdGJEyfk4+Pj0LE4G/ZNxdg3FWPflI/9UjH2TcWcbd/YbDadPXtWwcHBV60zRSBq1KiR6tSpo+zsbLv27OxsBQUFlfscd3d3ubu727X5+fnV1BCrxMfHxylebM6IfVMx9k3F2DflY79UjH1TMWfaN76+vtesMcVN1W5ubgoLC9PWrVuNtpKSEm3dulXh4eEOHBkAAHAGpjhDJEmxsbEaNmyYOnXqpC5dumjx4sU6f/68/vjHPzp6aAAAwMFME4iGDBmin3/+WfHx8bJarerYsaM2bdqkwMBARw/td3N3d9f06dPLXNID++Zq2DcVY9+Uj/1SMfZNxW7UfWOx2a71OTQAAICbmynuIQIAALgaAhEAADA9AhEAADA9AhEAADA9ApET+PnnnzVmzBg1bdpU7u7uCgoKUmRkpHbs2GFXl5qaqjp16mjAgAFl1vHf//5XFovFWLy9vdWuXTvFxMTo6NGjtTWV6zZ8+HANGjSoTPu2bdtksViUm5tr196mTRu5u7vLarXa1V1t2bZtm5KSksrt8/DwqIVZXtuJEyc0YsQIBQcHy83NTc2aNdMLL7ygX375pczvurwlKSmpwn0mSc2bN9fixYvtHpe3njlz5kgq+/ry9/fXgw8+qC+//LKW9sjVDR8+vNzxHzt2rMK+vn37lllPQkKC6tSpo3nz5pXp++1rxsXFRU2aNNGQIUOUlZVVG1OstIqOE6W/w4CAAJ09e9aur2PHjpoxY4Zd27FjxzRixAjjuHTLLbeoV69eSk5OVnFxsVH3233q4+Ojzp0768MPP5QkvfXWW/L09NSxY8fs1n3y5Ek1aNBAr7/+ejXO/Oqu99jyWz169LA7ZrRt21ZvvPGG0X/la+XWW2/VH//4R+Xk5NTaPrnW+4rFYtGGDRvKPO/K/XTlXO+44w4lJCTY/V2wK48PDRs2VJ8+ffTNN99UaT3p6elG2/r169WtWzf5+voa72sTJkww+qvzWE4gcgJRUVH65ptvtGbNGh05ckQfffSRevTooV9++cWubuXKlRo3bpxSUlJ08uTJctf1n//8R6dOndL+/fv1yiuvKDMzUx06dLD7UsqbxVdffaVff/1Vjz/+uNasWSNJuvfee3Xq1CljGTx4sPr27WvXdu+990q6/C2qv20/deqUfvzxR0dOSZL0ww8/qFOnTjp69KjeeecdHTt2TMuXLze+SNTb29tuzC+++KLatWtn1zZkyJDfvd2ZM2eW2R/jxo2zqyl9faWkpCg4OFgPP/xwmW+Ad5Qrf8+nTp1SixYtKux75513yqxj1apVmjRpklatWlXuNkpfMz/99JM++OADHT58WE888USNzuv3utZx4uzZs5o/f/5V17Fnzx7dc889yszM1NKlS3Xw4EFt27ZNzz33nJYtW6aMjAy7+tWrV+vUqVPat2+f7rvvPj3++OP69ttv9cwzzygyMlLDhw9XSUmJUT9q1CiFhYUpJiameiZdzco7tlxp1KhROnXqlA4dOqTBgwcrJibG7jVV+lr53//+pzfffFOffvqpnnnmmVrbJ5V9X6mM0rkePnxYU6ZMUXx8vJYvX16mrvT48Nlnn+ncuXPq16+fXdCs7HpKbd26VUOGDFFUVJT27NmjtLQ0zZ49W0VFRXZ11XYsr44/noqqO3PmjE2Sbdu2bVetO3v2rM3Ly8v23Xff2YYMGWKbPXu2Xf/x48dtkmzffPONXfulS5dsPXr0sDVr1sxWXFxc3cOvdsOGDbM98sgjZdq/+OILmyTbmTNnjLbhw4fbXnrpJdunn35qu+OOO37X+lavXm3z9fWtnkFXs759+9puvfVW24ULF+zaT506Zatfv77tT3/6k1379OnTbR06dCiznvL2WalmzZrZFi1aVOHjK5X3+jpw4IBNku3DDz+szLRqVEW/52v1/da2bdtst9xyi62wsNAWHBxs27Fjh11/ea+ZxMREmyRbXl5eFUdeva52nCj9HcbFxdm8vLxs2dnZRl+HDh1s06dPt9lsNltJSYktNDTUFhYWZrt06VK52ykpKTF+lmRbv3698Tg/P98myfbaa6/ZbDabLScnx9a4cWPbvHnzbDbb/78fs7KyqmvalVKdx5YHH3zQ9sILL9i1tWrVyjZ06FCbzVb+a2X27Nk2FxcX24ULF2p8n1TmfeXK31upK/dTeXO95557bI8++qjxuLzjw44dO2ySbJs2baryel544QVbjx49rjrX6jyWc4bIwby8vOTl5aUNGzaooKCgwrr33ntPbdq0UevWrfX0009r1apVdqcaK+Li4qIXXnhBP/74o9LS0qpz6A519uxZvf/++3r66afVu3dv5eXlOc3lm+tx+vRpffbZZ/rzn/+sevXq2fUFBQUpOjpa69atq9Tvvib9+uuvWrt2raTLfxrnZrBy5Uo9+eSTqlu3rp588kmtXLnyqvU5OTlav3696tSpozp16tTSKK+uMseJJ598Ui1bttTMmTPLXUd6eroyMzP1l7/8RS4u5b9FVPQHrouLi439Vvq6aNy4sVasWKFp06Zpy5Ytmjhxol577TWFhIRUdZo1qqrHlnr16qmwsPCq/SUlJSouLq7xfVLZ95Xfy2az6csvv9R33313zX/3pcev8vZJZdcTFBSkjIwMHTx48PoGXkkEIgdzdXVVUlKS1qxZIz8/P9133336f//v/+nAgQN2dStXrtTTTz8t6fLp/7y8PG3fvr1S22jTpo2ky9dnbwQbN240/kGXLv369bOreffdd9WqVSu1a9dOderU0dChQ6/5BnalvLy8a26nth09elQ2m02hoaHl9oeGhurMmTP6+eefK73OW2+9tcw8y7vvZfLkyWXqrnwjuPfee+Xl5SVPT0/Nnz9fYWFh6tWr1++bZA258nXz20tZ5b2mXnnlFaM/Pz9f//rXv4x/Y08//bTee+89nTt3zm4bpa8ZT09PBQYG6osvvlBMTIw8PT1rZ5LXUJnjROm9YStWrND3339fZh1HjhyRJLVu3dpoy8nJsdt3v71fRrocsry8vOTu7q6JEyeqefPmGjx4sNE/aNAg4/L1gw8+qGHDhlXbnH+Pmji2XLp0Sf/85z914MABPfTQQ+XWHD16VMuXL1enTp3k7e0tqWb3SWXfVyrrjTfeMH6/3bt3V0lJicaPH19hfW5urmbNmiUvLy916dKlyusZN26cOnfurPbt26t58+YaOnSoVq1aVSbkVdexnEDkBKKionTy5El99NFH6tu3r7Zt26Z77rlHSUlJkqTDhw9rz549evLJJyVdfrEPGTKk0gGg9H+IFf2vztn07NlT6enpdss//vEPu5pVq1YZB37p8hvY+++/X+Zm0avx9va+5nYcpTrPAH355Zdl5hkcHFymLi4urkxdp06d7GrWrVunb775Rh988IFatmyppKQk1a1bt9rGej2ufN0kJiZW2Jeenq4//elPRv8777yj22+/XR06dJB0+SbjZs2aad26dXbbKH3N7Nu3TwsWLNA999yj2bNn184Er+H3HCciIyN1//33a9q0aZVad8OGDY395ufnV+Z//YsWLVJ6ero+/fRTtW3bVv/4xz/k7+9vVzNt2jSVlJRo6tSpVZzh9avOY0vpm3u9evU0atQoTZw4UWPGjDH6S9+k69evr9atWyswMFDJycl266jJfXKt95XfIzo6Wunp6dqxY4f69eunv/71r8a9mL9V+h+mBg0aaP/+/Vq3bp3dn8eq7HpKeXp66pNPPtGxY8c0depUeXl56cUXX1SXLl104cIFo666juWm+Vtmzs7Dw0O9e/dW7969NW3aND333HOaPn26hg8frpUrV6q4uNjuTcxms8nd3V2vv/66fH19r7ruzMxMSTJuMHV2np6eatmypV3b//73P+PnQ4cOadeuXdqzZ48mT55stF+6dEnvvvuuRo0aVantuLi4lNmOo7Vs2VIWi0WZmZl69NFHy/RnZmaqQYMGaty4caXX2aJFC/n5+dm1ubqW/affqFGja+6PkJAQtWrVSq1atVJxcbEeffRRHTx40Cn+ZlF5r5vK9EmXz6xkZGTY7ZeSkhKtWrVKI0eONNp++5oJDQ3V999/rzFjxuitt96qpllU3bWOE1eaM2eOwsPDFRcXZ9feqlUrSZcD1t133y1JqlOnjjHv8l47QUFBatmypVq2bKnVq1erf//+OnTokAICAoya0ueV9/zaUp3HlujoaP31r39VvXr11KRJkzKXF729vfX1118bn0i88hK4VPP75GrvK97e3srLyyvznNzc3DLvKb6+vsZ+e++999SyZUt169ZNERERdnXr1q1T27Zt1bBhwzLHnN+znivdfvvtuv322/Xcc8/pr3/9q+644w6tW7fO+OPs1XUs5wyRk2rbtq3Onz+v4uJirV27VgsWLLBLv/v371dwcHC5n5T5rZKSEiUmJqpFixbGwe1Gt3LlSnXv3l379++32yexsbG/+7KZs2nYsKF69+6tN954Q7/++qtdn9VqVXJysoYMGeIUZ/sef/xxubq6lrl8cqP59ttvtW/fPm3bts3u9bRt2zalpqbqu+++q/C5L730ktatW6evv/66FkdcVlWOE126dNFjjz2ml156ya797rvvVps2bTR//ny7T0FVVpcuXRQWFuY0Z85+j99zbCl9c7/lllvKvdeq9E36tttuKzcMOULp+4p0+ZLolfeVXrp0Sfv379cdd9xR4Tq8vLz0wgsv6C9/+UuZM9khISG6/fbbyw1Dv2c9V9O8eXPVr1/fmEd14gyRg/3yyy964oknNGLECN11113y9vbWvn37NHfuXD3yyCPauHGjzpw5o5EjR5ZJ7VFRUVq5cqXdqf9ffvlFVqtVFy5c0MGDB7V48WLt2bNHn3zyidPc+Hk9ioqK9NZbb2nmzJm688477fqee+45LVy4UBkZGWrXrt0112Wz2cr9jpGAgIAKbyatDa+//rruvfdeRUZG6uWXX1aLFi2UkZGhuLg43XLLLTX2RnP27Nky+6N+/fry8fEpt95isWj8+PGaMWOGnn/+edWvX79GxlUdCgoKyszN1dVVjRo10sqVK9WlSxd17969zPM6d+6slStXlvu9RNLlN4BHH31U8fHx2rhxY42MvTIqc5wo73uXZs+erXbt2tmdobBYLFq9erV69+6t++67T1OmTFFoaKiKioqUkpKin3/++ZrHkgkTJujRRx/VpEmTdMstt1TPJGtYdR5bHO1a7yuSFBsbq5EjR6pNmzbq3bu3zp8/ryVLlujMmTN67rnnrrr+559/XrNmzdIHH3ygxx9/vMrjvNZ6ZsyYoQsXLqh///5q1qyZcnNzlZiYqKKiIvXu3duoq65jOWeIHMzLy0tdu3bVokWL1L17d915552aNm2aRo0apddff10rV65UREREuZfFoqKitG/fPrsb5SIiItSkSRO1b99eL730kkJDQ3XgwAH17NmzNqdVY1JSUvTLL7+UezkpNDRUoaGhlT5LlJ+fryZNmpRZcnJyqnvYv0urVq20b98+3XbbbRo8eLBuv/12jR49Wj179lRqamqZezOqS3x8fJl9MWnSpKs+Z9iwYSoqKqrVL9irik2bNpWZ2/3336/CwkL985//VFRUVLnPi4qK0tq1a8t878lvTZw4UZ988on27NlTU8O/psocJ/Lz88v03XHHHRoxYoQuXrxo196tWzelpaWpdevWiomJUdu2bXXvvffqnXfe0aJFi+zulSlP37591aJFixvqLFF1Hlsc7VrvK9LlG+H/8Y9/aNWqVQoLC1Pfvn1ltVqVkpJid99Pefz9/fXss89qxowZVTqLWNn1PPjgg/rhhx/07LPPqk2bNurXr5+sVqs2b95sd9N/dR3LLTZHf34XAADAwThDBAAATI9ABAAATI9ABAAATI9ABAAATI9ABAAATI9ABAAATI9ABAAATI9ABAAATI9ABOCm0KNHD02YMKFStdu2bZPFYlFubu51bbN58+ZavHjxda0DgHMgEAEAANMjEAEAANMjEAG46bz11lvq1KmTvL29FRQUpKeeeqrcP/S4Y8cO3XXXXfLw8FC3bt108OBBu/6vvvpKDzzwgOrVq6eQkBCNHz9e58+fr61pAKhFBCIAN52ioiLNmjVL+/fv14YNG/Tf//5Xw4cPL1MXFxenBQsWaO/evWrcuLEGDhxo/GX777//Xn379lVUVJQOHDigdevW6auvvtLYsWNreTYAaoOrowcAANVtxIgRxs+33XabEhMT1blzZ507d05eXl5G3/Tp09W7d29J0po1a3Trrbdq/fr1Gjx4sBISEhQdHW3cqN2qVSslJibqwQcf1LJly+Th4VGrcwJQszhDBOCmk5aWpoEDB6pp06by9vbWgw8+KEnKysqyqwsPDzd+9vf3V+vWrZWZmSlJ2r9/v5KSkuTl5WUskZGRKikp0fHjx2tvMgBqBWeIANxUzp8/r8jISEVGRio5OVmNGzdWVlaWIiMjVVhYWOn1nDt3Ts8//7zGjx9fpq9p06bVOWQAToBABOCm8t133+mXX37RnDlzFBISIknat29fubW7du0yws2ZM2d05MgRhYaGSpLuueceHTp0SC1btqydgQNwKC6ZAbipNG3aVG5ublqyZIl++OEHffTRR5o1a1a5tTNnztTWrVt18OBBDR8+XI0aNdKgQYMkSZMnT9bOnTs1duxYpaen6+jRo/rwww+5qRq4SRGIANxUGjdurKSkJL3//vtq27at5syZo/nz55dbO2fOHL3wwgsKCwuT1WrVxx9/LDc3N0nSXXfdpe3bt+vIkSN64IEHdPfddys+Pl7BwcG1OR0AtcRis9lsjh4EAACAI3GGCAAAmB6BCAAAmB6BCAAAmB6BCAAAmB6BCAAAmB6BCAAAmB6BCAAAmB6BCAAAmB6BCAAAmB6BCAAAmB6BCAAAmN7/B21QcVpDDnDsAAAAAElFTkSuQmCC\n"
          },
          "metadata": {}
        }
      ],
      "source": [
        "sns.countplot(x='label',data=df)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "JGjyLCUaaYsx"
      },
      "source": [
        "<h2>Preprocess Text</h2>"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "HFKMD1ShaYsy"
      },
      "outputs": [],
      "source": [
        "from parsivar import Normalizer\n",
        "import re\n",
        "import copy\n",
        "import csv\n",
        "from openpyxl import Workbook"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "jN5pGOBtaYsz"
      },
      "outputs": [],
      "source": [
        "# 1.remove English letter\n",
        "def removeEnglishLetter(text):\n",
        "    RemoveEnglish_text = re.sub(r'[A-Za-z]', '', text)\n",
        "    return RemoveEnglish_text"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "ICTQh-BAaYs0"
      },
      "outputs": [],
      "source": [
        "# 2.remove repeated more than twice in the non-standard spelling\n",
        "def removeDuplicateCharacters(text_org):\n",
        "    text:str=copy.deepcopy(text_org)\n",
        "    last=text[0]\n",
        "    counter=1\n",
        "    i=1\n",
        "    while i<len(text):\n",
        "        current=text[i]\n",
        "        if current==last:\n",
        "            counter+=1\n",
        "        else:\n",
        "            last=current\n",
        "            if counter>=3:\n",
        "                text=text[:i-counter]+text[i-1:]\n",
        "                i-=counter-1\n",
        "            counter=1\n",
        "        i+=1\n",
        "    if counter>=3:\n",
        "        text=text[:i-counter]+text[i-1:]\n",
        "        i-=counter-1\n",
        "        counter=1\n",
        "    return text"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "BU_AkJ-daYs0"
      },
      "outputs": [],
      "source": [
        "# 3.Remove Arabic diacritics\n",
        "def RemoveArabic(text):\n",
        "    removearabic = re.sub(r'[\\u064B-\\u065F]', '', text)\n",
        "    return removearabic"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "wPexYDdTaYs0"
      },
      "outputs": [],
      "source": [
        "# 4.Remove remaining non-Persian characters\n",
        "# def Removenonpersian(text):\n",
        "#     nonpersian_text = re.sub(r'[^\\u0600-\\u06FF\\uFB8A\\u067E\\u0686\\u06AF\\u200C]', '',text)\n",
        "#     return nonpersian_text"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "YAvtcTAeaYs0"
      },
      "outputs": [],
      "source": [
        "# 5.Remove hashtag sign while keeping the information included in hashtags\n",
        "def unTag(text):\n",
        "    removehashtag_text = re.sub(r'#', '', text)\n",
        "    removehashtag_text = re.sub(r'_', ' ', removehashtag_text)\n",
        "    return removehashtag_text"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "SSxzvlbBaYs1"
      },
      "outputs": [],
      "source": [
        "# 6.Remove Persian numeric characters\n",
        "def RemovePersiannumber(text):\n",
        "    RemovePersian_text = re.sub(r'[۰-۹]', '', text)\n",
        "    return RemovePersian_text"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "d04sdAmXaYs1"
      },
      "outputs": [],
      "source": [
        "# 7.Parsivar Normalizer\n",
        "def normalize_text(text):\n",
        "    normalizer = Normalizer()\n",
        "    normalized_text = normalizer.normalize(text)\n",
        "    return normalized_text"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "AzU6OX3IaYs1"
      },
      "outputs": [],
      "source": [
        "df = pd.read_csv(r'arman-text-emotion/dataset/train.csv')\n",
        "df['text_clean'] = df['text'].apply(removeEnglishLetter)\\\n",
        "                            .apply(removeDuplicateCharacters)\\\n",
        "                            .apply(RemoveArabic)\\\n",
        "                            .apply(unTag)\\\n",
        "                            .apply(RemovePersiannumber)\\\n",
        "                            .apply(normalize_text)\n",
        "\n",
        "df['text'] = df['text_clean']\n",
        "df.drop('text_clean', axis=1, inplace=True)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 206
        },
        "id": "nLzioduhaYs1",
        "outputId": "a2a34f36-516d-4425-8e75-f9d3c9a91745"
      },
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "                                                text  label\n",
              "0  خیلی کوچیک هستن و سایزشون بدرد نمیخوره میخوام ...    SAD\n",
              "1     از صدای پرنده دم دمای صبح متنفرم متنفرم متنفرم   HATE\n",
              "2  \" کیفیتش خیلی خوبه با شک خریدم ولی واقعا راضیم...    SAD\n",
              "3  چون همش با دوربین ثبت‌شده‌، ایا میشه اعتراض زد...  OTHER\n",
              "4                این وضع ب طرز خنده‌داری گریه داره .    SAD"
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-b269b9ca-f401-4765-b362-c796d9d2c7c9\" class=\"colab-df-container\">\n",
              "    <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>text</th>\n",
              "      <th>label</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>خیلی کوچیک هستن و سایزشون بدرد نمیخوره میخوام ...</td>\n",
              "      <td>SAD</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>از صدای پرنده دم دمای صبح متنفرم متنفرم متنفرم</td>\n",
              "      <td>HATE</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>\" کیفیتش خیلی خوبه با شک خریدم ولی واقعا راضیم...</td>\n",
              "      <td>SAD</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>چون همش با دوربین ثبت‌شده‌، ایا میشه اعتراض زد...</td>\n",
              "      <td>OTHER</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>این وضع ب طرز خنده‌داری گریه داره .</td>\n",
              "      <td>SAD</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "    <div class=\"colab-df-buttons\">\n",
              "\n",
              "  <div class=\"colab-df-container\">\n",
              "    <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-b269b9ca-f401-4765-b362-c796d9d2c7c9')\"\n",
              "            title=\"Convert this dataframe to an interactive table.\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\" viewBox=\"0 -960 960 960\">\n",
              "    <path d=\"M120-120v-720h720v720H120Zm60-500h600v-160H180v160Zm220 220h160v-160H400v160Zm0 220h160v-160H400v160ZM180-400h160v-160H180v160Zm440 0h160v-160H620v160ZM180-180h160v-160H180v160Zm440 0h160v-160H620v160Z\"/>\n",
              "  </svg>\n",
              "    </button>\n",
              "\n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    .colab-df-buttons div {\n",
              "      margin-bottom: 4px;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "    <script>\n",
              "      const buttonEl =\n",
              "        document.querySelector('#df-b269b9ca-f401-4765-b362-c796d9d2c7c9 button.colab-df-convert');\n",
              "      buttonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "      async function convertToInteractive(key) {\n",
              "        const element = document.querySelector('#df-b269b9ca-f401-4765-b362-c796d9d2c7c9');\n",
              "        const dataTable =\n",
              "          await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                    [key], {});\n",
              "        if (!dataTable) return;\n",
              "\n",
              "        const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "          '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "          + ' to learn more about interactive tables.';\n",
              "        element.innerHTML = '';\n",
              "        dataTable['output_type'] = 'display_data';\n",
              "        await google.colab.output.renderOutput(dataTable, element);\n",
              "        const docLink = document.createElement('div');\n",
              "        docLink.innerHTML = docLinkHtml;\n",
              "        element.appendChild(docLink);\n",
              "      }\n",
              "    </script>\n",
              "  </div>\n",
              "\n",
              "\n",
              "<div id=\"df-6178e5b4-d9cf-4591-8d5d-4041d495909e\">\n",
              "  <button class=\"colab-df-quickchart\" onclick=\"quickchart('df-6178e5b4-d9cf-4591-8d5d-4041d495909e')\"\n",
              "            title=\"Suggest charts\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "<svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "     width=\"24px\">\n",
              "    <g>\n",
              "        <path d=\"M19 3H5c-1.1 0-2 .9-2 2v14c0 1.1.9 2 2 2h14c1.1 0 2-.9 2-2V5c0-1.1-.9-2-2-2zM9 17H7v-7h2v7zm4 0h-2V7h2v10zm4 0h-2v-4h2v4z\"/>\n",
              "    </g>\n",
              "</svg>\n",
              "  </button>\n",
              "\n",
              "<style>\n",
              "  .colab-df-quickchart {\n",
              "      --bg-color: #E8F0FE;\n",
              "      --fill-color: #1967D2;\n",
              "      --hover-bg-color: #E2EBFA;\n",
              "      --hover-fill-color: #174EA6;\n",
              "      --disabled-fill-color: #AAA;\n",
              "      --disabled-bg-color: #DDD;\n",
              "  }\n",
              "\n",
              "  [theme=dark] .colab-df-quickchart {\n",
              "      --bg-color: #3B4455;\n",
              "      --fill-color: #D2E3FC;\n",
              "      --hover-bg-color: #434B5C;\n",
              "      --hover-fill-color: #FFFFFF;\n",
              "      --disabled-bg-color: #3B4455;\n",
              "      --disabled-fill-color: #666;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart {\n",
              "    background-color: var(--bg-color);\n",
              "    border: none;\n",
              "    border-radius: 50%;\n",
              "    cursor: pointer;\n",
              "    display: none;\n",
              "    fill: var(--fill-color);\n",
              "    height: 32px;\n",
              "    padding: 0;\n",
              "    width: 32px;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart:hover {\n",
              "    background-color: var(--hover-bg-color);\n",
              "    box-shadow: 0 1px 2px rgba(60, 64, 67, 0.3), 0 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "    fill: var(--button-hover-fill-color);\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart-complete:disabled,\n",
              "  .colab-df-quickchart-complete:disabled:hover {\n",
              "    background-color: var(--disabled-bg-color);\n",
              "    fill: var(--disabled-fill-color);\n",
              "    box-shadow: none;\n",
              "  }\n",
              "\n",
              "  .colab-df-spinner {\n",
              "    border: 2px solid var(--fill-color);\n",
              "    border-color: transparent;\n",
              "    border-bottom-color: var(--fill-color);\n",
              "    animation:\n",
              "      spin 1s steps(1) infinite;\n",
              "  }\n",
              "\n",
              "  @keyframes spin {\n",
              "    0% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "      border-left-color: var(--fill-color);\n",
              "    }\n",
              "    20% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    30% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    40% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    60% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    80% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "    90% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "  }\n",
              "</style>\n",
              "\n",
              "  <script>\n",
              "    async function quickchart(key) {\n",
              "      const quickchartButtonEl =\n",
              "        document.querySelector('#' + key + ' button');\n",
              "      quickchartButtonEl.disabled = true;  // To prevent multiple clicks.\n",
              "      quickchartButtonEl.classList.add('colab-df-spinner');\n",
              "      try {\n",
              "        const charts = await google.colab.kernel.invokeFunction(\n",
              "            'suggestCharts', [key], {});\n",
              "      } catch (error) {\n",
              "        console.error('Error during call to suggestCharts:', error);\n",
              "      }\n",
              "      quickchartButtonEl.classList.remove('colab-df-spinner');\n",
              "      quickchartButtonEl.classList.add('colab-df-quickchart-complete');\n",
              "    }\n",
              "    (() => {\n",
              "      let quickchartButtonEl =\n",
              "        document.querySelector('#df-6178e5b4-d9cf-4591-8d5d-4041d495909e button');\n",
              "      quickchartButtonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "    })();\n",
              "  </script>\n",
              "</div>\n",
              "    </div>\n",
              "  </div>\n"
            ]
          },
          "metadata": {},
          "execution_count": 35
        }
      ],
      "source": [
        "df.head()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "tganRWjsaYs2"
      },
      "outputs": [],
      "source": [
        "def preprocess_csv(input_file, output_file):\n",
        "    df = pd.read_csv(input_file)\n",
        "    df['text_clean'] = df['text'].apply(removeEnglishLetter)\\\n",
        "                                .apply(removeDuplicateCharacters)\\\n",
        "                                .apply(RemoveArabic)\\\n",
        "                                .apply(unTag)\\\n",
        "                                .apply(RemovePersiannumber)\\\n",
        "                                .apply(normalize_text)\n",
        "    df['text'] = df['text_clean']\n",
        "    df.drop('text_clean', axis=1, inplace=True)\n",
        "    df.to_csv(output_file, index=False, encoding='utf-8-sig')"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "3NijPIkMaYs2"
      },
      "outputs": [],
      "source": [
        "preprocess_csv(r'arman-text-emotion/dataset/train.csv', r'arman-text-emotion/dataset/train_clean.csv')"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "MVIwWrEeaYs2"
      },
      "outputs": [],
      "source": [
        "preprocess_csv(r'arman-text-emotion/dataset/test.csv', r'arman-text-emotion/dataset/test_clean.csv')"
      ]
    }
  ],
  "metadata": {
    "kernelspec": {
      "display_name": "Python 3",
      "language": "python",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.10.12"
    },
    "colab": {
      "provenance": []
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}