src/text_to_features.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "provenance": []
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "cells": [
    {
      "cell_type": "code",
      "execution_count": 1,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Ml0sHbFpw3XA",
        "outputId": "43bbb2af-b457-4fc9-f39f-b8cd068aff49"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n",
            "Collecting sklearn_crfsuite\n",
            "  Downloading sklearn_crfsuite-0.3.6-py2.py3-none-any.whl (12 kB)\n",
            "Collecting python-crfsuite>=0.8.3 (from sklearn_crfsuite)\n",
            "  Downloading python_crfsuite-0.9.9-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (993 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m993.5/993.5 kB\u001b[0m \u001b[31m26.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: six in /usr/local/lib/python3.10/dist-packages (from sklearn_crfsuite) (1.16.0)\n",
            "Requirement already satisfied: tabulate in /usr/local/lib/python3.10/dist-packages (from sklearn_crfsuite) (0.8.10)\n",
            "Requirement already satisfied: tqdm>=2.0 in /usr/local/lib/python3.10/dist-packages (from sklearn_crfsuite) (4.65.0)\n",
            "Installing collected packages: python-crfsuite, sklearn_crfsuite\n",
            "Successfully installed python-crfsuite-0.9.9 sklearn_crfsuite-0.3.6\n",
            "Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n",
            "Collecting eli5\n",
            "  Downloading eli5-0.13.0.tar.gz (216 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m216.2/216.2 kB\u001b[0m \u001b[31m9.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25h  Preparing metadata (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "Requirement already satisfied: attrs>17.1.0 in /usr/local/lib/python3.10/dist-packages (from eli5) (23.1.0)\n",
            "Requirement already satisfied: jinja2>=3.0.0 in /usr/local/lib/python3.10/dist-packages (from eli5) (3.1.2)\n",
            "Requirement already satisfied: numpy>=1.9.0 in /usr/local/lib/python3.10/dist-packages (from eli5) (1.22.4)\n",
            "Requirement already satisfied: scipy in /usr/local/lib/python3.10/dist-packages (from eli5) (1.10.1)\n",
            "Requirement already satisfied: six in /usr/local/lib/python3.10/dist-packages (from eli5) (1.16.0)\n",
            "Requirement already satisfied: scikit-learn>=0.20 in /usr/local/lib/python3.10/dist-packages (from eli5) (1.2.2)\n",
            "Requirement already satisfied: graphviz in /usr/local/lib/python3.10/dist-packages (from eli5) (0.20.1)\n",
            "Requirement already satisfied: tabulate>=0.7.7 in /usr/local/lib/python3.10/dist-packages (from eli5) (0.8.10)\n",
            "Requirement already satisfied: MarkupSafe>=2.0 in /usr/local/lib/python3.10/dist-packages (from jinja2>=3.0.0->eli5) (2.1.3)\n",
            "Requirement already satisfied: joblib>=1.1.1 in /usr/local/lib/python3.10/dist-packages (from scikit-learn>=0.20->eli5) (1.2.0)\n",
            "Requirement already satisfied: threadpoolctl>=2.0.0 in /usr/local/lib/python3.10/dist-packages (from scikit-learn>=0.20->eli5) (3.1.0)\n",
            "Building wheels for collected packages: eli5\n",
            "  Building wheel for eli5 (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for eli5: filename=eli5-0.13.0-py2.py3-none-any.whl size=107730 sha256=067cc72a95577b4974a9bb2a1160c340bd9855c3406c306b9279852d48544d13\n",
            "  Stored in directory: /root/.cache/pip/wheels/b8/58/ef/2cf4c306898c2338d51540e0922c8e0d6028e07007085c0004\n",
            "Successfully built eli5\n",
            "Installing collected packages: eli5\n",
            "Successfully installed eli5-0.13.0\n"
          ]
        }
      ],
      "source": [
        "!pip install sklearn_crfsuite\n",
        "!pip install eli5"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "from build_features import text_to_features\n",
        "\n",
        "# function to create features\n",
        "\n",
        "def text2num(wrd):\n",
        "\n",
        "  return[text_to_features(wrd, i) for i in range(len(wrd))]\n",
        "\n",
        "# function to create labels\n",
        "\n",
        "def text2lbl (wrd):\n",
        "\n",
        "   return [label for token, label in wrd]\n",
        "\n",
        "# Let's prepare the training data features using sent2features function which in turn uses text to features function.\n",
        "\n",
        "X = [text2num(x) for x in sentences]\n",
        "\n",
        "# Prepare the training data labels using text2lbl because the sentences are in a tuple consisting of words and tags.\n",
        "\n",
        "y = [text2lb1(x) for x in sentences]\n",
        "\n",
        "# Similarly, prepare the data.\n",
        "\n",
        "test_X = [text2num(x) for x in sentences]\n",
        "\n",
        "test_y = [text2lbl(x) for x in sentences]\n",
        "\n",
        "# Now that the training and test data are ready, let's initialize the model. First, let's initialize and build a CRF model without hyperparameter tuning.\n",
        "\n",
        "# building the CRF model\n",
        "\n",
        "ner_crf_model = CRF(algorithm='lbfgs', max_iterations=25)\n",
        "\n",
        "# tgraining the model with cross validation of 10\n",
        "\n",
        "ner_predictions = cross_val_predict(estimator= ner_crf_model, X=X, y=y, cv=10)\n",
        "\n",
        "#Let's evaluate the model on train data.\n",
        "\n",
        "Accu_rpt = flat_classification_report(y_true=y,y_pred=ner_predictions,zero_division=1)\n",
        "\n",
        "print(Accu_rpt)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "H5YR7xAuxBiV",
        "outputId": "18824054-a2f9-47b1-c59e-2d1b7e79a40f"
      },
      "execution_count": 2,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "                  precision    recall  f1-score   support\n",
            "\n",
            "         B-Actor       0.94      0.92      0.93      5010\n",
            "         B-Award       0.71      0.63      0.67       309\n",
            "B-Character_Name       0.72      0.39      0.51      1024\n",
            "      B-Director       0.85      0.81      0.83      1787\n",
            "         B-Genre       0.84      0.80      0.82      3384\n",
            "       B-Opinion       0.48      0.38      0.42       810\n",
            "        B-Origin       0.52      0.42      0.47       779\n",
            "          B-Plot       0.49      0.47      0.48      6468\n",
            "         B-Quote       0.78      0.37      0.51       126\n",
            "  B-Relationship       0.68      0.51      0.58       580\n",
            "    B-Soundtrack       0.69      0.22      0.33        50\n",
            "          B-Year       0.96      0.97      0.97      2702\n",
            "         I-Actor       0.94      0.92      0.93      6121\n",
            "         I-Award       0.81      0.73      0.77       719\n",
            "I-Character_Name       0.71      0.40      0.51       760\n",
            "      I-Director       0.89      0.81      0.85      1653\n",
            "         I-Genre       0.77      0.72      0.74      2283\n",
            "       I-Opinion       0.26      0.12      0.16       539\n",
            "        I-Origin       0.69      0.68      0.68      3340\n",
            "          I-Plot       0.86      0.94      0.90     62107\n",
            "         I-Quote       0.78      0.44      0.56       817\n",
            "  I-Relationship       0.53      0.41      0.46      1206\n",
            "    I-Soundtrack       0.81      0.30      0.44       158\n",
            "          I-Year       0.62      0.66      0.64       195\n",
            "               O       0.87      0.84      0.85     55895\n",
            "\n",
            "        accuracy                           0.84    158822\n",
            "       macro avg       0.73      0.59      0.64    158822\n",
            "    weighted avg       0.84      0.84      0.84    158822\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# building the CRF model\n",
        "\n",
        "crf_ner = CRF(algorithm='lbfgs', max_iterations=25)\n",
        "\n",
        "#Fitting model on train data.\n",
        "\n",
        "crf_ner.fit(X,y)\n",
        "\n",
        "# prediction on test data\n",
        "\n",
        "test_prediction=crf_ner.predict(test_X)\n",
        "\n",
        "# get labels\n",
        "\n",
        "lbs=list(crf_ner.classes_)\n",
        "\n",
        "#get accuracy\n",
        "\n",
        "metrics.flat_f1_score(test_y, test_prediction, average= 'weighted', labels=lbs)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "_zhdTIpdxTdi",
        "outputId": "efb183fd-a159-4a06-e303-c953fc51429f"
      },
      "execution_count": 3,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "0.8369950502328535\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "#sort the labels\n",
        "\n",
        "sorted_lbs=sorted(lbs, key= lambda name:(name [1:], name[0]))\n",
        "\n",
        "#get classification report\n",
        "\n",
        "print(metrics.flat_classification_report(test_y, test_prediction, labels=sorted_lbs, digits=4))"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "pjLPvJDkxVLu",
        "outputId": "890ff569-db9e-4923-e1c3-b21425268e76"
      },
      "execution_count": 4,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "                    precision    recall  f1-score  support\n",
            "\n",
            "               O       0.8714    0.8423    0.8566    14143\n",
            "         B-Actor       0.9265    0.9207    0.9236     1274\n",
            "         I-Actor       0.9336    0.9227    0.9281     1553\n",
            "         B-Award       0.6923    0.6818    0.6870       66\n",
            "         I-Award       0.7941    0.7347    0.7633      147\n",
            "B-Character_Name       0.7283    0.4452    0.5526      283\n",
            "I-Character_Name       0.7453    0.5286    0.6186      227\n",
            "      B-Director       0.8575    0.8494    0.8534      425\n",
            "      I-Director       0.9116    0.8783    0.8947      411\n",
            "         B-Genre       0.8355    0.8048    0.8199      789\n",
            "         I-Genre       0.7853    0.7463    0.7653      544\n",
            "       B-Opinion       0.4867    0.3744    0.4232      195\n",
            "       I-Opinion       0.3333    0.1608    0.2170      143\n",
            "        B-Origin       0.4803    0.3842    0.4269      190\n",
            "        I-Origin       0.7088    0.6386    0.6719      808\n",
            "          B-Plot       0.4859    0.4705    0.4781     1577\n",
            "          I-Plot       0.8517    0.9318    0.8899    14661\n",
            "         B-Quote       0.8947    0.3617    0.5152       47\n",
            "         I-Quote       0.8315    0.4384    0.5741      349\n",
            "  B-Relationship       0.7812    0.5848    0.6689      171\n",
            "  I-Relationship       0.5665    0.4567    0.5057      289\n",
            "    B-Soundtrack       0.0000    0.0000    0.0000        8\n",
            "    I-Soundtrack       0.0000    0.0000    0.0000       30\n",
            "          B-Year       0.9683    0.9713    0.9698      661\n",
            "          I-Year       0.6098    0.5682    0.5882       44\n",
            "\n",
            "        accuracy                           0.8412    39035\n",
            "       macro avg       0.6832    0.5879    0.6237    39035\n",
            "    weighted avg       0.8370    0.8412    0.8370    39035\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "The test data result shows an overall F1 score of 0.8370 and an accuracy of 0.8412. The accuracy increased by nearly 40% by adding more features, and the F1 score increased by 0.5. We can further increase the accuracy by performing hyperparameter tuning.\n",
        "\n",
        "Let's take a random sentence and predict the tag using the trained model.\n",
        "\n",
        "1. Create feature maps, as we did for training data, for input sentences using the word2feature function.\n",
        "\n",
        "2. Convert into an array and predict using crf.predict(input_vector).\n",
        "\n",
        "Next, convert each word into features.\n"
      ],
      "metadata": {
        "id": "fKfN2E9sxxry"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "def text_2_ftr_sample(words):\n",
        "  return [text_to_features(words, i) for i in range(len(words))]\n",
        "\n",
        "# Split the sentences and convert each word into features.\n",
        "\n",
        "# define sample sentence\n",
        "\n",
        "X_sample=['alien invasion is the movie directed by christoper nollen'.split()]\n",
        "\n",
        "# convert to features\n",
        "\n",
        "X_sample=[text_2_ftr_sample(x) for x in X_sample]\n",
        "\n",
        "# predicting the class\n",
        "\n",
        "crf_ner.predict(X_sample1)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "WQ0WigcFxsvi",
        "outputId": "cfb45bc0-3b48-4047-f313-2a368060fa82"
      },
      "execution_count": 5,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[['B-Actor', 'I-Actor', 'O', 'O', 'O', 'O', 'O', 'B-Director','I-Director']] \n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "Now, let's try the BERT model and check if it performs better than the CRF model.\n",
        "\n",
        "# BERT Transformer\n",
        "\n",
        "BERT (Bidirectional Encoder Representations from Transformers) is a model that is trained on large data sets. This pretrained model can be fine-tuned as per the requirement and used for different tasks such as sentiment analysis, question answering system, sentence classification, and others. BERT transfers learning in NLP, and it is a state-of-the-art method.\n",
        "\n",
        "BERT uses transformers, mainly the encoder part. The attention mechanism learns the contextual relationship between words and subwords. Unlike other models, Transformer's encoder learns all sequences at once. The input is a sequence of words (tokens) that are embed into vectors and then proceed to the neural networks. The output is the sequence of tokens that corresponds to the input token of the given sequence.\n",
        "\n",
        "Let's implement the BERT model.\n"
      ],
      "metadata": {
        "id": "3y4FGvQXx_dh"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# importing necessary libraries\n",
        "\n",
        "from sklearn.model_selection import train_test_split\n",
        "\n",
        "from sklearn.metrics import accuracy_score\n",
        "\n",
        "from sklearn.metrics import classification_report, make_scorer\n",
        "\n",
        "# importing NER models from simple transformers\n",
        "\n",
        "!pip install simpletransformers\n",
        "\n",
        "from simpletransformers.ner import NERModel, NERArgs\n",
        "\n",
        "# importing libraries for evaluation\n",
        "\n",
        "from sklearn_crfsuite.metrics import flat_classification_report\n",
        "\n",
        "from sklearn_crfsuite import CRF, scorers, metrics"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "JQtuQMW2xVcV",
        "outputId": "2bbd6e46-daae-4a26-c623-92d63b1d49fa"
      },
      "execution_count": 6,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n",
            "Collecting simpletransformers\n",
            "  Downloading simpletransformers-0.63.11-py3-none-any.whl (250 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m250.7/250.7 kB\u001b[0m \u001b[31m9.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: numpy in /usr/local/lib/python3.10/dist-packages (from simpletransformers) (1.22.4)\n",
            "Requirement already satisfied: requests in /usr/local/lib/python3.10/dist-packages (from simpletransformers) (2.27.1)\n",
            "Requirement already satisfied: tqdm>=4.47.0 in /usr/local/lib/python3.10/dist-packages (from simpletransformers) (4.65.0)\n",
            "Requirement already satisfied: regex in /usr/local/lib/python3.10/dist-packages (from simpletransformers) (2022.10.31)\n",
            "Collecting transformers>=4.6.0 (from simpletransformers)\n",
            "  Downloading transformers-4.30.2-py3-none-any.whl (7.2 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m7.2/7.2 MB\u001b[0m \u001b[31m86.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting datasets (from simpletransformers)\n",
            "  Downloading datasets-2.13.1-py3-none-any.whl (486 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m486.2/486.2 kB\u001b[0m \u001b[31m43.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: scipy in /usr/local/lib/python3.10/dist-packages (from simpletransformers) (1.10.1)\n",
            "Requirement already satisfied: scikit-learn in /usr/local/lib/python3.10/dist-packages (from simpletransformers) (1.2.2)\n",
            "Collecting seqeval (from simpletransformers)\n",
            "  Downloading seqeval-1.2.2.tar.gz (43 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m43.6/43.6 kB\u001b[0m \u001b[31m4.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25h  Preparing metadata (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "Requirement already satisfied: tensorboard in /usr/local/lib/python3.10/dist-packages (from simpletransformers) (2.12.3)\n",
            "Requirement already satisfied: pandas in /usr/local/lib/python3.10/dist-packages (from simpletransformers) (1.5.3)\n",
            "Collecting tokenizers (from simpletransformers)\n",
            "  Downloading tokenizers-0.13.3-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (7.8 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m7.8/7.8 MB\u001b[0m \u001b[31m73.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting wandb>=0.10.32 (from simpletransformers)\n",
            "  Downloading wandb-0.15.4-py3-none-any.whl (2.1 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m2.1/2.1 MB\u001b[0m \u001b[31m57.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting streamlit (from simpletransformers)\n",
            "  Downloading streamlit-1.23.1-py2.py3-none-any.whl (8.9 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m8.9/8.9 MB\u001b[0m \u001b[31m70.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting sentencepiece (from simpletransformers)\n",
            "  Downloading sentencepiece-0.1.99-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (1.3 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.3/1.3 MB\u001b[0m \u001b[31m50.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: filelock in /usr/local/lib/python3.10/dist-packages (from transformers>=4.6.0->simpletransformers) (3.12.2)\n",
            "Collecting huggingface-hub<1.0,>=0.14.1 (from transformers>=4.6.0->simpletransformers)\n",
            "  Downloading huggingface_hub-0.15.1-py3-none-any.whl (236 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m236.8/236.8 kB\u001b[0m \u001b[31m25.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: packaging>=20.0 in /usr/local/lib/python3.10/dist-packages (from transformers>=4.6.0->simpletransformers) (23.1)\n",
            "Requirement already satisfied: pyyaml>=5.1 in /usr/local/lib/python3.10/dist-packages (from transformers>=4.6.0->simpletransformers) (6.0)\n",
            "Collecting safetensors>=0.3.1 (from transformers>=4.6.0->simpletransformers)\n",
            "  Downloading safetensors-0.3.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (1.3 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.3/1.3 MB\u001b[0m \u001b[31m77.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: Click!=8.0.0,>=7.0 in /usr/local/lib/python3.10/dist-packages (from wandb>=0.10.32->simpletransformers) (8.1.3)\n",
            "Collecting GitPython!=3.1.29,>=1.0.0 (from wandb>=0.10.32->simpletransformers)\n",
            "  Downloading GitPython-3.1.31-py3-none-any.whl (184 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m184.3/184.3 kB\u001b[0m \u001b[31m16.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: psutil>=5.0.0 in /usr/local/lib/python3.10/dist-packages (from wandb>=0.10.32->simpletransformers) (5.9.5)\n",
            "Collecting sentry-sdk>=1.0.0 (from wandb>=0.10.32->simpletransformers)\n",
            "  Downloading sentry_sdk-1.26.0-py2.py3-none-any.whl (209 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m209.4/209.4 kB\u001b[0m \u001b[31m21.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting docker-pycreds>=0.4.0 (from wandb>=0.10.32->simpletransformers)\n",
            "  Downloading docker_pycreds-0.4.0-py2.py3-none-any.whl (9.0 kB)\n",
            "Collecting pathtools (from wandb>=0.10.32->simpletransformers)\n",
            "  Downloading pathtools-0.1.2.tar.gz (11 kB)\n",
            "  Preparing metadata (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "Collecting setproctitle (from wandb>=0.10.32->simpletransformers)\n",
            "  Downloading setproctitle-1.3.2-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl (30 kB)\n",
            "Requirement already satisfied: setuptools in /usr/local/lib/python3.10/dist-packages (from wandb>=0.10.32->simpletransformers) (67.7.2)\n",
            "Requirement already satisfied: appdirs>=1.4.3 in /usr/local/lib/python3.10/dist-packages (from wandb>=0.10.32->simpletransformers) (1.4.4)\n",
            "Requirement already satisfied: protobuf!=4.21.0,<5,>=3.19.0 in /usr/local/lib/python3.10/dist-packages (from wandb>=0.10.32->simpletransformers) (3.20.3)\n",
            "Requirement already satisfied: urllib3<1.27,>=1.21.1 in /usr/local/lib/python3.10/dist-packages (from requests->simpletransformers) (1.26.16)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.10/dist-packages (from requests->simpletransformers) (2023.5.7)\n",
            "Requirement already satisfied: charset-normalizer~=2.0.0 in /usr/local/lib/python3.10/dist-packages (from requests->simpletransformers) (2.0.12)\n",
            "Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.10/dist-packages (from requests->simpletransformers) (3.4)\n",
            "Requirement already satisfied: pyarrow>=8.0.0 in /usr/local/lib/python3.10/dist-packages (from datasets->simpletransformers) (9.0.0)\n",
            "Collecting dill<0.3.7,>=0.3.0 (from datasets->simpletransformers)\n",
            "  Downloading dill-0.3.6-py3-none-any.whl (110 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m110.5/110.5 kB\u001b[0m \u001b[31m10.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting xxhash (from datasets->simpletransformers)\n",
            "  Downloading xxhash-3.2.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (212 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m212.5/212.5 kB\u001b[0m \u001b[31m22.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting multiprocess (from datasets->simpletransformers)\n",
            "  Downloading multiprocess-0.70.14-py310-none-any.whl (134 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m134.3/134.3 kB\u001b[0m \u001b[31m16.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: fsspec[http]>=2021.11.1 in /usr/local/lib/python3.10/dist-packages (from datasets->simpletransformers) (2023.6.0)\n",
            "Collecting aiohttp (from datasets->simpletransformers)\n",
            "  Downloading aiohttp-3.8.4-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (1.0 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.0/1.0 MB\u001b[0m \u001b[31m57.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: python-dateutil>=2.8.1 in /usr/local/lib/python3.10/dist-packages (from pandas->simpletransformers) (2.8.2)\n",
            "Requirement already satisfied: pytz>=2020.1 in /usr/local/lib/python3.10/dist-packages (from pandas->simpletransformers) (2022.7.1)\n",
            "Requirement already satisfied: joblib>=1.1.1 in /usr/local/lib/python3.10/dist-packages (from scikit-learn->simpletransformers) (1.2.0)\n",
            "Requirement already satisfied: threadpoolctl>=2.0.0 in /usr/local/lib/python3.10/dist-packages (from scikit-learn->simpletransformers) (3.1.0)\n",
            "Requirement already satisfied: altair<6,>=4.0 in /usr/local/lib/python3.10/dist-packages (from streamlit->simpletransformers) (4.2.2)\n",
            "Collecting blinker<2,>=1.0.0 (from streamlit->simpletransformers)\n",
            "  Downloading blinker-1.6.2-py3-none-any.whl (13 kB)\n",
            "Requirement already satisfied: cachetools<6,>=4.0 in /usr/local/lib/python3.10/dist-packages (from streamlit->simpletransformers) (5.3.1)\n",
            "Collecting importlib-metadata<7,>=1.4 (from streamlit->simpletransformers)\n",
            "  Downloading importlib_metadata-6.7.0-py3-none-any.whl (22 kB)\n",
            "Requirement already satisfied: pillow<10,>=6.2.0 in /usr/local/lib/python3.10/dist-packages (from streamlit->simpletransformers) (8.4.0)\n",
            "Collecting pympler<2,>=0.9 (from streamlit->simpletransformers)\n",
            "  Downloading Pympler-1.0.1-py3-none-any.whl (164 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m164.8/164.8 kB\u001b[0m \u001b[31m18.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: rich<14,>=10.11.0 in /usr/local/lib/python3.10/dist-packages (from streamlit->simpletransformers) (13.4.2)\n",
            "Requirement already satisfied: tenacity<9,>=8.0.0 in /usr/local/lib/python3.10/dist-packages (from streamlit->simpletransformers) (8.2.2)\n",
            "Requirement already satisfied: toml<2 in /usr/local/lib/python3.10/dist-packages (from streamlit->simpletransformers) (0.10.2)\n",
            "Requirement already satisfied: typing-extensions<5,>=4.0.1 in /usr/local/lib/python3.10/dist-packages (from streamlit->simpletransformers) (4.6.3)\n",
            "Collecting tzlocal<5,>=1.1 (from streamlit->simpletransformers)\n",
            "  Downloading tzlocal-4.3.1-py3-none-any.whl (20 kB)\n",
            "Collecting validators<1,>=0.2 (from streamlit->simpletransformers)\n",
            "  Downloading validators-0.20.0.tar.gz (30 kB)\n",
            "  Preparing metadata (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "Collecting pydeck<1,>=0.1.dev5 (from streamlit->simpletransformers)\n",
            "  Downloading pydeck-0.8.1b0-py2.py3-none-any.whl (4.8 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m4.8/4.8 MB\u001b[0m \u001b[31m83.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: tornado<7,>=6.0.3 in /usr/local/lib/python3.10/dist-packages (from streamlit->simpletransformers) (6.3.1)\n",
            "Collecting watchdog (from streamlit->simpletransformers)\n",
            "  Downloading watchdog-3.0.0-py3-none-manylinux2014_x86_64.whl (82 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m82.1/82.1 kB\u001b[0m \u001b[31m8.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: absl-py>=0.4 in /usr/local/lib/python3.10/dist-packages (from tensorboard->simpletransformers) (1.4.0)\n",
            "Requirement already satisfied: grpcio>=1.48.2 in /usr/local/lib/python3.10/dist-packages (from tensorboard->simpletransformers) (1.54.2)\n",
            "Requirement already satisfied: google-auth<3,>=1.6.3 in /usr/local/lib/python3.10/dist-packages (from tensorboard->simpletransformers) (2.17.3)\n",
            "Requirement already satisfied: google-auth-oauthlib<1.1,>=0.5 in /usr/local/lib/python3.10/dist-packages (from tensorboard->simpletransformers) (1.0.0)\n",
            "Requirement already satisfied: markdown>=2.6.8 in /usr/local/lib/python3.10/dist-packages (from tensorboard->simpletransformers) (3.4.3)\n",
            "Requirement already satisfied: tensorboard-data-server<0.8.0,>=0.7.0 in /usr/local/lib/python3.10/dist-packages (from tensorboard->simpletransformers) (0.7.1)\n",
            "Requirement already satisfied: werkzeug>=1.0.1 in /usr/local/lib/python3.10/dist-packages (from tensorboard->simpletransformers) (2.3.6)\n",
            "Requirement already satisfied: wheel>=0.26 in /usr/local/lib/python3.10/dist-packages (from tensorboard->simpletransformers) (0.40.0)\n",
            "Requirement already satisfied: entrypoints in /usr/local/lib/python3.10/dist-packages (from altair<6,>=4.0->streamlit->simpletransformers) (0.4)\n",
            "Requirement already satisfied: jinja2 in /usr/local/lib/python3.10/dist-packages (from altair<6,>=4.0->streamlit->simpletransformers) (3.1.2)\n",
            "Requirement already satisfied: jsonschema>=3.0 in /usr/local/lib/python3.10/dist-packages (from altair<6,>=4.0->streamlit->simpletransformers) (4.3.3)\n",
            "Requirement already satisfied: toolz in /usr/local/lib/python3.10/dist-packages (from altair<6,>=4.0->streamlit->simpletransformers) (0.12.0)\n",
            "Requirement already satisfied: six>=1.4.0 in /usr/local/lib/python3.10/dist-packages (from docker-pycreds>=0.4.0->wandb>=0.10.32->simpletransformers) (1.16.0)\n",
            "Requirement already satisfied: attrs>=17.3.0 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets->simpletransformers) (23.1.0)\n",
            "Collecting multidict<7.0,>=4.5 (from aiohttp->datasets->simpletransformers)\n",
            "  Downloading multidict-6.0.4-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (114 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m114.5/114.5 kB\u001b[0m \u001b[31m12.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting async-timeout<5.0,>=4.0.0a3 (from aiohttp->datasets->simpletransformers)\n",
            "  Downloading async_timeout-4.0.2-py3-none-any.whl (5.8 kB)\n",
            "Collecting yarl<2.0,>=1.0 (from aiohttp->datasets->simpletransformers)\n",
            "  Downloading yarl-1.9.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (268 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m268.8/268.8 kB\u001b[0m \u001b[31m19.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting frozenlist>=1.1.1 (from aiohttp->datasets->simpletransformers)\n",
            "  Downloading frozenlist-1.3.3-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl (149 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m149.6/149.6 kB\u001b[0m \u001b[31m14.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting aiosignal>=1.1.2 (from aiohttp->datasets->simpletransformers)\n",
            "  Downloading aiosignal-1.3.1-py3-none-any.whl (7.6 kB)\n",
            "Collecting gitdb<5,>=4.0.1 (from GitPython!=3.1.29,>=1.0.0->wandb>=0.10.32->simpletransformers)\n",
            "  Downloading gitdb-4.0.10-py3-none-any.whl (62 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m62.7/62.7 kB\u001b[0m \u001b[31m5.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: pyasn1-modules>=0.2.1 in /usr/local/lib/python3.10/dist-packages (from google-auth<3,>=1.6.3->tensorboard->simpletransformers) (0.3.0)\n",
            "Requirement already satisfied: rsa<5,>=3.1.4 in /usr/local/lib/python3.10/dist-packages (from google-auth<3,>=1.6.3->tensorboard->simpletransformers) (4.9)\n",
            "Requirement already satisfied: requests-oauthlib>=0.7.0 in /usr/local/lib/python3.10/dist-packages (from google-auth-oauthlib<1.1,>=0.5->tensorboard->simpletransformers) (1.3.1)\n",
            "Requirement already satisfied: zipp>=0.5 in /usr/local/lib/python3.10/dist-packages (from importlib-metadata<7,>=1.4->streamlit->simpletransformers) (3.15.0)\n",
            "Requirement already satisfied: markdown-it-py>=2.2.0 in /usr/local/lib/python3.10/dist-packages (from rich<14,>=10.11.0->streamlit->simpletransformers) (3.0.0)\n",
            "Requirement already satisfied: pygments<3.0.0,>=2.13.0 in /usr/local/lib/python3.10/dist-packages (from rich<14,>=10.11.0->streamlit->simpletransformers) (2.14.0)\n",
            "Collecting pytz-deprecation-shim (from tzlocal<5,>=1.1->streamlit->simpletransformers)\n",
            "  Downloading pytz_deprecation_shim-0.1.0.post0-py2.py3-none-any.whl (15 kB)\n",
            "Requirement already satisfied: decorator>=3.4.0 in /usr/local/lib/python3.10/dist-packages (from validators<1,>=0.2->streamlit->simpletransformers) (4.4.2)\n",
            "Requirement already satisfied: MarkupSafe>=2.1.1 in /usr/local/lib/python3.10/dist-packages (from werkzeug>=1.0.1->tensorboard->simpletransformers) (2.1.3)\n",
            "Collecting smmap<6,>=3.0.1 (from gitdb<5,>=4.0.1->GitPython!=3.1.29,>=1.0.0->wandb>=0.10.32->simpletransformers)\n",
            "  Downloading smmap-5.0.0-py3-none-any.whl (24 kB)\n",
            "Requirement already satisfied: pyrsistent!=0.17.0,!=0.17.1,!=0.17.2,>=0.14.0 in /usr/local/lib/python3.10/dist-packages (from jsonschema>=3.0->altair<6,>=4.0->streamlit->simpletransformers) (0.19.3)\n",
            "Requirement already satisfied: mdurl~=0.1 in /usr/local/lib/python3.10/dist-packages (from markdown-it-py>=2.2.0->rich<14,>=10.11.0->streamlit->simpletransformers) (0.1.2)\n",
            "Requirement already satisfied: pyasn1<0.6.0,>=0.4.6 in /usr/local/lib/python3.10/dist-packages (from pyasn1-modules>=0.2.1->google-auth<3,>=1.6.3->tensorboard->simpletransformers) (0.5.0)\n",
            "Requirement already satisfied: oauthlib>=3.0.0 in /usr/local/lib/python3.10/dist-packages (from requests-oauthlib>=0.7.0->google-auth-oauthlib<1.1,>=0.5->tensorboard->simpletransformers) (3.2.2)\n",
            "Collecting tzdata (from pytz-deprecation-shim->tzlocal<5,>=1.1->streamlit->simpletransformers)\n",
            "  Downloading tzdata-2023.3-py2.py3-none-any.whl (341 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m341.8/341.8 kB\u001b[0m \u001b[31m30.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hBuilding wheels for collected packages: seqeval, validators, pathtools\n",
            "  Building wheel for seqeval (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for seqeval: filename=seqeval-1.2.2-py3-none-any.whl size=16165 sha256=ec673ef57e947169de95dd0f6fa5b6cde40f5cdc4bc0b480525d933bbd70b74b\n",
            "  Stored in directory: /root/.cache/pip/wheels/1a/67/4a/ad4082dd7dfc30f2abfe4d80a2ed5926a506eb8a972b4767fa\n",
            "  Building wheel for validators (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for validators: filename=validators-0.20.0-py3-none-any.whl size=19579 sha256=35c7de4305992a124cee00821b4865aa326a89d40ef2c9be4ce3a5b6f00399ff\n",
            "  Stored in directory: /root/.cache/pip/wheels/f2/ed/dd/d3a556ad245ef9dc570c6bcd2f22886d17b0b408dd3bbb9ac3\n",
            "  Building wheel for pathtools (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for pathtools: filename=pathtools-0.1.2-py3-none-any.whl size=8791 sha256=2a04e8ed8500c9aa8bb9fe3f4f42cdd6aed6243c784aee9fc39c87ac78894057\n",
            "  Stored in directory: /root/.cache/pip/wheels/e7/f3/22/152153d6eb222ee7a56ff8617d80ee5207207a8c00a7aab794\n",
            "Successfully built seqeval validators pathtools\n",
            "Installing collected packages: tokenizers, sentencepiece, safetensors, pathtools, xxhash, watchdog, validators, tzdata, smmap, setproctitle, sentry-sdk, pympler, multidict, importlib-metadata, frozenlist, docker-pycreds, dill, blinker, async-timeout, yarl, pytz-deprecation-shim, pydeck, multiprocess, huggingface-hub, gitdb, aiosignal, tzlocal, transformers, seqeval, GitPython, aiohttp, wandb, streamlit, datasets, simpletransformers\n",
            "  Attempting uninstall: tzlocal\n",
            "    Found existing installation: tzlocal 5.0.1\n",
            "    Uninstalling tzlocal-5.0.1:\n",
            "      Successfully uninstalled tzlocal-5.0.1\n",
            "Successfully installed GitPython-3.1.31 aiohttp-3.8.4 aiosignal-1.3.1 async-timeout-4.0.2 blinker-1.6.2 datasets-2.13.1 dill-0.3.6 docker-pycreds-0.4.0 frozenlist-1.3.3 gitdb-4.0.10 huggingface-hub-0.15.1 importlib-metadata-6.7.0 multidict-6.0.4 multiprocess-0.70.14 pathtools-0.1.2 pydeck-0.8.1b0 pympler-1.0.1 pytz-deprecation-shim-0.1.0.post0 safetensors-0.3.1 sentencepiece-0.1.99 sentry-sdk-1.26.0 seqeval-1.2.2 setproctitle-1.3.2 simpletransformers-0.63.11 smmap-5.0.0 streamlit-1.23.1 tokenizers-0.13.3 transformers-4.30.2 tzdata-2023.3 tzlocal-4.3.1 validators-0.20.0 wandb-0.15.4 watchdog-3.0.0 xxhash-3.2.0 yarl-1.9.2\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "from sklearn.preprocessing import LabelEncoder\n",
        "# Let's encode the sentence column using Label Encoder.\n",
        "\n",
        "# encoding sentence values\n",
        "\n",
        "data[\"sentence_id\"] = LabelEncoder().fit_transform(data[\"sentence_id\"])\n",
        "\n",
        "data1[\"sentence_id\"] = LabelEncoder().fit_transform(data1[\"sentence_id\"])\n",
        "\n",
        "# Let's convert all labels into uppercase.\n",
        "\n",
        "#converting labels to upper string as it is required format\n",
        "\n",
        "data[\"labels\"] = data[\"labels\"].str.upper()\n",
        "\n",
        "data1[\"labels\"] = data[\"labels\"].str.upper()\n",
        "\n",
        "# Next, separate the train and test data\n",
        "\n",
        "X= data[[\"sentence_id\", \"words\"]]\n",
        "\n",
        "Y =data[\"labels\"]\n",
        "\n",
        "# Then, create a train and test data frame.\n",
        "\n",
        "# building up train and test data to dataframe\n",
        "\n",
        "ner_tr_dt = pd.DataFrame({\"sentence_id\": data[\"sentence_id\"], \"words\":data[\"words\"],\"labels\":data[\"labels\"]})\n",
        "\n",
        "test_data = pd.DataFrame({\"sentence_id\": data1[\"sentence_id\"], \"words\":data1[\"words\"],\"labels\":data1 [\"labels\"]})\n",
        "\n",
        "# Also, let's store the list of unique labels.\n",
        "\n",
        "#label values\n",
        "\n",
        "label = ner_tr_dt[\"labels\"].unique().tolist()"
      ],
      "metadata": {
        "id": "5rkrHG9fyIyZ"
      },
      "execution_count": 7,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "We need to fine-tune the BERT model so that we can use parameters. Here, we changed epochs number and batch size.\n",
        "\n",
        "To improve the model further, we can change other parameters as well.\n"
      ],
      "metadata": {
        "id": "9bjm1SBuzBF-"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "#fine tuning our model on custom data\n",
        "\n",
        "args = NERArgs()\n",
        "\n",
        "#set the # of epoch\n",
        "\n",
        "args.num_train_epochs = 2\n",
        "\n",
        "#learning rate\n",
        "\n",
        "args.learning_rate = 1e-6\n",
        "\n",
        "args.overwrite_output_dir =True\n",
        "\n",
        "#train and evaluation batch size\n",
        "\n",
        "args.train_batch_size = 6\n",
        "\n",
        "args.eval_batch_size = 6"
      ],
      "metadata": {
        "id": "bJRy2hS4yccF"
      },
      "execution_count": 8,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "#initializing the model\n",
        "\n",
        "Ner_bert_mdl= NERModel('bert', 'bert-base-cased', labels=label, args = args)\n",
        "\n",
        "#training our model\n",
        "\n",
        "Ner_bert_mdl.train_model(ner_tr_dt, eval_data = test_data, acc=accuracy_score)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "OY5J19bKzJ_r",
        "outputId": "3b5a14ce-1c0a-4d3c-d8b5-7153e9dead9a"
      },
      "execution_count": 9,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "100%|███████████████████████| 3/3 [00:08<00:00, 8.78it/s]\n",
            "\n",
            "Epoch 5 of 5: 100%|███████████████████████| 5/5 [07:38<00:00, 95.51it/s]\n",
            "\n",
            "Epochs 0/5. Running Loss: 0.4331: 100%|███████████████████████| 489/489 [01:20<00:00, 6.66it/s]\n",
            "\n",
            "Epochs 1/5. Running Loss: 0.4114: 100%|███████████████████████| 489/489 [01:21<00:00, 6.61it/s]\n",
            "\n",
            "Epochs 2/5. Running Loss: 0.2505: 100%|███████████████████████| 489/489 [01:21<00:00, 6.61it/s]\n",
            "\n",
            "Epochs 3/5. Running Loss: 0.1116: 100%|███████████████████████| 489/489 [01:21<00:00, 6.56it/s]\n",
            "\n",
            "Epochs 4/5. Running Loss: 0.0743: 100%|███████████████████████| 489/489 [01:45<00:00, 6.60it/s]\n",
            "\n",
            "(2445, 0.27327230073252584)\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "We can observe a loss of 0.27 in the final epoch"
      ],
      "metadata": {
        "id": "KlbGYxSvzqcC"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# function to store labels and words of each sentence in list\n",
        "\n",
        "class sent_generate(object):\n",
        "\n",
        "  def __init__(self, data):\n",
        "\n",
        "    self.n_sent = 1.0\n",
        "\n",
        "    self.data = data\n",
        "\n",
        "    self.empty = False\n",
        "\n",
        "    fn_group = lambda s: [(a, b) for a,b in zip(s[\"words\"].values.tolist(),s[\"labels\"].values.tolist())]\n",
        "\n",
        "    self.grouped = self.data.groupby(\"sentence_id\").apply(fn_group)\n",
        "\n",
        "    self.sentences = [x for x in self.grouped]\n",
        "\n",
        "# storing words and labels of each sentence in single list of train data\n",
        "\n",
        "Sent_get = sent_generate(ner_tr_dt)\n",
        "\n",
        "sentences = Sent_get.sentences\n",
        "\n",
        "print(sentences[0])\n",
        "\n",
        "# This is how a sentence will look like.\n",
        "\n",
        "def txt_2_lbs (sent):\n",
        "\n",
        "  return [label for token, label in sent]\n",
        "\n",
        "y_train_group = [txt_2_lbs(x) for x in sentences[:4]]"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "-7tjxkulzltR",
        "outputId": "1fd5f727-e3ee-48e5-9460-2d261316e2af"
      },
      "execution_count": 10,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            " [('steve', 'B-ACTOR'), ('mcqueen', 'I-ACTOR'), ('provided', 'O'), ('a', 'O'), ('thrilling', '8-PLOT')]\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Storing words and labels of each sentence in single list of test data\n",
        "\n",
        "Sent_get = sent_generate(test_data)\n",
        "\n",
        "Sentences = Sent_get.sentences\n",
        "\n",
        "#This is how a sentence will look like.\n",
        "\n",
        "print(sentences[0])\n",
        "\n",
        "def txt_2_lbs(sent):\n",
        "\n",
        "  return [label for token, label in sent]\n",
        "\n",
        "y_test = [txt_2_lbs(x) for x in sentences[:5]]"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "_I1HdYVI02mL",
        "outputId": "a9dac51f-548a-4368-d38c-3f85b68b071e"
      },
      "execution_count": 11,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[('1', 'O'), ('need', 'O'), ('that', 'O'), ('movie', 'O'), ('which', 'O')]\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Let's evaluate the model on test data.\n",
        "\n",
        "#evaluating on test data\n",
        "\n",
        "result, model_outputs, preds_list = Ner_bert_mdl.eval_model(test_data)\n",
        "\n",
        "#individual group report\n",
        "\n",
        "accu_rpt = flat_classification_report(y_pred=preds_list, y_true=y_test)\n",
        "\n",
        "print(accu_rpt)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "hH2bsouT1Efu",
        "outputId": "b5f98b8d-8361-4299-8d25-6623f2f0de8a"
      },
      "execution_count": 12,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "                  precision    recall  f1-score   support\n",
            "\n",
            "         B-Actor       0.95      0.97      0.96      1274\n",
            "         B-Award       0.68      0.70      0.69        66\n",
            "B-Character_Name       0.71      0.72      0.72       283\n",
            "      B-Director       0.88      0.92      0.90       425\n",
            "         B-Genre       0.83      0.85      0.84       789\n",
            "       B-Opinion       0.47      0.52      0.50       195\n",
            "        B-Origin       0.48      0.45      0.47       190\n",
            "          B-Plot       0.54      0.52      0.53      1577\n",
            "         B-Quote       0.86      0.81      0.84        47\n",
            "  B-Relationship       0.72      0.67      0.69       171\n",
            "    B-Soundtrack       0.44      0.50      0.47         8\n",
            "          B-Year       0.98      0.98      0.98       661\n",
            "         I-Actor       0.96      0.96      0.96      1553\n",
            "         I-Award       0.77      0.81      0.79       147\n",
            "I-Character_Name       0.69      0.68      0.69       227\n",
            "      I-Director       0.95      0.93      0.94       411\n",
            "         I-Genre       0.81      0.76      0.79       544\n",
            "       I-Opinion       0.38      0.20      0.26       143\n",
            "        I-Origin       0.70      0.75      0.72       808\n",
            "          I-Plot       0.92      0.94      0.93     14661\n",
            "         I-Quote       0.91      0.79      0.85       349\n",
            "  I-Relationship       0.63      0.57      0.60       289\n",
            "    I-Soundtrack       0.48      0.37      0.42        30\n",
            "          I-Year       0.72      0.82      0.77        44\n",
            "               O       0.90      0.88      0.88     14143\n",
            "\n",
            "        accuracy                           0.88     39035\n",
            "       macro avg       0.73      0.72      0.73     39035\n",
            "    weighted avg       0.88      0.88      0.88     39035\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "The accuracy of test data is 88%. Let's evaluate the model on train data also to see if there is any overfitting.\n"
      ],
      "metadata": {
        "id": "2zDhRn7r1cU0"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "#evaluating on train data\n",
        "\n",
        "result_train, model_outputs_train, preds_list_train = Ner_bert_mdl.eval_model(ner_tr_dt)\n",
        "\n",
        "#individual group report of train data\n",
        "\n",
        "report_train = flat_classification_report (y_pred=preds_list, y_true = y_train_group)\n",
        "\n",
        "print(report_train)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "duK3QXRQ1YTM",
        "outputId": "db59d0e6-d365-4643-f9fe-c627adb98ba9"
      },
      "execution_count": 18,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "                  precision    recall  f1-score   support\n",
            "\n",
            "         B-Actor       1.00      1.00      1.00      5010\n",
            "         B-Award       0.97      0.97      0.97       309\n",
            "B-Character_Name       0.99      0.99      0.99      1024\n",
            "      B-Director       0.94      0.99      0.97      1787\n",
            "         B-Genre       0.97      0.95      0.96      3384\n",
            "       B-Opinion       0.83      0.94      0.88       810\n",
            "        B-Origin       0.88      0.86      0.87       779\n",
            "          B-Plot       0.94      0.93      0.94      6468\n",
            "         B-Quote       0.99      0.95      0.97       126\n",
            "  B-Relationship       0.91      0.90      0.90       580\n",
            "    B-Soundtrack       0.92      0.92      0.92        50\n",
            "          B-Year       1.00      1.00      1.00      2702\n",
            "         I-Actor       1.00      1.00      1.00      6121\n",
            "         I-Award       0.96      0.99      0.97       719\n",
            "I-Character_Name       0.99      0.99      0.99       760\n",
            "      I-Director       1.00      1.00      1.00      1653\n",
            "         I-Genre       0.96      0.90      0.93      2283\n",
            "       I-Opinion       0.98      0.98      0.98       539\n",
            "        I-Origin       0.92      0.98      0.95      3340\n",
            "          I-Plot       1.00      0.99      1.00     62107\n",
            "         I-Quote       0.99      1.00      0.99       817\n",
            "  I-Relationship       0.94      0.97      0.95      1206\n",
            "    I-Soundtrack       0.97      0.98      0.98       158\n",
            "          I-Year       0.88      0.97      0.92       195\n",
            "               O       0.99      0.99      0.99     55895\n",
            "\n",
            "        accuracy                           0.98    158822\n",
            "       macro avg       0.96      0.97      0.96    158822\n",
            "    weighted avg       0.99      0.98      0.98    158822\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "The accuracy of the training data is 98%. Compared to the CRF model, the BERT model is performing great.\n",
        "\n",
        "Now, let's take a random sentence and predict the tag using the BERT model built."
      ],
      "metadata": {
        "id": "nCvT8dii2j04"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "prediction, model_output = Ner_bert_mdl.predict([\"aliens invading is movie by christoper nollen\"])\n",
        "\n",
        "# Here, the model generates predictions that tag given words from each sentence, and model_output are outputs generated by the model."
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "vf3awY4r2bU6",
        "outputId": "c6c42634-19f0-4a62-fbb0-a7dc003eb050"
      },
      "execution_count": 19,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[[{'aliens': 'B-PLOT'}, \n",
            "  {'invading': 'I-PLOT'}, \n",
            "  {'is': '0'}, \n",
            "  {'movie': '0'}, \n",
            "  {'by': '0'}, \n",
            "  {'christoper': 'B-DIRECTOR'}, \n",
            "  {'nollen': 'I-DIRECTOR'}]]\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Next Steps\n",
        "\n",
        "*   Add more features to the model example combination of words with gives a proper meaning for CRF model.\n",
        "\n",
        "*   The current implementation considers only two hyperparameters in the CV search for the CRF model; however, the CRF model offers more parameters that can be further tuned to improve the performance.\n",
        "\n",
        "*  Use an LSTM neural network and creating an ensemble model with CRF\n",
        "\n",
        "*  For the BERT model, we can add a CRF layer to get abhore efficient network as BERT finds efficient patterns between words.\n"
      ],
      "metadata": {
        "id": "U1j4J2O13N88"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Named Entity Recognition Using CRF and BERT ends here."
      ],
      "metadata": {
        "id": "qEo42xkV3tOj"
      }
    }
  ]
}