BERT_for_ABSA.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "BERT_for_ABSA.ipynb",
      "provenance": [],
      "collapsed_sections": [],
      "toc_visible": true,
      "include_colab_link": true
    },
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/LorenzoAgnolucci/BERT_for_ABSA/blob/master/BERT_for_ABSA.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "jNAX-Dsz9bYp"
      },
      "source": [
        "!pip install transformers"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "IinUWR8_AM1V"
      },
      "source": [
        "from google.colab import drive\n",
        "drive.mount('/gdrive', force_remount=True)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "fnQa1GPcoIOJ"
      },
      "source": [
        "\n",
        "\n",
        "---\n",
        "\n",
        "\n",
        "# BERT-pair\n",
        "\n",
        "\n",
        "---\n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ncOhC8Aoc6pp"
      },
      "source": [
        "#@title Choose a dataset and a task { run: \"auto\", display-mode: \"form\" }\n",
        "base_dir = \"/gdrive/MyDrive/Machine_Learning\" #@param {type:\"string\"}\n",
        "dataset_type = \"sentihood\" #@param [\"sentihood\", \"semeval2014\"]\n",
        "task = \"NLI_M\" #@param [\"QA_M\", \"NLI_M\", \"QA_B\", \"NLI_B\"]"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "dlOFFHdiGlSw"
      },
      "source": [
        "import pandas as pd\n",
        "import random\n",
        "\n",
        "\n",
        "if dataset_type == \"sentihood\":\n",
        "    id2label = {0: \"None\", 1: \"Positive\", 2: \"Negative\"}\n",
        "    label2id = {\"None\": 0, \"Positive\": 1, \"Negative\": 2}\n",
        "elif dataset_type == \"semeval2014\":\n",
        "    id2label = {0: \"positive\", 1: \"neutral\", 2: \"negative\", 3: \"conflict\", 4: \"none\"}\n",
        "    label2id = {\"positive\": 0, \"neutral\" : 1, \"negative\" : 2, \"conflict\": 3, \"none\": 4}\n",
        "\n",
        "if task.endswith(\"B\"):\n",
        "    num_classes = 2\n",
        "else:\n",
        "    if dataset_type == \"sentihood\":\n",
        "        num_classes = 3\n",
        "    elif dataset_type == \"semeval2014\":\n",
        "        num_classes = 5\n",
        "\n",
        "\n",
        "def get_dataset(path):\n",
        "    original_sentences = []\n",
        "    auxiliary_sentences = []\n",
        "    labels = []\n",
        "    data = pd.read_csv(path, header=0, sep=\"\\t\").values.tolist()\n",
        "    for row in data:\n",
        "        original_sentences.append(row[1])\n",
        "        auxiliary_sentences.append(row[2])\n",
        "        labels.append(row[3])\n",
        "    return original_sentences, auxiliary_sentences, labels\n",
        "\n",
        "\n",
        "\n",
        "train_original_sentences, train_auxiliary_sentences, train_labels = get_dataset(f\"{base_dir}/data/{dataset_type}/BERT-pair/train_{task}.csv\")\n",
        "if dataset_type == \"sentihood\":\n",
        "    val_original_sentences, val_auxiliary_sentences, val_labels = get_dataset(f\"{base_dir}/data/{dataset_type}/BERT-pair/dev_{task}.csv\")\n",
        "elif dataset_type == \"semeval2014\":\n",
        "    val_original_sentences, val_auxiliary_sentences, val_labels = get_dataset(f\"{base_dir}/data/{dataset_type}/BERT-pair/test_{task}.csv\")\n",
        "test_original_sentences, test_auxiliary_sentences, test_labels = get_dataset(f\"{base_dir}/data/{dataset_type}/BERT-pair/test_{task}.csv\")"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "MhieorAyUUhg"
      },
      "source": [
        "from transformers import BertTokenizer\n",
        "\n",
        "tokenizer = BertTokenizer.from_pretrained('bert-base-uncased')\n",
        "\n",
        "train_encodings = tokenizer(train_original_sentences, train_auxiliary_sentences, truncation=True, padding=True)\n",
        "val_encodings = tokenizer(val_original_sentences, val_auxiliary_sentences, truncation=True, padding=True)\n",
        "test_encodings = tokenizer(test_original_sentences, test_auxiliary_sentences, truncation=True, padding=True)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "sQV80Jd-gQBq"
      },
      "source": [
        "import torch\n",
        "\n",
        "class ABSA_Dataset(torch.utils.data.Dataset):\n",
        "    def __init__(self, encodings, labels):\n",
        "        self.encodings = encodings\n",
        "        self.labels = labels\n",
        "\n",
        "    def __getitem__(self, idx):\n",
        "        item = {key: torch.tensor(val[idx]) for key, val in self.encodings.items()}\n",
        "        item['labels'] = torch.tensor(self.labels[idx])\n",
        "        return item\n",
        "\n",
        "    def __len__(self):\n",
        "        return len(self.labels)\n",
        "\n",
        "train_dataset = ABSA_Dataset(train_encodings, train_labels)\n",
        "val_dataset = ABSA_Dataset(val_encodings, val_labels)\n",
        "test_dataset = ABSA_Dataset(test_encodings, test_labels)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "hLMesy14Re3T"
      },
      "source": [
        "import sys\n",
        "if base_dir not in sys.path:\n",
        "    sys.path.insert(0, f'{base_dir}/')\n",
        "import numpy as np\n",
        "from scipy.special import softmax\n",
        "import evaluation\n",
        "\n",
        "\n",
        "def get_test_labels(data_dir, dataset_type):\n",
        "    original_sentences = []\n",
        "    auxiliary_sentences = []\n",
        "    labels = []\n",
        "    data = pd.read_csv(f\"{data_dir}/{dataset_type}/BERT-pair/test_NLI_M.csv\", header=0, sep=\"\\t\").values.tolist()\n",
        "    for row in data:\n",
        "        labels.append(row[3])\n",
        "    return labels\n",
        "\n",
        "\n",
        "def get_predictions(data, task, dataset_type):\n",
        "    predicted_labels = []\n",
        "    scores = []\n",
        "    if task.endswith(\"B\"):\n",
        "        if dataset_type == \"sentihood\":\n",
        "            if task.endswith(\"B\"):\n",
        "                count_aspect_rows = 0\n",
        "                current_aspect_scores = []\n",
        "                for row in data:\n",
        "                    current_aspect_scores.append(row[2])\n",
        "                    count_aspect_rows += 1\n",
        "                    if count_aspect_rows % 3 == 0:\n",
        "                        sum_current_aspect_scores = np.sum(current_aspect_scores)\n",
        "                        current_aspect_scores = [score / sum_current_aspect_scores for score in current_aspect_scores]\n",
        "                        scores.append(current_aspect_scores)\n",
        "                        predicted_labels.append(np.argmax(current_aspect_scores))\n",
        "                        current_aspect_scores = []\n",
        "        elif dataset_type == \"semeval2014\":\n",
        "            if task.endswith(\"B\"):\n",
        "                count_aspect_rows = 0\n",
        "                current_aspect_scores = []\n",
        "                for row in data:\n",
        "                    current_aspect_scores.append(row[2])\n",
        "                    count_aspect_rows += 1\n",
        "                    if count_aspect_rows % 5 == 0:\n",
        "                        sum_current_aspect_scores = np.sum(current_aspect_scores)\n",
        "                        current_aspect_scores = [score / sum_current_aspect_scores for score in current_aspect_scores]\n",
        "                        scores.append(current_aspect_scores)\n",
        "                        predicted_labels.append(np.argmax(current_aspect_scores))\n",
        "                        current_aspect_scores = []\n",
        "    return predicted_labels, scores\n",
        "\n",
        "\n",
        "if dataset_type == \"sentihood\":\n",
        "    def compute_metrics(predictions):\n",
        "        scores = [softmax(prediction) for prediction in predictions[0]]\n",
        "        predicted_labels = [np.argmax(x) for x in scores]\n",
        "        if task.endswith(\"B\"):\n",
        "            data = np.insert(scores, 0, predicted_labels, axis=1)\n",
        "            predicted_labels, scores = get_predictions(data, task, dataset_type)\n",
        "        test_labels = get_test_labels(f\"{base_dir}/data\", dataset_type)\n",
        "        metrics = {}\n",
        "        metrics[\"strict_acc\"] = evaluation.compute_sentihood_aspect_strict_accuracy(test_labels, predicted_labels)\n",
        "        metrics[\"F1\"] = evaluation.compute_sentihood_aspect_macro_F1(test_labels, predicted_labels)\n",
        "        metrics[\"aspect_AUC\"] = evaluation.compute_sentihood_aspect_macro_AUC(test_labels, scores)\n",
        "        sentiment_macro_AUC, sentiment_accuracy = evaluation.compute_sentihood_sentiment_classification_metrics(test_labels, scores)\n",
        "        metrics[\"sentiment_acc\"] = sentiment_accuracy\n",
        "        metrics[\"sentiment_AUC\"] = sentiment_macro_AUC\n",
        "        return metrics\n",
        "\n",
        "elif dataset_type == \"semeval2014\":\n",
        "    def compute_metrics(predictions):\n",
        "        scores = [softmax(prediction) for prediction in predictions[0]]\n",
        "        predicted_labels = [np.argmax(x) for x in scores]\n",
        "        if task.endswith(\"B\"):\n",
        "            data = np.insert(scores, 0, predicted_labels, axis=1)\n",
        "            predicted_labels, scores = get_predictions(data, task, dataset_type)\n",
        "        test_labels = get_test_labels(f\"{base_dir}/data\", dataset_type)\n",
        "        metrics = {}\n",
        "        p, r, f1 = evaluation.compute_semeval_PRF(test_labels, predicted_labels)\n",
        "        metrics[\"P\"] = p\n",
        "        metrics[\"R\"] = r\n",
        "        metrics[\"F1\"] = f1\n",
        "        metrics[\"4-way\"] = evaluation.compute_semeval_accuracy(test_labels, predicted_labels, scores, 4)\n",
        "        metrics[\"3-way\"] = evaluation.compute_semeval_accuracy(test_labels, predicted_labels, scores, 3)\n",
        "        metrics[\"binary\"] = evaluation.compute_semeval_accuracy(test_labels, predicted_labels, scores, 2)\n",
        "        return metrics"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "0TNnE3UesI1N"
      },
      "source": [
        "from transformers import BertForSequenceClassification, Trainer, TrainingArguments, BertConfig\n",
        "\n",
        "from transformers import logging\n",
        "logging.set_verbosity_debug()\n",
        "\n",
        "\n",
        "epochs = 4\n",
        "batch_size = 24\n",
        "num_steps = len(train_dataset) * epochs // batch_size\n",
        "warmup_steps = num_steps // 10  # 10% of the training steps\n",
        "save_steps = num_steps // epochs    # Save a checkpoint at the end of each epoch\n",
        "\n",
        "\n",
        "training_args = TrainingArguments(\n",
        "    output_dir = f'{base_dir}/models/{dataset_type}/BERT-pair/{task}/',          \n",
        "    num_train_epochs = epochs,              \n",
        "    per_device_train_batch_size = batch_size,  \n",
        "    per_device_eval_batch_size = batch_size,   \n",
        "    warmup_steps = warmup_steps,   \n",
        "    weight_decay = 0.01,               \n",
        "    logging_dir = f'{base_dir}/logs/{dataset_type}/BERT-pair/{task}/',            \n",
        "    logging_steps = 10,\n",
        "    evaluation_strategy = 'epoch',\n",
        "    learning_rate = 2e-5,\n",
        "    save_steps = save_steps\n",
        ")\n",
        "\n",
        "config = BertConfig.from_pretrained(\n",
        "    'bert-base-uncased',\n",
        "    architectures = ['BertForSequenceClassification'],\n",
        "    hidden_size = 768,\n",
        "    num_hidden_layers = 12,\n",
        "    num_attention_heads = 12,\n",
        "    hidden_dropout_prob = 0.1,\n",
        "    num_labels = num_classes\n",
        ")    \n",
        "\n",
        "load_finetuned_model = True\n",
        "if not load_finetuned_model:\n",
        "    model = BertForSequenceClassification.from_pretrained('bert-base-uncased', config=config)\n",
        "\n",
        "    trainer = Trainer(\n",
        "        model=model,                         \n",
        "        args=training_args,                  \n",
        "        train_dataset=train_dataset,         \n",
        "        eval_dataset=val_dataset,\n",
        "        compute_metrics=compute_metrics             \n",
        "    )\n",
        "    trainer.train()\n",
        "\n",
        "    model.save_pretrained(f\"{base_dir}/models/{dataset_type}/BERT-pair/{task}/last_step\")\n",
        "\n",
        "else:\n",
        "    model = BertForSequenceClassification.from_pretrained(f\"{base_dir}/models/{dataset_type}/BERT-pair/{task}/last_step\")\n",
        "\n",
        "    trainer = Trainer(\n",
        "        model=model,                         \n",
        "        args=training_args,                  \n",
        "        train_dataset=train_dataset,         \n",
        "        eval_dataset=val_dataset,\n",
        "        compute_metrics=compute_metrics             \n",
        "    )\n"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "3_33_i5bR7tS"
      },
      "source": [
        "evaluation_result = trainer.evaluate(test_dataset)\n",
        "print(evaluation_result)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "fOr0LtJdbPjM"
      },
      "source": [
        "import pandas as pd\n",
        "\n",
        "\n",
        "results = trainer.predict(test_dataset)\n",
        "\n",
        "scores = [softmax(prediction) for prediction in results.predictions]\n",
        "predicted_labels = [np.argmax(x) for x in scores]"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "KUgiS6XM-lNE"
      },
      "source": [
        "csv_output = np.insert(scores, 0, predicted_labels, axis=1)\n",
        "df = pd.DataFrame(csv_output)\n",
        "df[0] = df[0].astype(\"int\")\n",
        "if task.endswith(\"B\"):\n",
        "    header = [\"predicted_label\", \"no\", \"yes\"]\n",
        "else:\n",
        "    header = [\"predicted_label\"]\n",
        "    for label in label2id.keys():\n",
        "        header.append(label)\n",
        "df.to_csv(f\"{base_dir}/results/{dataset_type}/BERT-pair/{task}.csv\", index=False, header=header)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "7t8j0FJWhV6R"
      },
      "source": [
        "import sys\n",
        "if base_dir not in sys.path:\n",
        "    sys.path.insert(0, f'{base_dir}/')\n",
        "import evaluation\n",
        "evaluation.main(task, dataset_type, f\"{base_dir}/data\", f\"{base_dir}/results\")"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "9DVoml__ohye"
      },
      "source": [
        "\n",
        "\n",
        "---\n",
        "\n",
        "\n",
        "# BERT-single\n",
        "\n",
        "\n",
        "---\n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Us9ezWVjqKBd"
      },
      "source": [
        "#@title Choose a dataset and a task { run: \"auto\", display-mode: \"form\" }\n",
        "base_dir = \"/gdrive/MyDrive/Machine_Learning\" #@param {type:\"string\"}\n",
        "dataset_type = \"semeval2014\" #@param [\"sentihood\", \"semeval2014\"]\n",
        "task = \"single\" #@param [\"single\"]"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "huv7NHH0yh5f"
      },
      "source": [
        "import pandas as pd\n",
        "import random\n",
        "\n",
        "\n",
        "if dataset_type == \"sentihood\":\n",
        "    id2label = {0: \"None\", 1: \"Positive\", 2: \"Negative\"}\n",
        "    label2id = {\"None\": 0, \"Positive\": 1, \"Negative\": 2}\n",
        "elif dataset_type == \"semeval2014\":\n",
        "    id2label = {0: \"positive\", 1: \"neutral\", 2: \"negative\", 3: \"conflict\", 4: \"none\"}\n",
        "    label2id = {\"positive\": 0, \"neutral\" : 1, \"negative\" : 2, \"conflict\": 3, \"none\": 4}\n",
        "\n",
        "if dataset_type == \"sentihood\":\n",
        "    num_classes = 3\n",
        "    locations = [\"location_1_\", \"location_2_\"]\n",
        "    aspects = [\"general\", \"price\", \"safety\", \"transit location\"]\n",
        "elif dataset_type == \"semeval2014\":\n",
        "    num_classes = 5\n",
        "    locations = [\"\"]\n",
        "    aspects = [\"ambience\", \"anecdotes\", \"food\", \"price\", \"service\"]\n",
        "\n",
        "\n",
        "def get_dataset(path):\n",
        "    original_sentences = []\n",
        "    labels = []\n",
        "    data = pd.read_csv(path, header=0, sep=\"\\t\").values.tolist()\n",
        "    for row in data:\n",
        "        original_sentences.append(row[1])\n",
        "        labels.append(row[3])\n",
        "    return original_sentences, labels\n",
        "\n",
        "\n",
        "train_original_sentences = {}\n",
        "train_labels = {}\n",
        "val_original_sentences = {}\n",
        "val_labels = {}\n",
        "test_original_sentences = {}\n",
        "test_labels = {}\n",
        "\n",
        "for location in locations:\n",
        "    train_original_sentences[location] = {}\n",
        "    train_labels[location] = {}\n",
        "    val_original_sentences[location] = {}\n",
        "    val_labels[location] = {}\n",
        "    test_original_sentences[location] = {}\n",
        "    test_labels[location] = {}\n",
        "    for aspect in aspects:\n",
        "        train_original_sentences[location][aspect], train_labels[location][aspect] = get_dataset(f\"{base_dir}/data/{dataset_type}/BERT-single/{location}{aspect}/train.csv\")\n",
        "        if dataset_type == \"sentihood\":\n",
        "            val_original_sentences[location][aspect], val_labels[location][aspect] = get_dataset(f\"{base_dir}/data/{dataset_type}/BERT-single/{location}{aspect}/dev.csv\")\n",
        "        elif dataset_type == \"semeval2014\":\n",
        "            val_original_sentences[location][aspect], val_labels[location][aspect] = get_dataset(f\"{base_dir}/data/{dataset_type}/BERT-single/{location}{aspect}/test.csv\")\n",
        "        test_original_sentences[location][aspect], test_labels[location][aspect] = get_dataset(f\"{base_dir}/data/{dataset_type}/BERT-single/{location}{aspect}/test.csv\")"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "smEGjmdiy90j"
      },
      "source": [
        "from transformers import BertTokenizer\n",
        "\n",
        "tokenizer = BertTokenizer.from_pretrained('bert-base-uncased')\n",
        "\n",
        "train_encodings = {}\n",
        "val_encodings = {}\n",
        "test_encodings = {}\n",
        "for location in locations:\n",
        "    train_encodings[location] = {}\n",
        "    val_encodings[location] = {}\n",
        "    test_encodings[location] = {}\n",
        "    for aspect in aspects:\n",
        "        train_encodings[location][aspect] = tokenizer(train_original_sentences[location][aspect], truncation=True, padding=True)\n",
        "        val_encodings[location][aspect] = tokenizer(val_original_sentences[location][aspect], truncation=True, padding=True)\n",
        "        test_encodings[location][aspect] = tokenizer(test_original_sentences[location][aspect], truncation=True, padding=True)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "I_FhmCkKzGDs"
      },
      "source": [
        "import torch\n",
        "\n",
        "class ABSA_Dataset(torch.utils.data.Dataset):\n",
        "    def __init__(self, encodings, labels):\n",
        "        self.encodings = encodings\n",
        "        self.labels = labels\n",
        "\n",
        "    def __getitem__(self, idx):\n",
        "        item = {key: torch.tensor(val[idx]) for key, val in self.encodings.items()}\n",
        "        item['labels'] = torch.tensor(self.labels[idx])\n",
        "        return item\n",
        "\n",
        "    def __len__(self):\n",
        "        return len(self.labels)\n",
        "\n",
        "\n",
        "train_dataset = {}\n",
        "val_dataset = {}\n",
        "test_dataset = {}\n",
        "for location in locations:\n",
        "    train_dataset[location] = {}\n",
        "    val_dataset[location] = {}\n",
        "    test_dataset[location] = {}\n",
        "    for aspect in aspects:\n",
        "        train_dataset[location][aspect] = ABSA_Dataset(train_encodings[location][aspect], train_labels[location][aspect])\n",
        "        val_dataset[location][aspect] = ABSA_Dataset(val_encodings[location][aspect], val_labels[location][aspect])\n",
        "        test_dataset[location][aspect] = ABSA_Dataset(test_encodings[location][aspect], test_labels[location][aspect])"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "zubkNmxFzLTy"
      },
      "source": [
        "from transformers import BertForSequenceClassification, Trainer, TrainingArguments, BertConfig\n",
        "from transformers import logging\n",
        "import gc\n",
        "import pandas as pd\n",
        "import numpy as np\n",
        "from scipy.special import softmax\n",
        "\n",
        "\n",
        "logging.set_verbosity_debug()\n",
        "\n",
        "\n",
        "epochs = 4\n",
        "batch_size = 24\n",
        "\n",
        "header = [\"predicted_label\"]\n",
        "for label in label2id.keys():\n",
        "    header.append(label)\n",
        "\n",
        "config = BertConfig.from_pretrained(\n",
        "        'bert-base-uncased',\n",
        "        architectures = ['BertForSequenceClassification'],\n",
        "        hidden_size = 768,\n",
        "        num_hidden_layers = 12,\n",
        "        num_attention_heads = 12,\n",
        "        hidden_dropout_prob = 0.1,\n",
        "        num_labels = num_classes\n",
        "    )    \n",
        "\n",
        "for location in locations:\n",
        "    for aspect in aspects:\n",
        "        num_steps = len(train_dataset[location][aspect]) * epochs // batch_size\n",
        "        warmup_steps = num_steps // 10  # 10% of the training steps\n",
        "        save_steps = num_steps // epochs    # Save a checkpoint at the end of each epoch\n",
        "\n",
        "\n",
        "        training_args = TrainingArguments(\n",
        "            output_dir = f'{base_dir}/models/{dataset_type}/BERT-single/{location}{aspect}/',          \n",
        "            num_train_epochs = epochs,              \n",
        "            per_device_train_batch_size = batch_size,  \n",
        "            per_device_eval_batch_size = batch_size,   \n",
        "            warmup_steps = warmup_steps,   \n",
        "            weight_decay = 0.01,               \n",
        "            logging_dir = f'{base_dir}/logs/{dataset_type}/BERT-single/{location}{aspect}/',            \n",
        "            logging_steps = 10,\n",
        "            evaluation_strategy = 'epoch',\n",
        "            learning_rate = 2e-5,\n",
        "            save_steps = save_steps,\n",
        "            seed=21\n",
        "        )\n",
        "\n",
        "        model = BertForSequenceClassification.from_pretrained('bert-base-uncased', config=config)\n",
        "\n",
        "        trainer = Trainer(\n",
        "            model=model,                         \n",
        "            args=training_args,                  \n",
        "            train_dataset=train_dataset[location][aspect],         \n",
        "            eval_dataset=val_dataset[location][aspect]             \n",
        "        )\n",
        "\n",
        "        trainer.train()\n",
        "\n",
        "        model.save_pretrained(f\"{base_dir}/models/{dataset_type}/BERT-single/{location}{aspect}/last_step\")\n",
        "\n",
        "        results = trainer.predict(test_dataset[location][aspect])\n",
        "\n",
        "        scores = [softmax(prediction) for prediction in results.predictions]\n",
        "        predicted_labels = [np.argmax(x) for x in scores]\n",
        "\n",
        "        csv_output = np.insert(scores, 0, predicted_labels, axis=1)\n",
        "        df = pd.DataFrame(csv_output)\n",
        "        df[0] = df[0].astype(\"int\")\n",
        "        df.to_csv(f\"{base_dir}/results/{dataset_type}/BERT-single/{location}{aspect}.csv\", index=False, header=header)\n",
        "\n",
        "        del training_args\n",
        "        del model\n",
        "        del trainer\n",
        "        del results\n",
        "        del scores\n",
        "        del predicted_labels\n",
        "        del csv_output\n",
        "        del df\n",
        "        gc.collect()\n"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "2XE4Ts6ezgaT"
      },
      "source": [
        "import sys\r\n",
        "if base_dir not in sys.path:\r\n",
        "    sys.path.insert(0, f'{base_dir}/')\r\n",
        "import evaluation\r\n",
        "\r\n",
        "\r\n",
        "evaluation.main(task, dataset_type, f\"{base_dir}/data\", f\"{base_dir}/results\")"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "epQjGhtGlntP"
      },
      "source": [
        "# Paper Code"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "wZZAWOVjhRpO"
      },
      "source": [
        "import nltk\n",
        "nltk.download('punkt')"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ltVH-Do9clTB"
      },
      "source": [
        "!git clone https://github.com/HSLCY/ABSA-BERT-pair.git\n",
        "%cd ABSA-BERT-pair/\n",
        "%cd generate/\n",
        "!bash make.sh sentihood\n",
        "!bash make.sh semeval"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "PKIL8rhMdRGo"
      },
      "source": [
        "%cd ../\n",
        "!wget https://storage.googleapis.com/bert_models/2020_02_20/uncased_L-12_H-768_A-12.zip\n",
        "!unzip -d uncased_L-12_H-768_A-12 uncased_L-12_H-768_A-12.zip"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "sMVmQLUhiPid"
      },
      "source": [
        "!python convert_tf_checkpoint_to_pytorch.py \\\n",
        "--tf_checkpoint_path uncased_L-12_H-768_A-12/bert_model.ckpt \\\n",
        "--bert_config_file uncased_L-12_H-768_A-12/bert_config.json \\\n",
        "--pytorch_dump_path uncased_L-12_H-768_A-12/pytorch_model.bin"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "oKj6RmGCdTse"
      },
      "source": [
        "!CUDA_VISIBLE_DEVICES=0 python run_classifier_TABSA.py \\\n",
        "--task_name sentihood_QA_M \\\n",
        "--data_dir data/sentihood/bert-pair/ \\\n",
        "--vocab_file uncased_L-12_H-768_A-12/vocab.txt \\\n",
        "--bert_config_file uncased_L-12_H-768_A-12/bert_config.json \\\n",
        "--init_checkpoint uncased_L-12_H-768_A-12/pytorch_model.bin \\\n",
        "--eval_test \\\n",
        "--do_lower_case \\\n",
        "--max_seq_length 512 \\\n",
        "--train_batch_size 8 \\\n",
        "--learning_rate 2e-5 \\\n",
        "--num_train_epochs 4.0 \\\n",
        "--output_dir results/ \\\n",
        "--seed 42"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Zdf5UcHUdkYT"
      },
      "source": [
        "!python evaluation.py --task_name sentihood_QA_M --pred_data_dir results/test_ep_4.txt"
      ],
      "execution_count": null,
      "outputs": []
    }
  ]
}