multi_label_BERT.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "d86c949e",
   "metadata": {},
   "source": [
    "# Experiment 1: Multi-label classification with BERT (~ 5 hours)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "7f9b0d48",
   "metadata": {},
   "source": [
    "## Import libraries "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "cb374804",
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "import pandas as pd\n",
    "import evaluate\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "from sklearn.metrics import f1_score, accuracy_score, roc_auc_score\n",
    "from transformers import BertModel, AutoTokenizer, AutoConfig, EvalPrediction \n",
    "from transformers import TrainingArguments, Trainer,TrainerCallback,BertForSequenceClassification\n",
    "from datasets import load_dataset, ClassLabel, Value, load_metric, load_from_disk,ReadInstruction\n",
    "from pytorch_transformers import WEIGHTS_NAME, CONFIG_NAME\n",
    "from sklearn.preprocessing import LabelBinarizer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "3f786524",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_train= load_dataset(\"csv\", data_files=\"/Users/ruochentan1/PycharmProjects/bert/df_train.csv\",\n",
    "                       split=\"train\") # already did one hot encoding of the rating column  \n",
    "df_valid = load_dataset(\"csv\", data_files=\"/Users/ruochentan1/PycharmProjects/bert/df_valid.csv\",\n",
    "                       split=\"train\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "cc29d96d",
   "metadata": {},
   "outputs": [],
   "source": [
    "# remove unuseful columns \n",
    "train = df_train.remove_columns(['Unnamed: 0', 'unixReviewTime',\"reviewTime\"])\n",
    "valid = df_valid.remove_columns(['Unnamed: 0','unixReviewTime',\"reviewTime\"])\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "dfb23b3b",
   "metadata": {},
   "outputs": [],
   "source": [
    "# extract a list of label \n",
    "labels = [label for label in train.features.keys() if label not in ['Rating', \"UserId\",\"ItemId\",\"reviewText\"]]\n",
    "id2label = {idx:label for idx, label in enumerate(labels)}\n",
    "label2id = {label:idx for idx, label in enumerate(labels)}\n",
    "labels"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "9b0d46d0",
   "metadata": {},
   "source": [
    "## Data encoding "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0e367afd",
   "metadata": {},
   "outputs": [],
   "source": [
    "tokenizer = AutoTokenizer.from_pretrained(\"bert-base-uncased\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c30c2cf0",
   "metadata": {},
   "outputs": [],
   "source": [
    "def preprocess_data(examples):\n",
    "    # take a batch of texts\n",
    "    text = examples[\"reviewText\"]\n",
    "    # encode them\n",
    "    encoding = tokenizer(text, padding=\"max_length\", truncation=True, max_length=128)\n",
    "    # add labels\n",
    "    labels_batch = {k: examples[k] for k in examples.keys() if k in labels}\n",
    "    # create numpy array of shape (batch_size, num_labels)\n",
    "    labels_matrix = np.zeros((len(text), len(labels)))\n",
    "    # fill numpy array\n",
    "    for idx, label in enumerate(labels):\n",
    "        labels_matrix[:, idx] = labels_batch[label]\n",
    "\n",
    "    encoding[\"labels\"] = labels_matrix.tolist()\n",
    "\n",
    "    return encoding"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "159c8571",
   "metadata": {},
   "outputs": [],
   "source": [
    "encoded_train = train.map(preprocess_data, batched=True, remove_columns=train.column_names)\n",
    "encoded_valid = valid.map(preprocess_data,batched =True, remove_columns = valid.column_names)\n",
    "tokenizer.decode(example['input_ids'])\n",
    "encoded_train.set_format(\"torch\")\n",
    "encoded_valid.set_format(\"torch\")\n",
    "encoded_train.save_to_disk(\"train_tokenized\")\n",
    "encoded_valid.save_to_disk(\"valid_tokenized\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "92ab3647",
   "metadata": {},
   "source": [
    "## Define Model and Training "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "5378c940",
   "metadata": {},
   "outputs": [],
   "source": [
    "model = BertForSequenceClassification.from_pretrained(\"bert-base-uncased\", problem_type=\"multi_label_classification\",\n",
    "                                                           num_labels=len(labels),\n",
    "                                                           id2label=id2label,\n",
    "                                                           label2id=label2id)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6da77093",
   "metadata": {},
   "outputs": [],
   "source": [
    "def multi_label_metrics(predictions, labels, threshold=0.5):\n",
    "    # first, apply sigmoid on predictions which are of shape (batch_size, num_labels)\n",
    "    sigmoid = torch.nn.Sigmoid()\n",
    "    probs = sigmoid(torch.Tensor(predictions))\n",
    "    # next, use threshold to turn them into integer predictions\n",
    "    y_pred = np.zeros(probs.shape)\n",
    "    y_pred[np.where(probs >= threshold)] = 1\n",
    "    # finally, compute metrics\n",
    "    y_true = labels\n",
    "    f1_micro_average = f1_score(y_true=y_true, y_pred=y_pred, average='weighted')\n",
    "    roc_auc = roc_auc_score(y_true, y_pred, average='weighted')\n",
    "    accuracy = accuracy_score(y_true, y_pred)\n",
    "    # return as dictionary\n",
    "    metrics = {'f1': f1_micro_average,\n",
    "               'roc_auc': roc_auc,\n",
    "               'accuracy': accuracy}\n",
    "    return metrics"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "de955d53",
   "metadata": {},
   "outputs": [],
   "source": [
    "def compute_metrics(p: EvalPrediction):\n",
    "    preds = p.predictions[0] if isinstance(p.predictions,\n",
    "                                           tuple) else p.predictions\n",
    "    result = multi_label_metrics(\n",
    "        predictions=preds,\n",
    "        labels=p.label_ids)\n",
    "    return result"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4c543780",
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "os.environ[\"TOKENIZERS_PARALLELISM\"] = \"false\" # warning "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0f905619",
   "metadata": {},
   "outputs": [],
   "source": [
    "training_args = TrainingArguments(\n",
    "    evaluation_strategy = \"epoch\", # show eval loss in each epoch\n",
    "    logging_strategy=\"epoch\", # show validation?\n",
    "    save_strategy = \"epoch\",\n",
    "    learning_rate=2e-5,\n",
    "    per_device_train_batch_size=16,\n",
    "    per_device_eval_batch_size=64,\n",
    "    num_train_epochs=4,\n",
    "    weight_decay=0.01,\n",
    "    metric_for_best_model=\"f1\",\n",
    "    load_best_model_at_end=True,\n",
    "    seed = 1998,\n",
    "     output_dir=\"/Users/ruochentan1/PycharmProjects/bert/\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6e3eb8d9",
   "metadata": {},
   "outputs": [],
   "source": [
    "trainer = Trainer(\n",
    "    model,\n",
    "    training_args,\n",
    "    train_dataset=encoded_train,\n",
    "    eval_dataset= encoded_valid,\n",
    "    tokenizer=tokenizer,\n",
    "    compute_metrics=compute_metrics\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f0e241c1",
   "metadata": {},
   "outputs": [],
   "source": [
    "class CustomCallback(TrainerCallback):\n",
    "\n",
    "    def __init__(self, trainer) -> None:\n",
    "        super().__init__()\n",
    "        self._trainer = trainer\n",
    "\n",
    "    def on_epoch_end(self, args, state, control, **kwargs):\n",
    "        if control.should_evaluate:\n",
    "            control_copy = copy.deepcopy(control)\n",
    "            self._trainer.evaluate(eval_dataset=self._trainer.train_dataset, metric_key_prefix=\"train\")\n",
    "            return control_copy # initiate callback to get the training evaluation \n",
    "trainer.add_callback(CustomCallback(trainer))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "74b7123b",
   "metadata": {},
   "outputs": [],
   "source": [
    "trainer.train()\n",
    "trainer.state.log_history # sum all history of training loss and metrics\n",
    "trainer.evaluate() # evaluate on trianning dataset"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "8ca0f90e",
   "metadata": {},
   "source": [
    "## Save the fine-tuned model "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "60d8b6f9",
   "metadata": {},
   "outputs": [],
   "source": [
    "output_dir = \"/Users/ruochentan1/PycharmProjects/bert\"\n",
    "\n",
    "# Step 1: Save a model, configuration and vocabulary that you have fine-tuned\n",
    "\n",
    "# If we have a distributed model, save only the encapsulated model\n",
    "# (it was wrapped in PyTorch DistributedDataParallel or DataParallel)\n",
    "model_to_save = model.module if hasattr(model, 'module') else model\n",
    "\n",
    "# If we save using the predefined names, we can load using `from_pretrained`\n",
    "output_model_file = os.path.join(output_dir, WEIGHTS_NAME)\n",
    "output_config_file = os.path.join(output_dir, CONFIG_NAME)\n",
    "\n",
    "torch.save(model_to_save.state_dict(), output_model_file)\n",
    "model_to_save.config.to_json_file(output_config_file)\n",
    "tokenizer.save_vocabulary(output_dir)\n",
    "\n",
    "# Step 2: Re-load the saved model and vocabulary\n",
    "\n",
    "# Example for a Bert model\n",
    "output_dir = \"/Users/ruochentan1/PycharmProjects/bert\"\n",
    "model = BertForSequenceClassification.from_pretrained(output_dir)\n",
    "tokenizer = BertTokenizer.from_pretrained(output_dir)  # Add specific options if needed"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "aa49ba25",
   "metadata": {},
   "source": [
    "## Predict with test data set "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e3e10b67",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_test = load_dataset(\"csv\", data_files=\"/Users/ruochentan1/PycharmProjects/bert/df_test_full.csv\",\n",
    "                       split=\"train\")\n",
    "test = df_test.remove_columns(['Unnamed: 0',\"unixReviewTime\",\"reviewTime\"])\n",
    "labels = [label for label in test.features.keys() if label not in ['Rating', \"reviewerID\",\"ItemId\",\"reviewText\",\"UserId\"]]\n",
    "id2label = {idx:label for idx, label in enumerate(labels)}\n",
    "label2id = {label:idx for idx, label in enumerate(labels)}\n",
    "labels\n",
    "encoded_test = test.map(preprocess_data, batched=True, remove_columns= test.column_names) # with its own labels\n",
    "encoded_test.set_format(\"torch\")\n",
    "encoded_test.save_to_disk(\"test_tokenized\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "3f35e723",
   "metadata": {},
   "outputs": [],
   "source": [
    "predictions = trainer.predict(encoded_test) # return as logits\n",
    "preds = np.argmax(predictions.predictions, axis=-1) # get predictions "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "35dd0a4b",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_test = pd.read_csv(\"/Users/ruochentan1/PycharmProjects/bert/df_test.csv\")\n",
    "df_test[\"preds\"] = preds\n",
    "df_test[\"preds\"] = df_test.preds.replace([0, 1, 2, 3,4], [1, 2, 3, 4,5])\n",
    "df_test_full = pd.read_csv(\"/Users/ruochentan1/PycharmProjects/bert/df_test_full.csv\")\n",
    "df_test[\"true\"] = df_test_full[\"Rating\"]\n",
    "df_test.to_csv(\"df_test_result.csv\",index =False, header = True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d13abd43",
   "metadata": {},
   "outputs": [],
   "source": [
    "y_pred = df_test.loc[df_test['Rating'].isnull(), 'preds']\n",
    "y_true = df_test.loc[df_test['Rating'].isnull(),'true'] # compare to only the missing rows \n",
    "print(metrics.classification_report(y_true, y_pred))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "86395af9",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Convert y_pred to one-hot encoded format \n",
    "lb = LabelBinarizer()\n",
    "y_pred_one_hot = lb.fit_transform(y_pred)\n",
    "# Compute roc_auc_score\n",
    "roc_auc = roc_auc_score(y_true, y_pred_one_hot, multi_class='ovr')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "9bef0d72",
   "metadata": {},
   "source": [
    "## Plot the model learning curve "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c0903c02",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Generate a sequence of integers to represent the epoch numbers\n",
    "epochs = range(1, 5)\n",
    "train_values = [ 0.15463,  0.109419,  0.082568,  0.069241]\n",
    "val_values = [  0.18120, 0.160794, 0.16858,   0.169248]\n",
    "# Plot and label the training and validation loss values\n",
    "plt.plot(epochs, train_values, label ='Training Loss')\n",
    "plt.plot(epochs, val_values, label='Validation Loss')\n",
    "# Add in a title and axes labels\n",
    "plt.title('Training and Validation Loss')\n",
    "plt.xlabel('Epochs')\n",
    "plt.ylabel('Loss')\n",
    "# Display the plot\n",
    "plt.legend(loc='best')\n",
    "plt.show()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}