english-sentiment-analysis.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# English Sentiment Analysis\n",
    "- copied form [Fine-tuning Sandbox](https://github.com/ShawhinT/YouTube-Blog/blob/main/LLMs/fine-tuning/ft-example.ipynb)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from datasets import load_dataset, DatasetDict, Dataset\n",
    "\n",
    "from transformers import (\n",
    "    AutoTokenizer,\n",
    "    AutoConfig, \n",
    "    AutoModelForSequenceClassification,\n",
    "    DataCollatorWithPadding,\n",
    "    TrainingArguments,\n",
    "    Trainer)\n",
    "\n",
    "from peft import PeftModel, PeftConfig, get_peft_model, LoraConfig\n",
    "import evaluate\n",
    "import torch\n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Downloading readme: 100%|██████████| 592/592 [00:00<00:00, 1.07MB/s]\n",
      "Downloading data: 100%|██████████| 836k/836k [00:01<00:00, 613kB/s]\n",
      "Downloading data: 100%|██████████| 853k/853k [00:01<00:00, 784kB/s]t]\n",
      "Downloading data files: 100%|██████████| 2/2 [00:02<00:00,  1.26s/it]\n",
      "Extracting data files: 100%|██████████| 2/2 [00:00<00:00, 53.80it/s]\n",
      "Generating train split: 100%|██████████| 1000/1000 [00:00<00:00, 42167.79 examples/s]\n",
      "Generating validation split: 100%|██████████| 1000/1000 [00:00<00:00, 130908.36 examples/s]\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "DatasetDict({\n",
       "    train: Dataset({\n",
       "        features: ['label', 'text'],\n",
       "        num_rows: 1000\n",
       "    })\n",
       "    validation: Dataset({\n",
       "        features: ['label', 'text'],\n",
       "        num_rows: 1000\n",
       "    })\n",
       "})"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# load dataset\n",
    "dataset = load_dataset('shawhin/imdb-truncated')\n",
    "dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.5"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# display % of training data with label=1\n",
    "np.array(dataset['train']['label']).sum()/len(dataset['train']['label'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "config.json: 100%|██████████| 483/483 [00:00<00:00, 3.21MB/s]\n",
      "model.safetensors: 100%|██████████| 268M/268M [00:21<00:00, 12.6MB/s] \n",
      "Some weights of DistilBertForSequenceClassification were not initialized from the model checkpoint at distilbert-base-uncased and are newly initialized: ['pre_classifier.weight', 'classifier.bias', 'pre_classifier.bias', 'classifier.weight']\n",
      "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n"
     ]
    }
   ],
   "source": [
    "model_checkpoint = 'distilbert-base-uncased'\n",
    "# model_checkpoint = 'roberta-base' # you can alternatively use roberta-base but this model is bigger thus training will take longer\n",
    "\n",
    "# define label maps\n",
    "id2label = {0: \"Negative\", 1: \"Positive\"}\n",
    "label2id = {\"Negative\":0, \"Positive\":1}\n",
    "\n",
    "# generate classification model from model_checkpoint\n",
    "model = AutoModelForSequenceClassification.from_pretrained(\n",
    "    model_checkpoint, num_labels=2, id2label=id2label, label2id=label2id)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "DistilBertForSequenceClassification(\n",
       "  (distilbert): DistilBertModel(\n",
       "    (embeddings): Embeddings(\n",
       "      (word_embeddings): Embedding(30522, 768, padding_idx=0)\n",
       "      (position_embeddings): Embedding(512, 768)\n",
       "      (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "      (dropout): Dropout(p=0.1, inplace=False)\n",
       "    )\n",
       "    (transformer): Transformer(\n",
       "      (layer): ModuleList(\n",
       "        (0-5): 6 x TransformerBlock(\n",
       "          (attention): MultiHeadSelfAttention(\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "            (q_lin): Linear(in_features=768, out_features=768, bias=True)\n",
       "            (k_lin): Linear(in_features=768, out_features=768, bias=True)\n",
       "            (v_lin): Linear(in_features=768, out_features=768, bias=True)\n",
       "            (out_lin): Linear(in_features=768, out_features=768, bias=True)\n",
       "          )\n",
       "          (sa_layer_norm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "          (ffn): FFN(\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "            (lin1): Linear(in_features=768, out_features=3072, bias=True)\n",
       "            (lin2): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (activation): GELUActivation()\n",
       "          )\n",
       "          (output_layer_norm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "        )\n",
       "      )\n",
       "    )\n",
       "  )\n",
       "  (pre_classifier): Linear(in_features=768, out_features=768, bias=True)\n",
       "  (classifier): Linear(in_features=768, out_features=2, bias=True)\n",
       "  (dropout): Dropout(p=0.2, inplace=False)\n",
       ")"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# display architecture\n",
    "model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "tokenizer_config.json: 100%|██████████| 28.0/28.0 [00:00<00:00, 133kB/s]\n",
      "vocab.txt: 100%|██████████| 232k/232k [00:00<00:00, 1.16MB/s]\n",
      "tokenizer.json: 100%|██████████| 466k/466k [00:00<00:00, 1.54MB/s]\n"
     ]
    }
   ],
   "source": [
    "# create tokenizer\n",
    "tokenizer = AutoTokenizer.from_pretrained(model_checkpoint, add_prefix_space=True)\n",
    "\n",
    "# add pad token if none exists\n",
    "if tokenizer.pad_token is None:\n",
    "    tokenizer.add_special_tokens({'pad_token': '[PAD]'})\n",
    "    model.resize_token_embeddings(len(tokenizer))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "# create tokenize function\n",
    "def tokenize_function(examples):\n",
    "    # extract text\n",
    "    text = examples[\"text\"]\n",
    "\n",
    "    #tokenize and truncate text\n",
    "    tokenizer.truncation_side = \"left\"\n",
    "    tokenized_inputs = tokenizer(\n",
    "        text,\n",
    "        return_tensors=\"np\",\n",
    "        truncation=True,\n",
    "        max_length=512\n",
    "    )\n",
    "\n",
    "    return tokenized_inputs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Map: 100%|██████████| 1000/1000 [00:00<00:00, 4052.50 examples/s]\n",
      "Map: 100%|██████████| 1000/1000 [00:00<00:00, 5461.83 examples/s]\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "DatasetDict({\n",
       "    train: Dataset({\n",
       "        features: ['label', 'text', 'input_ids', 'attention_mask'],\n",
       "        num_rows: 1000\n",
       "    })\n",
       "    validation: Dataset({\n",
       "        features: ['label', 'text', 'input_ids', 'attention_mask'],\n",
       "        num_rows: 1000\n",
       "    })\n",
       "})"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# tokenize training and validation datasets\n",
    "tokenized_dataset = dataset.map(tokenize_function, batched=True)\n",
    "tokenized_dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "# create data collator\n",
    "data_collator = DataCollatorWithPadding(tokenizer=tokenizer)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "# import accuracy evaluation metric\n",
    "accuracy = evaluate.load(\"accuracy\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "# define an evaluation function to pass into trainer later\n",
    "def compute_metrics(p):\n",
    "    predictions, labels = p\n",
    "    predictions = np.argmax(predictions, axis=1)\n",
    "\n",
    "    return {\"accuracy\": accuracy.compute(predictions=predictions, references=labels)}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Untrained model predictions:\n",
      "----------------------------\n",
      "It was good. - Negative\n",
      "Not a fan, don't recommed. - Negative\n",
      "Better than the first one. - Negative\n",
      "This is not worth watching even once. - Negative\n",
      "This one is a pass. - Negative\n"
     ]
    }
   ],
   "source": [
    "# define list of examples\n",
    "text_list = [\"It was good.\", \"Not a fan, don't recommed.\", \"Better than the first one.\", \"This is not worth watching even once.\", \"This one is a pass.\"]\n",
    "\n",
    "print(\"Untrained model predictions:\")\n",
    "print(\"----------------------------\")\n",
    "for text in text_list:\n",
    "    # tokenize text\n",
    "    inputs = tokenizer.encode(text, return_tensors=\"pt\")\n",
    "    # compute logits\n",
    "    logits = model(inputs).logits\n",
    "    # convert logits to label\n",
    "    predictions = torch.argmax(logits)\n",
    "\n",
    "    print(text + \" - \" + id2label[predictions.tolist()])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "peft_config = LoraConfig(task_type=\"SEQ_CLS\",\n",
    "                        r=4,\n",
    "                        lora_alpha=32,\n",
    "                        lora_dropout=0.01,\n",
    "                        target_modules = ['q_lin'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "LoraConfig(peft_type=<PeftType.LORA: 'LORA'>, auto_mapping=None, base_model_name_or_path=None, revision=None, task_type='SEQ_CLS', inference_mode=False, r=4, target_modules={'q_lin'}, lora_alpha=32, lora_dropout=0.01, fan_in_fan_out=False, bias='none', modules_to_save=None, init_lora_weights=True, layers_to_transform=None, layers_pattern=None, rank_pattern={}, alpha_pattern={})"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "peft_config"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "trainable params: 628,994 || all params: 67,584,004 || trainable%: 0.9306847223789819\n"
     ]
    }
   ],
   "source": [
    "model = get_peft_model(model, peft_config)\n",
    "model.print_trainable_parameters()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "# hyperparameters\n",
    "lr = 1e-3\n",
    "batch_size = 4\n",
    "num_epochs = 10"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "# define training arguments\n",
    "training_args = TrainingArguments(\n",
    "    output_dir= model_checkpoint + \"-lora-text-classification\",\n",
    "    learning_rate=lr,\n",
    "    per_device_train_batch_size=batch_size,\n",
    "    per_device_eval_batch_size=batch_size,\n",
    "    num_train_epochs=num_epochs,\n",
    "    weight_decay=0.01,\n",
    "    evaluation_strategy=\"epoch\",\n",
    "    save_strategy=\"epoch\",\n",
    "    load_best_model_at_end=True,\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  0%|          | 0/2500 [00:00<?, ?it/s]You're using a DistilBertTokenizerFast tokenizer. Please note that with a fast tokenizer, using the `__call__` method is faster than using a method to encode the text followed by a call to the `pad` method to get a padded encoding.\n",
      "                                                  \n",
      " 10%|█         | 250/2500 [02:22<11:29,  3.26it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'eval_loss': 0.40093478560447693, 'eval_accuracy': {'accuracy': 0.869}, 'eval_runtime': 56.667, 'eval_samples_per_second': 17.647, 'eval_steps_per_second': 4.412, 'epoch': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 20%|██        | 500/2500 [03:47<08:03,  4.14it/s]   "
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 0.4313, 'learning_rate': 0.0008, 'epoch': 2.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                                  \n",
      " 20%|██        | 500/2500 [04:08<08:03,  4.14it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'eval_loss': 0.4657118618488312, 'eval_accuracy': {'accuracy': 0.863}, 'eval_runtime': 20.9568, 'eval_samples_per_second': 47.717, 'eval_steps_per_second': 11.929, 'epoch': 2.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                                    \n",
      " 30%|███       | 750/2500 [05:46<33:03,  1.13s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'eval_loss': 0.5446674823760986, 'eval_accuracy': {'accuracy': 0.886}, 'eval_runtime': 20.9959, 'eval_samples_per_second': 47.628, 'eval_steps_per_second': 11.907, 'epoch': 3.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 40%|████      | 1000/2500 [06:55<06:11,  4.04it/s] "
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 0.2333, 'learning_rate': 0.0006, 'epoch': 4.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                                   \n",
      " 40%|████      | 1000/2500 [07:16<06:11,  4.04it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'eval_loss': 0.5675832629203796, 'eval_accuracy': {'accuracy': 0.888}, 'eval_runtime': 21.4394, 'eval_samples_per_second': 46.643, 'eval_steps_per_second': 11.661, 'epoch': 4.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                                     \n",
      " 50%|█████     | 1250/2500 [08:40<04:52,  4.28it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'eval_loss': 0.7689547538757324, 'eval_accuracy': {'accuracy': 0.89}, 'eval_runtime': 20.703, 'eval_samples_per_second': 48.302, 'eval_steps_per_second': 12.076, 'epoch': 5.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 60%|██████    | 1500/2500 [09:41<07:55,  2.10it/s]  "
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 0.0882, 'learning_rate': 0.0004, 'epoch': 6.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                                   \n",
      " 60%|██████    | 1500/2500 [10:01<07:55,  2.10it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'eval_loss': 0.8811860680580139, 'eval_accuracy': {'accuracy': 0.884}, 'eval_runtime': 20.5743, 'eval_samples_per_second': 48.604, 'eval_steps_per_second': 12.151, 'epoch': 6.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                                     \n",
      " 70%|███████   | 1750/2500 [11:21<02:45,  4.54it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'eval_loss': 0.8201422095298767, 'eval_accuracy': {'accuracy': 0.889}, 'eval_runtime': 20.7139, 'eval_samples_per_second': 48.277, 'eval_steps_per_second': 12.069, 'epoch': 7.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 80%|████████  | 2000/2500 [12:13<01:40,  4.97it/s]  "
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 0.0306, 'learning_rate': 0.0002, 'epoch': 8.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                                   \n",
      " 80%|████████  | 2000/2500 [12:34<01:40,  4.97it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'eval_loss': 0.9280931353569031, 'eval_accuracy': {'accuracy': 0.883}, 'eval_runtime': 20.603, 'eval_samples_per_second': 48.537, 'eval_steps_per_second': 12.134, 'epoch': 8.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                                   \n",
      " 90%|█████████ | 2250/2500 [13:49<00:42,  5.88it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'eval_loss': 0.9449716806411743, 'eval_accuracy': {'accuracy': 0.886}, 'eval_runtime': 20.6595, 'eval_samples_per_second': 48.404, 'eval_steps_per_second': 12.101, 'epoch': 9.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 2500/2500 [14:43<00:00,  4.94it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 0.0089, 'learning_rate': 0.0, 'epoch': 10.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                                   \n",
      "100%|██████████| 2500/2500 [15:03<00:00,  2.77it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'eval_loss': 0.9498960971832275, 'eval_accuracy': {'accuracy': 0.887}, 'eval_runtime': 20.4932, 'eval_samples_per_second': 48.797, 'eval_steps_per_second': 12.199, 'epoch': 10.0}\n",
      "{'train_runtime': 903.8184, 'train_samples_per_second': 11.064, 'train_steps_per_second': 2.766, 'train_loss': 0.1584586856842041, 'epoch': 10.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "TrainOutput(global_step=2500, training_loss=0.1584586856842041, metrics={'train_runtime': 903.8184, 'train_samples_per_second': 11.064, 'train_steps_per_second': 2.766, 'train_loss': 0.1584586856842041, 'epoch': 10.0})"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# creater trainer object\n",
    "trainer = Trainer(\n",
    "    model=model,\n",
    "    args=training_args,\n",
    "    train_dataset=tokenized_dataset[\"train\"],\n",
    "    eval_dataset=tokenized_dataset[\"validation\"],\n",
    "    tokenizer=tokenizer,\n",
    "    data_collator=data_collator, # this will dynamically pad examples in each batch to be equal length\n",
    "    compute_metrics=compute_metrics,\n",
    ")\n",
    "\n",
    "# train model\n",
    "trainer.train()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Trained model predictions:\n",
      "--------------------------\n",
      "It was good. - Positive\n",
      "Not a fan, don't recommed. - Negative\n",
      "Better than the first one. - Positive\n",
      "This is not worth watching even once. - Negative\n",
      "This one is a pass. - Positive\n"
     ]
    }
   ],
   "source": [
    "model.to('mps') # moving to mps for Mac (can alternatively do 'cpu')\n",
    "\n",
    "print(\"Trained model predictions:\")\n",
    "print(\"--------------------------\")\n",
    "for text in text_list:\n",
    "    inputs = tokenizer.encode(text, return_tensors=\"pt\").to(\"mps\") # moving to mps for Mac (can alternatively do 'cpu')\n",
    "\n",
    "    logits = model(inputs).logits\n",
    "    predictions = torch.max(logits,1).indices\n",
    "\n",
    "    print(text + \" - \" + id2label[predictions.tolist()[0]])"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}