QWEN2.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "torch.cuda.empty_cache()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "===================================BUG REPORT===================================\n",
      "Welcome to bitsandbytes. For bug reports, please run\n",
      "\n",
      "python -m bitsandbytes\n",
      "\n",
      " and submit this information together with your error trace to: https://github.com/TimDettmers/bitsandbytes/issues\n",
      "================================================================================\n",
      "bin /opt/conda/envs/pytorch/lib/python3.10/site-packages/bitsandbytes/libbitsandbytes_cuda121.so\n",
      "CUDA SETUP: CUDA runtime path found: /usr/local/cuda/lib64/libcudart.so\n",
      "CUDA SETUP: Highest compute capability among GPUs detected: 8.6\n",
      "CUDA SETUP: Detected CUDA version 121\n",
      "CUDA SETUP: Loading binary /opt/conda/envs/pytorch/lib/python3.10/site-packages/bitsandbytes/libbitsandbytes_cuda121.so...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/conda/envs/pytorch/lib/python3.10/site-packages/bitsandbytes/cuda_setup/main.py:149: UserWarning: /opt/conda/envs/pytorch did not contain ['libcudart.so', 'libcudart.so.11.0', 'libcudart.so.12.0'] as expected! Searching further paths...\n",
      "  warn(msg)\n"
     ]
    }
   ],
   "source": [
    "import json\n",
    "import os\n",
    "from pprint import pprint\n",
    "import bitsandbytes as bnb\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "import transformers\n",
    "from datasets import load_dataset\n",
    "from huggingface_hub import notebook_login\n",
    "from peft import (\n",
    "    LoraConfig,\n",
    "    PeftConfig,\n",
    "    PeftModel,\n",
    "    get_peft_model,\n",
    "    prepare_model_for_kbit_training\n",
    ")\n",
    "from transformers import (\n",
    "    AutoConfig,\n",
    "    AutoModelForCausalLM,\n",
    "    AutoTokenizer,\n",
    "    BitsAndBytesConfig\n",
    ")\n",
    "from transformers import Trainer, GPTQConfig, deepspeed\n",
    "from dataclasses import dataclass, field\n",
    "from typing import Dict, Optional, List\n",
    "\n",
    "#os.environ[\"CUDA_VISIBLE_DEVICES\"] = \"0\"\n",
    "os.environ[\"PYTORCH_CUDA_ALLOC_CONF\"] = \"max_split_size_mb:64\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "torch.cuda.is_available()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "@dataclass\n",
    "class ModelArguments:\n",
    "    model_name_or_path: Optional[str] = field(default=\"Qwen/Qwen-14B-Chat-Int4\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "argmodel = ModelArguments()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "config = transformers.AutoConfig.from_pretrained(\n",
    "        argmodel.model_name_or_path,\n",
    "        trust_remote_code=True,\n",
    "    )\n",
    "config.use_cache = False"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "You passed `quantization_config` to `from_pretrained` but the model you're loading already has a `quantization_config` attribute and has already quantized weights. However, loading attributes (e.g. disable_exllama, use_cuda_fp16) will be overwritten with the one you passed to `from_pretrained`. The rest will be ignored.\n",
      "CUDA extension not installed.\n",
      "CUDA extension not installed.\n",
      "Warning: please make sure that you are using the latest codes and checkpoints, especially if you used Qwen-7B before 09.25.2023.请使用最新模型和代码，尤其如果你在9月25日前已经开始使用Qwen-7B，千万注意不要使用错误代码和模型。\n",
      "Try importing flash-attention for faster inference...\n",
      "Warning: import flash_attn rms_norm fail, please install FlashAttention layer_norm to get higher efficiency https://github.com/Dao-AILab/flash-attention/tree/main/csrc/layer_norm\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "c94d344184af44c1b19f34b515805685",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Loading checkpoint shards:   0%|          | 0/5 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "\n",
    "\n",
    "\n",
    "#model = AutoModelForCausalLM.from_pretrained(\n",
    "#    argmodel.model_name_or_path,\n",
    "#    pad_token_id= tokenizer.pad_token_id,\n",
    "#    device_map =\"auto\",\n",
    "#    trust_remote_code=True,\n",
    "#    quantization_config=bnb_config)\n",
    "\n",
    "model = transformers.AutoModelForCausalLM.from_pretrained(\n",
    "        argmodel.model_name_or_path,\n",
    "        config=config,\n",
    "        device_map =\"cuda\",\n",
    "        trust_remote_code=True,\n",
    "        quantization_config=GPTQConfig(\n",
    "            bits=4, disable_exllama=True\n",
    "        )\n",
    "    )\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "tokenizer = transformers.AutoTokenizer.from_pretrained(\n",
    "    argmodel.model_name_or_path,\n",
    "    model_max_length=2048,\n",
    "    padding_side=\"right\",\n",
    "    use_fast=False,\n",
    "    trust_remote_code=True,\n",
    ")\n",
    "tokenizer.pad_token_id = tokenizer.eod_id\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "def print_trainable_parameters(model):\n",
    "  \"\"\"\n",
    "  Prints the number of trainable parameters in the model.\n",
    "  \"\"\"\n",
    "  trainable_params = 0\n",
    "  all_param = 0\n",
    "  for _, param in model.named_parameters():\n",
    "    all_param += param.numel()\n",
    "    if param.requires_grad:\n",
    "      trainable_params += param.numel()\n",
    "  print(\n",
    "      f\"trainable params: {trainable_params} || all params: {all_param} || trainables%: {100 * trainable_params / all_param}\"\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "model.gradient_checkpointing_enable()\n",
    "model = prepare_model_for_kbit_training(model)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "@dataclass\n",
    "class LoraArguments:\n",
    "    lora_r: int = 64\n",
    "    lora_alpha: int = 16\n",
    "    lora_dropout: float = 0.05\n",
    "    lora_target_modules: List[str] = field(\n",
    "        default_factory=lambda: [\"c_attn\", \"c_proj\", \"w1\", \"w2\"]\n",
    "    )\n",
    "    lora_weight_path: str = \"\"\n",
    "    lora_bias: str = \"none\"\n",
    "    q_lora: bool = False\n",
    "    task_type=\"CAUSAL_LM\",\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "trainable params: 1780285440 || all params: 3337835520 || trainables%: 53.33652390396996\n"
     ]
    }
   ],
   "source": [
    "arg = LoraArguments()\n",
    "\n",
    "config = LoraConfig(\n",
    "    r=64,\n",
    "    lora_alpha=16,\n",
    "    target_modules=arg.lora_target_modules,\n",
    "    lora_dropout=0.05,\n",
    "    bias=\"none\",\n",
    "    task_type=\"CAUSAL_LM\",\n",
    "    modules_to_save = [\"wte\", \"lm_head\"]\n",
    ")\n",
    "\n",
    "\n",
    "model = get_peft_model(model, config)\n",
    "print_trainable_parameters(model)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "generation_config = model.generation_config\n",
    "generation_config.max_new_tokens = 200\n",
    "generation_config.temperature = 0.7\n",
    "generation_config.top_p = 0.7\n",
    "generation_config.num_return_sequences = 1\n",
    "\n",
    "#generation_config.pad_token_id = tokenizer.eos_token_id\n",
    "#generation_config.eos_token_id = \"<|endoftext|>\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "device(type='cuda', index=0)"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#check device type\n",
    "device = model.device\n",
    "device"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "device = model.device\n",
    "def single_out(logits):\n",
    "    temperature = 0.9\n",
    "    print(logits['logits'].shape)\n",
    "    logits = logits['logits'][:, -1, :]\n",
    "    logits = logits / temperature\n",
    "    greedy = False\n",
    "\n",
    "    \n",
    "    if greedy:\n",
    "        out = torch.argmax(logits, dim=1).reshape(-1, 1)\n",
    "        return out\n",
    "\n",
    "    # Initialize mask with ones\n",
    "    mask = torch.ones_like(logits).bool()\n",
    "    top_p  = 0.5\n",
    "    top_k = 0\n",
    "    if top_p > 0.0:\n",
    "        sorted_logits, sorted_indices = torch.sort(logits, descending=True)\n",
    "        cumulative_probs = torch.cumsum(torch.softmax(sorted_logits, dim=1), dim=1)\n",
    "        sorted_mask = cumulative_probs > top_p\n",
    "        # Ensure at least the most probable is included if sorted_mask contains all True \n",
    "        if sorted_mask.all():\n",
    "            sorted_mask[..., :1] = 0\n",
    "        to_scatter = sorted_mask.type_as(logits) * float('-inf')\n",
    "        to_scatter[sorted_mask == 0] = logits.gather(1, sorted_indices)[sorted_mask == 0]\n",
    "        logits.scatter_(1, sorted_indices, to_scatter)\n",
    "    elif top_k > 0:\n",
    "        top_k = min(top_k, logits.shape[1])            \n",
    "        values, _ = torch.topk(logits, top_k)\n",
    "        # smallest allowed value\n",
    "        kth_values = values[..., -1]\n",
    "        logits = torch.where(logits < kth_values.unsqueeze(-1), torch.tensor(float('-inf')).type_as(logits), logits)\n",
    "\n",
    "\n",
    "    probs = torch.softmax(logits, dim=1)\n",
    "    m = torch.argmax(probs,dim=1)\n",
    "    return m.reshape(-1, 1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "def gen_out(prompt):\n",
    "    idx = tokenizer(prompt, return_tensors=\"pt\").to(device)\n",
    "    for _ in range(100):\n",
    "        logits = model(**idx)\n",
    "        next_token = single_out(logits)\n",
    "        next_tokened = tokenizer.decode(next_token[0], skip_special_tokens=False)\n",
    "        # print(idx.input_ids[0].shape)\n",
    "        idx = tokenizer.decode(idx.input_ids[0], skip_special_tokens=False)\n",
    "        \n",
    "        idx = tokenizer(idx + next_tokened, return_tensors=\"pt\",).to(device)\n",
    "\n",
    "        if next_token.item() == tokenizer.eod_id:\n",
    "             break\n",
    "    return idx\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# LOAD TRAINING DATA"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 81,
   "metadata": {},
   "outputs": [],
   "source": [
    "traindata = load_dataset(\"csv\", data_files=\"datasetV3.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 82,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "DatasetDict({\n",
       "    train: Dataset({\n",
       "        features: ['title', 'context', 'question', 'answers'],\n",
       "        num_rows: 32609\n",
       "    })\n",
       "})"
      ]
     },
     "execution_count": 82,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "traindata"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 83,
   "metadata": {},
   "outputs": [],
   "source": [
    "def fil_nan(text):\n",
    "    if text == None:\n",
    "        return \" \"\n",
    "    else:\n",
    "        return text"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 88,
   "metadata": {},
   "outputs": [],
   "source": [
    "def generate_prompt(data_point):\n",
    "  return f\"\"\"\n",
    "  human: title->{fil_nan(data_point[\"title\"])}\\ncontext->{fil_nan(data_point[\"context\"])}\\nquestion->{fil_nan(data_point[\"question\"])}\\n\n",
    "  assistant: {data_point[\"answers\"]}\n",
    "  \"\"\".strip()\n",
    "\n",
    "\n",
    "def generate_and_tokenize_prompt(data_point):\n",
    "  full_prompt = generate_prompt(data_point)\n",
    "  tokenized_full_prompt = tokenizer(full_prompt)\n",
    "  \n",
    "  # Check if the tokenized prompt exceeds the maximum length\n",
    "  if len(tokenized_full_prompt['input_ids']) > 2048:\n",
    "    # Sliding window approach\n",
    "    window_size = 2048\n",
    "    stride = 100  # Adjust the stride value as needed\n",
    "    num_windows = (len(tokenized_full_prompt['input_ids']) - window_size) // stride + 1\n",
    "    \n",
    "    # Create a list to store the windows\n",
    "    windows = []\n",
    "    \n",
    "    # Generate the windows\n",
    "    for i in range(num_windows):\n",
    "      start_index = i * stride\n",
    "      end_index = start_index + window_size\n",
    "      window = {\n",
    "        'input_ids': tokenized_full_prompt['input_ids'][start_index:end_index],\n",
    "        'attention_mask': tokenized_full_prompt['attention_mask'][start_index:end_index]\n",
    "      }\n",
    "      windows.append(window)\n",
    "    \n",
    "    return windows\n",
    "  else:\n",
    "    return [tokenized_full_prompt]\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 85,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "DatasetDict({\n",
       "    train: Dataset({\n",
       "        features: ['title', 'context', 'question', 'answers'],\n",
       "        num_rows: 32609\n",
       "    })\n",
       "})"
      ]
     },
     "execution_count": 85,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "traindata"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 89,
   "metadata": {},
   "outputs": [],
   "source": [
    "list_sup = []\n",
    "for i in range(32609):\n",
    "    gen_lst = generate_and_tokenize_prompt(traindata[\"train\"][i])\n",
    "    for j in range(len(gen_lst)):\n",
    "        list_sup.append(gen_lst[j])\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 90,
   "metadata": {},
   "outputs": [],
   "source": [
    "texts = []\n",
    "for i in list_sup:\n",
    "    texts.append(tokenizer.decode(i[\"input_ids\"]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 91,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 92,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.DataFrame(texts, columns=[\"text\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 107,
   "metadata": {},
   "outputs": [],
   "source": [
    "df.to_csv(\"datasetV4.csv\", index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 120,
   "metadata": {},
   "outputs": [],
   "source": [
    "truedata = load_dataset(\"csv\", data_files=\"datasetV4.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 122,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'human: title->พัทธ์ธีรา ศรุติพงศ์โภคิน\\ncontext->พัทธ์ธีรา ศรุติพงศ์โภคิน (เกิด 3 ธันวาคม พ.ศ. 2533) หรือชื่อเล่นว่า อร เป็นนักแสดงหญิงชาวไทย สำเร็จมัธยมศึกษาจากCatholic Cathedral College ประเทศนิวซีแลนด์ และปริญญาตรีจากRaffles International College สาขา Business Marketing\\n\\nเข้าสู่วงการตั้งแต่อายุ 6 ขวบ จากการแสดงละครเวทีกับ ครูชลประคัลภ์ จันทร์เรือง จากนั้นก็เล่นโฆษณาในวัยเด็ก 2- 3 ชิ้น และยังเคยแสดงช่วงละครสั้น ในรายการซุปเปอร์จิ๋ว ประมาณปี 2542\\n\\nปัจจุบันเป็นทั้ง นักแสดง , พิธีกร และ วีเจ อยู่ที่คลื่น เก็ท 102.5 Bangkok International Hits Music Station และยังเป็นพิธีกรให้กับช่อง ทรู มิวสิก\\nquestion->พัทธ์ธีรา ศรุติพงศ์โภคิน เกิดวันที่เท่าไร\\n\\n  assistant: 3 ธันวาคม พ.ศ. 2533'"
      ]
     },
     "execution_count": 122,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "truedata['train'][0]['text']\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 123,
   "metadata": {},
   "outputs": [],
   "source": [
    "def gen_tok(datapoint):\n",
    "    tok = tokenizer(datapoint['text'],truncation=True,padding='max_length', max_length=2048)\n",
    "    return tok"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 124,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "05bb0903fa2d43ff80720668b015323e",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Map:   0%|          | 0/57957 [00:00<?, ? examples/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "traindata[\"train\"] = truedata[\"train\"].shuffle().map(gen_tok)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Finetune the model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 125,
   "metadata": {},
   "outputs": [],
   "source": [
    "training_args = transformers.TrainingArguments(\n",
    "      per_device_train_batch_size=3,\n",
    "      gradient_accumulation_steps=10,\n",
    "      num_train_epochs=20,\n",
    "      learning_rate=2e-4,\n",
    "      save_total_limit=3,\n",
    "      logging_steps=5,\n",
    "      output_dir=\"experiments\",\n",
    "      optim=\"paged_adamw_8bit\",\n",
    "      lr_scheduler_type=\"cosine\",\n",
    "      warmup_ratio=0.05,\n",
    ")\n",
    "\n",
    "trainer = transformers.Trainer(\n",
    "    model=model,\n",
    "    train_dataset=traindata[\"train\"],\n",
    "    args=training_args,\n",
    "    data_collator=transformers.DataCollatorForLanguageModeling(tokenizer, mlm=False)\n",
    ")\n",
    "model.config.use_cache = False \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 126,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "You are using 8-bit optimizers with a version of `bitsandbytes` < 0.41.1. It is recommended to update your version as a major bug has been fixed in 8-bit optimizers.\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      \n",
       "      <progress value='12' max='11591' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [   12/11591 04:38 < 89:34:46, 0.04 it/s, Epoch 0.00/1]\n",
       "    </div>\n",
       "    <table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       " <tr style=\"text-align: left;\">\n",
       "      <th>Step</th>\n",
       "      <th>Training Loss</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>2.071300</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>2.790500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>2.164300</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>2.445100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>5</td>\n",
       "      <td>2.885900</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>6</td>\n",
       "      <td>2.886500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>7</td>\n",
       "      <td>2.447300</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>8</td>\n",
       "      <td>2.767600</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>9</td>\n",
       "      <td>2.286400</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>10</td>\n",
       "      <td>2.274700</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table><p>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[1;32m/home/ubuntu/QWEN2.ipynb Cell 34\u001b[0m line \u001b[0;36m1\n\u001b[0;32m----> <a href='vscode-notebook-cell://ssh-remote%2Baws-ec2/home/ubuntu/QWEN2.ipynb#Y124sdnNjb2RlLXJlbW90ZQ%3D%3D?line=0'>1</a>\u001b[0m trainer\u001b[39m.\u001b[39;49mtrain()\n",
      "File \u001b[0;32m/opt/conda/envs/pytorch/lib/python3.10/site-packages/transformers/trainer.py:1555\u001b[0m, in \u001b[0;36mTrainer.train\u001b[0;34m(self, resume_from_checkpoint, trial, ignore_keys_for_eval, **kwargs)\u001b[0m\n\u001b[1;32m   1553\u001b[0m         hf_hub_utils\u001b[39m.\u001b[39menable_progress_bars()\n\u001b[1;32m   1554\u001b[0m \u001b[39melse\u001b[39;00m:\n\u001b[0;32m-> 1555\u001b[0m     \u001b[39mreturn\u001b[39;00m inner_training_loop(\n\u001b[1;32m   1556\u001b[0m         args\u001b[39m=\u001b[39;49margs,\n\u001b[1;32m   1557\u001b[0m         resume_from_checkpoint\u001b[39m=\u001b[39;49mresume_from_checkpoint,\n\u001b[1;32m   1558\u001b[0m         trial\u001b[39m=\u001b[39;49mtrial,\n\u001b[1;32m   1559\u001b[0m         ignore_keys_for_eval\u001b[39m=\u001b[39;49mignore_keys_for_eval,\n\u001b[1;32m   1560\u001b[0m     )\n",
      "File \u001b[0;32m/opt/conda/envs/pytorch/lib/python3.10/site-packages/transformers/trainer.py:1837\u001b[0m, in \u001b[0;36mTrainer._inner_training_loop\u001b[0;34m(self, batch_size, args, resume_from_checkpoint, trial, ignore_keys_for_eval)\u001b[0m\n\u001b[1;32m   1834\u001b[0m     \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mcontrol \u001b[39m=\u001b[39m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mcallback_handler\u001b[39m.\u001b[39mon_step_begin(args, \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mstate, \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mcontrol)\n\u001b[1;32m   1836\u001b[0m \u001b[39mwith\u001b[39;00m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39maccelerator\u001b[39m.\u001b[39maccumulate(model):\n\u001b[0;32m-> 1837\u001b[0m     tr_loss_step \u001b[39m=\u001b[39m \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49mtraining_step(model, inputs)\n\u001b[1;32m   1839\u001b[0m \u001b[39mif\u001b[39;00m (\n\u001b[1;32m   1840\u001b[0m     args\u001b[39m.\u001b[39mlogging_nan_inf_filter\n\u001b[1;32m   1841\u001b[0m     \u001b[39mand\u001b[39;00m \u001b[39mnot\u001b[39;00m is_torch_tpu_available()\n\u001b[1;32m   1842\u001b[0m     \u001b[39mand\u001b[39;00m (torch\u001b[39m.\u001b[39misnan(tr_loss_step) \u001b[39mor\u001b[39;00m torch\u001b[39m.\u001b[39misinf(tr_loss_step))\n\u001b[1;32m   1843\u001b[0m ):\n\u001b[1;32m   1844\u001b[0m     \u001b[39m# if loss is nan or inf simply add the average of previous logged losses\u001b[39;00m\n\u001b[1;32m   1845\u001b[0m     tr_loss \u001b[39m+\u001b[39m\u001b[39m=\u001b[39m tr_loss \u001b[39m/\u001b[39m (\u001b[39m1\u001b[39m \u001b[39m+\u001b[39m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mstate\u001b[39m.\u001b[39mglobal_step \u001b[39m-\u001b[39m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_globalstep_last_logged)\n",
      "File \u001b[0;32m/opt/conda/envs/pytorch/lib/python3.10/site-packages/transformers/trainer.py:2693\u001b[0m, in \u001b[0;36mTrainer.training_step\u001b[0;34m(self, model, inputs)\u001b[0m\n\u001b[1;32m   2691\u001b[0m         scaled_loss\u001b[39m.\u001b[39mbackward()\n\u001b[1;32m   2692\u001b[0m \u001b[39melse\u001b[39;00m:\n\u001b[0;32m-> 2693\u001b[0m     \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49maccelerator\u001b[39m.\u001b[39;49mbackward(loss)\n\u001b[1;32m   2695\u001b[0m \u001b[39mreturn\u001b[39;00m loss\u001b[39m.\u001b[39mdetach() \u001b[39m/\u001b[39m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39margs\u001b[39m.\u001b[39mgradient_accumulation_steps\n",
      "File \u001b[0;32m/opt/conda/envs/pytorch/lib/python3.10/site-packages/accelerate/accelerator.py:1989\u001b[0m, in \u001b[0;36mAccelerator.backward\u001b[0;34m(self, loss, **kwargs)\u001b[0m\n\u001b[1;32m   1987\u001b[0m     \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mscaler\u001b[39m.\u001b[39mscale(loss)\u001b[39m.\u001b[39mbackward(\u001b[39m*\u001b[39m\u001b[39m*\u001b[39mkwargs)\n\u001b[1;32m   1988\u001b[0m \u001b[39melse\u001b[39;00m:\n\u001b[0;32m-> 1989\u001b[0m     loss\u001b[39m.\u001b[39;49mbackward(\u001b[39m*\u001b[39;49m\u001b[39m*\u001b[39;49mkwargs)\n",
      "File \u001b[0;32m/opt/conda/envs/pytorch/lib/python3.10/site-packages/torch/_tensor.py:487\u001b[0m, in \u001b[0;36mTensor.backward\u001b[0;34m(self, gradient, retain_graph, create_graph, inputs)\u001b[0m\n\u001b[1;32m    477\u001b[0m \u001b[39mif\u001b[39;00m has_torch_function_unary(\u001b[39mself\u001b[39m):\n\u001b[1;32m    478\u001b[0m     \u001b[39mreturn\u001b[39;00m handle_torch_function(\n\u001b[1;32m    479\u001b[0m         Tensor\u001b[39m.\u001b[39mbackward,\n\u001b[1;32m    480\u001b[0m         (\u001b[39mself\u001b[39m,),\n\u001b[0;32m   (...)\u001b[0m\n\u001b[1;32m    485\u001b[0m         inputs\u001b[39m=\u001b[39minputs,\n\u001b[1;32m    486\u001b[0m     )\n\u001b[0;32m--> 487\u001b[0m torch\u001b[39m.\u001b[39;49mautograd\u001b[39m.\u001b[39;49mbackward(\n\u001b[1;32m    488\u001b[0m     \u001b[39mself\u001b[39;49m, gradient, retain_graph, create_graph, inputs\u001b[39m=\u001b[39;49minputs\n\u001b[1;32m    489\u001b[0m )\n",
      "File \u001b[0;32m/opt/conda/envs/pytorch/lib/python3.10/site-packages/torch/autograd/__init__.py:200\u001b[0m, in \u001b[0;36mbackward\u001b[0;34m(tensors, grad_tensors, retain_graph, create_graph, grad_variables, inputs)\u001b[0m\n\u001b[1;32m    195\u001b[0m     retain_graph \u001b[39m=\u001b[39m create_graph\n\u001b[1;32m    197\u001b[0m \u001b[39m# The reason we repeat same the comment below is that\u001b[39;00m\n\u001b[1;32m    198\u001b[0m \u001b[39m# some Python versions print out the first line of a multi-line function\u001b[39;00m\n\u001b[1;32m    199\u001b[0m \u001b[39m# calls in the traceback and some print out the last line\u001b[39;00m\n\u001b[0;32m--> 200\u001b[0m Variable\u001b[39m.\u001b[39;49m_execution_engine\u001b[39m.\u001b[39;49mrun_backward(  \u001b[39m# Calls into the C++ engine to run the backward pass\u001b[39;49;00m\n\u001b[1;32m    201\u001b[0m     tensors, grad_tensors_, retain_graph, create_graph, inputs,\n\u001b[1;32m    202\u001b[0m     allow_unreachable\u001b[39m=\u001b[39;49m\u001b[39mTrue\u001b[39;49;00m, accumulate_grad\u001b[39m=\u001b[39;49m\u001b[39mTrue\u001b[39;49;00m)\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "trainer.train()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "model.save_pretrained(\"trained-model\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "config = PeftConfig.from_pretrained(\"trained-model\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "LoraConfig(peft_type=<PeftType.LORA: 'LORA'>, auto_mapping=None, base_model_name_or_path='Qwen/Qwen-14B-Chat-Int4', revision=None, task_type='CAUSAL_LM', inference_mode=True, r=64, target_modules={'w1', 'w2', 'c_attn', 'c_proj'}, lora_alpha=16, lora_dropout=0.05, fan_in_fan_out=False, bias='none', modules_to_save=None, init_lora_weights=True, layers_to_transform=None, layers_pattern=None, rank_pattern={}, alpha_pattern={})"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "config"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "You passed `quantization_config` to `from_pretrained` but the model you're loading already has a `quantization_config` attribute and has already quantized weights. However, loading attributes (e.g. disable_exllama, use_cuda_fp16) will be overwritten with the one you passed to `from_pretrained`. The rest will be ignored.\n",
      "CUDA extension not installed.\n",
      "CUDA extension not installed.\n",
      "Warning: please make sure that you are using the latest codes and checkpoints, especially if you used Qwen-7B before 09.25.2023.请使用最新模型和代码，尤其如果你在9月25日前已经开始使用Qwen-7B，千万注意不要使用错误代码和模型。\n",
      "Try importing flash-attention for faster inference...\n",
      "Warning: import flash_attn rms_norm fail, please install FlashAttention layer_norm to get higher efficiency https://github.com/Dao-AILab/flash-attention/tree/main/csrc/layer_norm\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "5a976105a23d451abcf04c7bcbedd512",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Loading checkpoint shards:   0%|          | 0/5 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "base_model = transformers.AutoModelForCausalLM.from_pretrained(\n",
    "        argmodel.model_name_or_path,\n",
    "        config=config,\n",
    "        device_map =\"cuda\",\n",
    "        trust_remote_code=True,\n",
    "        quantization_config=GPTQConfig(\n",
    "            bits=4, disable_exllama=True\n",
    "        )\n",
    "    )\n",
    "\n",
    "#base_model = prepare_model_for_kbit_training(base_model)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Warning: please make sure that you are using the latest codes and checkpoints, especially if you used Qwen-7B before 09.25.2023.请使用最新模型和代码，尤其如果你在9月25日前已经开始使用Qwen-7B，千万注意不要使用错误代码和模型。\n",
      "Try importing flash-attention for faster inference...\n",
      "Warning: import flash_attn rms_norm fail, please install FlashAttention layer_norm to get higher efficiency https://github.com/Dao-AILab/flash-attention/tree/main/csrc/layer_norm\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "14e15e209d3e46c98c8385a0fdb6b75c",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Loading checkpoint shards:   0%|          | 0/5 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "not_base_model = transformers.AutoModelForCausalLM.from_pretrained(\n",
    "        argmodel.model_name_or_path,\n",
    "        config=config,\n",
    "        device_map =\"cuda\",\n",
    "        trust_remote_code=True,\n",
    "    )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "ename": "ValueError",
     "evalue": "Cannot merge LORA layers when the model is gptq quantized",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mValueError\u001b[0m                                Traceback (most recent call last)",
      "\u001b[1;32m/home/ubuntu/QWEN.ipynb Cell 72\u001b[0m line \u001b[0;36m3\n\u001b[1;32m      <a href='vscode-notebook-cell://ssh-remote%2Bgo_aws/home/ubuntu/QWEN.ipynb#Y131sdnNjb2RlLXJlbW90ZQ%3D%3D?line=0'>1</a>\u001b[0m model_to_merge \u001b[39m=\u001b[39m PeftModel\u001b[39m.\u001b[39mfrom_pretrained(model, \u001b[39m\"\u001b[39m\u001b[39mtrained-model\u001b[39m\u001b[39m\"\u001b[39m)\n\u001b[0;32m----> <a href='vscode-notebook-cell://ssh-remote%2Bgo_aws/home/ubuntu/QWEN.ipynb#Y131sdnNjb2RlLXJlbW90ZQ%3D%3D?line=2'>3</a>\u001b[0m merged_model \u001b[39m=\u001b[39m model_to_merge\u001b[39m.\u001b[39;49mmerge_and_unload()\n\u001b[1;32m      <a href='vscode-notebook-cell://ssh-remote%2Bgo_aws/home/ubuntu/QWEN.ipynb#Y131sdnNjb2RlLXJlbW90ZQ%3D%3D?line=3'>4</a>\u001b[0m merged_model\u001b[39m.\u001b[39msave_pretrained(merged_model)\n",
      "File \u001b[0;32m/opt/conda/envs/pytorch/lib/python3.10/site-packages/peft/tuners/lora/model.py:698\u001b[0m, in \u001b[0;36mLoraModel.merge_and_unload\u001b[0;34m(self, progressbar, safe_merge)\u001b[0m\n\u001b[1;32m    674\u001b[0m \u001b[39mdef\u001b[39;00m \u001b[39mmerge_and_unload\u001b[39m(\u001b[39mself\u001b[39m, progressbar: \u001b[39mbool\u001b[39m \u001b[39m=\u001b[39m \u001b[39mFalse\u001b[39;00m, safe_merge: \u001b[39mbool\u001b[39m \u001b[39m=\u001b[39m \u001b[39mFalse\u001b[39;00m):\n\u001b[1;32m    675\u001b[0m \u001b[39m    \u001b[39m\u001b[39mr\u001b[39m\u001b[39m\"\"\"\u001b[39;00m\n\u001b[1;32m    676\u001b[0m \u001b[39m    This method merges the LoRa layers into the base model. This is needed if someone wants to use the base model\u001b[39;00m\n\u001b[1;32m    677\u001b[0m \u001b[39m    as a standalone model.\u001b[39;00m\n\u001b[0;32m   (...)\u001b[0m\n\u001b[1;32m    696\u001b[0m \u001b[39m    ```\u001b[39;00m\n\u001b[1;32m    697\u001b[0m \u001b[39m    \"\"\"\u001b[39;00m\n\u001b[0;32m--> 698\u001b[0m     \u001b[39mreturn\u001b[39;00m \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49m_unload_and_optionally_merge(progressbar\u001b[39m=\u001b[39;49mprogressbar, safe_merge\u001b[39m=\u001b[39;49msafe_merge)\n",
      "File \u001b[0;32m/opt/conda/envs/pytorch/lib/python3.10/site-packages/peft/tuners/lora/model.py:382\u001b[0m, in \u001b[0;36mLoraModel._unload_and_optionally_merge\u001b[0;34m(self, merge, progressbar, safe_merge)\u001b[0m\n\u001b[1;32m    380\u001b[0m \u001b[39mif\u001b[39;00m merge:\n\u001b[1;32m    381\u001b[0m     \u001b[39mif\u001b[39;00m \u001b[39mgetattr\u001b[39m(\u001b[39mself\u001b[39m\u001b[39m.\u001b[39mmodel, \u001b[39m\"\u001b[39m\u001b[39mquantization_method\u001b[39m\u001b[39m\"\u001b[39m, \u001b[39mNone\u001b[39;00m) \u001b[39m==\u001b[39m \u001b[39m\"\u001b[39m\u001b[39mgptq\u001b[39m\u001b[39m\"\u001b[39m:\n\u001b[0;32m--> 382\u001b[0m         \u001b[39mraise\u001b[39;00m \u001b[39mValueError\u001b[39;00m(\u001b[39m\"\u001b[39m\u001b[39mCannot merge LORA layers when the model is gptq quantized\u001b[39m\u001b[39m\"\u001b[39m)\n\u001b[1;32m    384\u001b[0m key_list \u001b[39m=\u001b[39m [key \u001b[39mfor\u001b[39;00m key, _ \u001b[39min\u001b[39;00m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mmodel\u001b[39m.\u001b[39mnamed_modules() \u001b[39mif\u001b[39;00m \u001b[39m\"\u001b[39m\u001b[39mlora\u001b[39m\u001b[39m\"\u001b[39m \u001b[39mnot\u001b[39;00m \u001b[39min\u001b[39;00m key]\n\u001b[1;32m    385\u001b[0m desc \u001b[39m=\u001b[39m \u001b[39m\"\u001b[39m\u001b[39mUnloading \u001b[39m\u001b[39m\"\u001b[39m \u001b[39m+\u001b[39m (\u001b[39m\"\u001b[39m\u001b[39mand merging \u001b[39m\u001b[39m\"\u001b[39m \u001b[39mif\u001b[39;00m merge \u001b[39melse\u001b[39;00m \u001b[39m\"\u001b[39m\u001b[39m\"\u001b[39m) \u001b[39m+\u001b[39m \u001b[39m\"\u001b[39m\u001b[39mmodel\u001b[39m\u001b[39m\"\u001b[39m\n",
      "\u001b[0;31mValueError\u001b[0m: Cannot merge LORA layers when the model is gptq quantized"
     ]
    }
   ],
   "source": [
    "model_to_merge = PeftModel.from_pretrained(model, \"trained-model\")\n",
    "\n",
    "merged_model = model_to_merge.merge_and_unload()\n",
    "merged_model.save_pretrained(merged_model)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "PeftModelForCausalLM(\n",
       "  (base_model): LoraModel(\n",
       "    (model): PeftModelForCausalLM(\n",
       "      (base_model): LoraModel(\n",
       "        (model): QWenLMHeadModel(\n",
       "          (transformer): QWenModel(\n",
       "            (wte): ModulesToSaveWrapper(\n",
       "              (original_module): Embedding(152064, 5120)\n",
       "              (modules_to_save): ModuleDict(\n",
       "                (default): Embedding(152064, 5120)\n",
       "              )\n",
       "            )\n",
       "            (drop): Dropout(p=0.0, inplace=False)\n",
       "            (rotary_emb): RotaryEmbedding()\n",
       "            (h): ModuleList(\n",
       "              (0-39): 40 x QWenBlock(\n",
       "                (ln_1): RMSNorm()\n",
       "                (attn): QWenAttention(\n",
       "                  (core_attention_flash): FlashSelfAttention()\n",
       "                  (attn_dropout): Dropout(p=0.0, inplace=False)\n",
       "                  (c_attn): QuantLinear(\n",
       "                    (lora_dropout): ModuleDict(\n",
       "                      (default): Dropout(p=0.05, inplace=False)\n",
       "                    )\n",
       "                    (lora_A): ModuleDict(\n",
       "                      (default): Linear(in_features=5120, out_features=64, bias=False)\n",
       "                    )\n",
       "                    (lora_B): ModuleDict(\n",
       "                      (default): Linear(in_features=64, out_features=15360, bias=False)\n",
       "                    )\n",
       "                    (lora_embedding_A): ParameterDict()\n",
       "                    (lora_embedding_B): ParameterDict()\n",
       "                    (quant_linear_module): QuantLinear()\n",
       "                  )\n",
       "                  (c_proj): QuantLinear(\n",
       "                    (lora_dropout): ModuleDict(\n",
       "                      (default): Dropout(p=0.05, inplace=False)\n",
       "                    )\n",
       "                    (lora_A): ModuleDict(\n",
       "                      (default): Linear(in_features=5120, out_features=64, bias=False)\n",
       "                    )\n",
       "                    (lora_B): ModuleDict(\n",
       "                      (default): Linear(in_features=64, out_features=5120, bias=False)\n",
       "                    )\n",
       "                    (lora_embedding_A): ParameterDict()\n",
       "                    (lora_embedding_B): ParameterDict()\n",
       "                    (quant_linear_module): QuantLinear()\n",
       "                  )\n",
       "                )\n",
       "                (ln_2): RMSNorm()\n",
       "                (mlp): QWenMLP(\n",
       "                  (c_proj): QuantLinear(\n",
       "                    (lora_dropout): ModuleDict(\n",
       "                      (default): Dropout(p=0.05, inplace=False)\n",
       "                    )\n",
       "                    (lora_A): ModuleDict(\n",
       "                      (default): Linear(in_features=13696, out_features=64, bias=False)\n",
       "                    )\n",
       "                    (lora_B): ModuleDict(\n",
       "                      (default): Linear(in_features=64, out_features=5120, bias=False)\n",
       "                    )\n",
       "                    (lora_embedding_A): ParameterDict()\n",
       "                    (lora_embedding_B): ParameterDict()\n",
       "                    (quant_linear_module): QuantLinear()\n",
       "                  )\n",
       "                  (w1): QuantLinear(\n",
       "                    (lora_dropout): ModuleDict(\n",
       "                      (default): Dropout(p=0.05, inplace=False)\n",
       "                    )\n",
       "                    (lora_A): ModuleDict(\n",
       "                      (default): Linear(in_features=5120, out_features=64, bias=False)\n",
       "                    )\n",
       "                    (lora_B): ModuleDict(\n",
       "                      (default): Linear(in_features=64, out_features=13696, bias=False)\n",
       "                    )\n",
       "                    (lora_embedding_A): ParameterDict()\n",
       "                    (lora_embedding_B): ParameterDict()\n",
       "                    (quant_linear_module): QuantLinear()\n",
       "                  )\n",
       "                  (w2): QuantLinear(\n",
       "                    (lora_dropout): ModuleDict(\n",
       "                      (default): Dropout(p=0.05, inplace=False)\n",
       "                    )\n",
       "                    (lora_A): ModuleDict(\n",
       "                      (default): Linear(in_features=5120, out_features=64, bias=False)\n",
       "                    )\n",
       "                    (lora_B): ModuleDict(\n",
       "                      (default): Linear(in_features=64, out_features=13696, bias=False)\n",
       "                    )\n",
       "                    (lora_embedding_A): ParameterDict()\n",
       "                    (lora_embedding_B): ParameterDict()\n",
       "                    (quant_linear_module): QuantLinear()\n",
       "                  )\n",
       "                )\n",
       "              )\n",
       "            )\n",
       "            (ln_f): RMSNorm()\n",
       "          )\n",
       "          (lm_head): ModulesToSaveWrapper(\n",
       "            (original_module): Linear(in_features=5120, out_features=152064, bias=False)\n",
       "            (modules_to_save): ModuleDict(\n",
       "              (default): Linear(in_features=5120, out_features=152064, bias=False)\n",
       "            )\n",
       "          )\n",
       "        )\n",
       "      )\n",
       "    )\n",
       "  )\n",
       ")"
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model_to_merge"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.float16"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.base_model_torch_dtype"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "PeftModelForCausalLM(\n",
       "  (base_model): LoraModel(\n",
       "    (model): QWenLMHeadModel(\n",
       "      (transformer): QWenModel(\n",
       "        (wte): ModulesToSaveWrapper(\n",
       "          (original_module): Embedding(152064, 5120)\n",
       "          (modules_to_save): ModuleDict(\n",
       "            (default): Embedding(152064, 5120)\n",
       "          )\n",
       "        )\n",
       "        (drop): Dropout(p=0.0, inplace=False)\n",
       "        (rotary_emb): RotaryEmbedding()\n",
       "        (h): ModuleList(\n",
       "          (0-39): 40 x QWenBlock(\n",
       "            (ln_1): RMSNorm()\n",
       "            (attn): QWenAttention(\n",
       "              (core_attention_flash): FlashSelfAttention()\n",
       "              (attn_dropout): Dropout(p=0.0, inplace=False)\n",
       "              (c_attn): QuantLinear(\n",
       "                (lora_dropout): ModuleDict(\n",
       "                  (default): Dropout(p=0.05, inplace=False)\n",
       "                )\n",
       "                (lora_A): ModuleDict(\n",
       "                  (default): Linear(in_features=5120, out_features=64, bias=False)\n",
       "                )\n",
       "                (lora_B): ModuleDict(\n",
       "                  (default): Linear(in_features=64, out_features=15360, bias=False)\n",
       "                )\n",
       "                (lora_embedding_A): ParameterDict()\n",
       "                (lora_embedding_B): ParameterDict()\n",
       "                (quant_linear_module): QuantLinear()\n",
       "              )\n",
       "              (c_proj): QuantLinear(\n",
       "                (lora_dropout): ModuleDict(\n",
       "                  (default): Dropout(p=0.05, inplace=False)\n",
       "                )\n",
       "                (lora_A): ModuleDict(\n",
       "                  (default): Linear(in_features=5120, out_features=64, bias=False)\n",
       "                )\n",
       "                (lora_B): ModuleDict(\n",
       "                  (default): Linear(in_features=64, out_features=5120, bias=False)\n",
       "                )\n",
       "                (lora_embedding_A): ParameterDict()\n",
       "                (lora_embedding_B): ParameterDict()\n",
       "                (quant_linear_module): QuantLinear()\n",
       "              )\n",
       "            )\n",
       "            (ln_2): RMSNorm()\n",
       "            (mlp): QWenMLP(\n",
       "              (c_proj): QuantLinear(\n",
       "                (lora_dropout): ModuleDict(\n",
       "                  (default): Dropout(p=0.05, inplace=False)\n",
       "                )\n",
       "                (lora_A): ModuleDict(\n",
       "                  (default): Linear(in_features=13696, out_features=64, bias=False)\n",
       "                )\n",
       "                (lora_B): ModuleDict(\n",
       "                  (default): Linear(in_features=64, out_features=5120, bias=False)\n",
       "                )\n",
       "                (lora_embedding_A): ParameterDict()\n",
       "                (lora_embedding_B): ParameterDict()\n",
       "                (quant_linear_module): QuantLinear()\n",
       "              )\n",
       "              (w1): QuantLinear(\n",
       "                (lora_dropout): ModuleDict(\n",
       "                  (default): Dropout(p=0.05, inplace=False)\n",
       "                )\n",
       "                (lora_A): ModuleDict(\n",
       "                  (default): Linear(in_features=5120, out_features=64, bias=False)\n",
       "                )\n",
       "                (lora_B): ModuleDict(\n",
       "                  (default): Linear(in_features=64, out_features=13696, bias=False)\n",
       "                )\n",
       "                (lora_embedding_A): ParameterDict()\n",
       "                (lora_embedding_B): ParameterDict()\n",
       "                (quant_linear_module): QuantLinear()\n",
       "              )\n",
       "              (w2): QuantLinear(\n",
       "                (lora_dropout): ModuleDict(\n",
       "                  (default): Dropout(p=0.05, inplace=False)\n",
       "                )\n",
       "                (lora_A): ModuleDict(\n",
       "                  (default): Linear(in_features=5120, out_features=64, bias=False)\n",
       "                )\n",
       "                (lora_B): ModuleDict(\n",
       "                  (default): Linear(in_features=64, out_features=13696, bias=False)\n",
       "                )\n",
       "                (lora_embedding_A): ParameterDict()\n",
       "                (lora_embedding_B): ParameterDict()\n",
       "                (quant_linear_module): QuantLinear()\n",
       "              )\n",
       "            )\n",
       "          )\n",
       "        )\n",
       "        (ln_f): RMSNorm()\n",
       "      )\n",
       "      (lm_head): ModulesToSaveWrapper(\n",
       "        (original_module): Linear(in_features=5120, out_features=152064, bias=False)\n",
       "        (modules_to_save): ModuleDict(\n",
       "          (default): Linear(in_features=5120, out_features=152064, bias=False)\n",
       "        )\n",
       "      )\n",
       "    )\n",
       "  )\n",
       ")"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "pytorch",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}