BERT_Text_Classification_CPU.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Text Classification with BERT in Pytorch"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "@description: It is a Python3 file to implement text classification with BERT in Pytorch.\n",
    "\n",
    "@author: Yantong Lai\n",
    "\n",
    "@date: 01/14/2020"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "xYRNlKwDcjuG"
   },
   "source": [
    "## 0. Install transformers"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 598
    },
    "colab_type": "code",
    "id": "GYNo1pdapqq0",
    "outputId": "049a7b40-d8ff-4afb-c94a-2b0614afc3a1"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Requirement already satisfied: transformers in /usr/local/lib/python3.7/site-packages (2.3.0)\n",
      "Requirement already satisfied: sentencepiece in /usr/local/lib/python3.7/site-packages (from transformers) (0.1.83)\n",
      "Requirement already satisfied: requests in /usr/local/lib/python3.7/site-packages (from transformers) (2.22.0)\n",
      "Requirement already satisfied: numpy in /usr/local/lib/python3.7/site-packages (from transformers) (1.18.1)\n",
      "Requirement already satisfied: tqdm in /usr/local/lib/python3.7/site-packages (from transformers) (4.36.1)\n",
      "Requirement already satisfied: regex!=2019.12.17 in /usr/local/lib/python3.7/site-packages (from transformers) (2019.8.19)\n",
      "Requirement already satisfied: sacremoses in /usr/local/lib/python3.7/site-packages (from transformers) (0.0.35)\n",
      "Requirement already satisfied: boto3 in /usr/local/lib/python3.7/site-packages (from transformers) (1.10.2)\n",
      "Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /usr/local/lib/python3.7/site-packages (from requests->transformers) (1.25.6)\n",
      "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.7/site-packages (from requests->transformers) (2019.9.11)\n",
      "Requirement already satisfied: chardet<3.1.0,>=3.0.2 in /usr/local/lib/python3.7/site-packages (from requests->transformers) (3.0.4)\n",
      "Requirement already satisfied: idna<2.9,>=2.5 in /usr/local/lib/python3.7/site-packages (from requests->transformers) (2.8)\n",
      "Requirement already satisfied: six in /usr/local/lib/python3.7/site-packages (from sacremoses->transformers) (1.12.0)\n",
      "Requirement already satisfied: joblib in /usr/local/lib/python3.7/site-packages (from sacremoses->transformers) (0.14.0)\n",
      "Requirement already satisfied: click in /usr/local/lib/python3.7/site-packages (from sacremoses->transformers) (7.0)\n",
      "Requirement already satisfied: s3transfer<0.3.0,>=0.2.0 in /usr/local/lib/python3.7/site-packages (from boto3->transformers) (0.2.1)\n",
      "Requirement already satisfied: jmespath<1.0.0,>=0.7.1 in /usr/local/lib/python3.7/site-packages (from boto3->transformers) (0.9.4)\n",
      "Requirement already satisfied: botocore<1.14.0,>=1.13.2 in /usr/local/lib/python3.7/site-packages (from boto3->transformers) (1.13.2)\n",
      "Requirement already satisfied: python-dateutil<3.0.0,>=2.1; python_version >= \"2.7\" in /usr/local/lib/python3.7/site-packages (from botocore<1.14.0,>=1.13.2->boto3->transformers) (2.8.0)\n",
      "Requirement already satisfied: docutils<0.16,>=0.10 in /usr/local/lib/python3.7/site-packages (from botocore<1.14.0,>=1.13.2->boto3->transformers) (0.15.2)\n"
     ]
    }
   ],
   "source": [
    "! pip3 install transformers"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 80
    },
    "colab_type": "code",
    "id": "3wKQArDbU2Y4",
    "outputId": "341f67e0-02b5-48ae-f37c-0e46da3d15c5"
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.7/site-packages/tensorflow/python/framework/dtypes.py:516: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_qint8 = np.dtype([(\"qint8\", np.int8, 1)])\n",
      "/usr/local/lib/python3.7/site-packages/tensorflow/python/framework/dtypes.py:517: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_quint8 = np.dtype([(\"quint8\", np.uint8, 1)])\n",
      "/usr/local/lib/python3.7/site-packages/tensorflow/python/framework/dtypes.py:518: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_qint16 = np.dtype([(\"qint16\", np.int16, 1)])\n",
      "/usr/local/lib/python3.7/site-packages/tensorflow/python/framework/dtypes.py:519: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_quint16 = np.dtype([(\"quint16\", np.uint16, 1)])\n",
      "/usr/local/lib/python3.7/site-packages/tensorflow/python/framework/dtypes.py:520: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_qint32 = np.dtype([(\"qint32\", np.int32, 1)])\n",
      "/usr/local/lib/python3.7/site-packages/tensorflow/python/framework/dtypes.py:525: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  np_resource = np.dtype([(\"resource\", np.ubyte, 1)])\n",
      "/usr/local/lib/python3.7/site-packages/tensorboard/compat/tensorflow_stub/dtypes.py:541: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_qint8 = np.dtype([(\"qint8\", np.int8, 1)])\n",
      "/usr/local/lib/python3.7/site-packages/tensorboard/compat/tensorflow_stub/dtypes.py:542: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_quint8 = np.dtype([(\"quint8\", np.uint8, 1)])\n",
      "/usr/local/lib/python3.7/site-packages/tensorboard/compat/tensorflow_stub/dtypes.py:543: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_qint16 = np.dtype([(\"qint16\", np.int16, 1)])\n",
      "/usr/local/lib/python3.7/site-packages/tensorboard/compat/tensorflow_stub/dtypes.py:544: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_quint16 = np.dtype([(\"quint16\", np.uint16, 1)])\n",
      "/usr/local/lib/python3.7/site-packages/tensorboard/compat/tensorflow_stub/dtypes.py:545: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_qint32 = np.dtype([(\"qint32\", np.int32, 1)])\n",
      "/usr/local/lib/python3.7/site-packages/tensorboard/compat/tensorflow_stub/dtypes.py:550: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  np_resource = np.dtype([(\"resource\", np.ubyte, 1)])\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "'2.3.0'"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import transformers\n",
    "\n",
    "transformers.__version__"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "igCBreITXoSG"
   },
   "source": [
    "## 1. Data Pre-processing"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "9VZmrKLTX2wF"
   },
   "source": [
    "### 1.1 Configure CPU"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 34
    },
    "colab_type": "code",
    "id": "o_lplUg3XrVe",
    "outputId": "1c26fe99-cfd1-4627-924f-55f06c576bb1"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "device(type='cpu')"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import torch\n",
    "\n",
    "device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n",
    "\n",
    "device"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "Uj8SMwkZYNX6"
   },
   "source": [
    "### 1.2 Read dataset file"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "f3too4KCtRsu"
   },
   "source": [
    "We use the Yelp provided polarity dataset for text classification. You could download the Yelp polarity dataset via [link](https://www.kaggle.com/irustandi/yelp-review-polarity/version/1#test.csv) in Kaggle."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "YdREZ_lRYPRY"
   },
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "\n",
    "train_filename = \"yelp_review_polarity_csv/train.csv\"\n",
    "test_filename = \"yelp_review_polarity_csv/test.csv\"\n",
    "\n",
    "df_train = pd.read_csv(train_filename, header=None, names=['label', 'text'])\n",
    "df_test = pd.read_csv(test_filename, header=None, names=['label', 'text'])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Take a look at the train data and test data."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 204
    },
    "colab_type": "code",
    "id": "I3xB6E5arl0M",
    "outputId": "e914efb2-e096-4155-9425-5c7649926c95"
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>label</th>\n",
       "      <th>text</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>Unfortunately, the frustration of being Dr. Go...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2</td>\n",
       "      <td>Been going to Dr. Goldberg for over 10 years. ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1</td>\n",
       "      <td>I don't know what Dr. Goldberg was like before...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1</td>\n",
       "      <td>I'm writing this review to give you a heads up...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>2</td>\n",
       "      <td>All the food is great here. But the best thing...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   label                                               text\n",
       "0      1  Unfortunately, the frustration of being Dr. Go...\n",
       "1      2  Been going to Dr. Goldberg for over 10 years. ...\n",
       "2      1  I don't know what Dr. Goldberg was like before...\n",
       "3      1  I'm writing this review to give you a heads up...\n",
       "4      2  All the food is great here. But the best thing..."
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_train.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 204
    },
    "colab_type": "code",
    "id": "JmrZ6fFTsJtj",
    "outputId": "a6a6a1e0-c6fd-4ca5-8e06-781d76b6fa49"
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>label</th>\n",
       "      <th>text</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2</td>\n",
       "      <td>Contrary to other reviews, I have zero complai...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>Last summer I had an appointment to get new ti...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>Friendly staff, same starbucks fair you get an...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1</td>\n",
       "      <td>The food is good. Unfortunately the service is...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>2</td>\n",
       "      <td>Even when we didn't have a car Filene's Baseme...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   label                                               text\n",
       "0      2  Contrary to other reviews, I have zero complai...\n",
       "1      1  Last summer I had an appointment to get new ti...\n",
       "2      2  Friendly staff, same starbucks fair you get an...\n",
       "3      1  The food is good. Unfortunately the service is...\n",
       "4      2  Even when we didn't have a car Filene's Baseme..."
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_test.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Sample 10 items which label is 1."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 359
    },
    "colab_type": "code",
    "id": "YjK2ldiypY4G",
    "outputId": "4881d0bd-94fe-4b9b-fbf6-ed76521d9912"
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>label</th>\n",
       "      <th>text</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>269164</th>\n",
       "      <td>1</td>\n",
       "      <td>Reading the other reviews, I think we must hav...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23975</th>\n",
       "      <td>1</td>\n",
       "      <td>If you want pizza, go anywhere else, anywhere!...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>31799</th>\n",
       "      <td>1</td>\n",
       "      <td>I scheduled an appointment and no one showed u...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>266851</th>\n",
       "      <td>1</td>\n",
       "      <td>I'm a bit of a sushi snob, so when I heard tha...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>156393</th>\n",
       "      <td>1</td>\n",
       "      <td>So I took my parents here during Christmas Day...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>211913</th>\n",
       "      <td>1</td>\n",
       "      <td>The only reason I keep going back to this loca...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>211693</th>\n",
       "      <td>1</td>\n",
       "      <td>This is not a terrible place, but it's far fro...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>187305</th>\n",
       "      <td>1</td>\n",
       "      <td>NOT THE LEAST BIT AUTHENTIC!!!!\\n\\nI can't get...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>269394</th>\n",
       "      <td>1</td>\n",
       "      <td>I went here recently for lunch. I had the Kung...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>53115</th>\n",
       "      <td>1</td>\n",
       "      <td>Once again, I came back and gave this place an...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        label                                               text\n",
       "269164      1  Reading the other reviews, I think we must hav...\n",
       "23975       1  If you want pizza, go anywhere else, anywhere!...\n",
       "31799       1  I scheduled an appointment and no one showed u...\n",
       "266851      1  I'm a bit of a sushi snob, so when I heard tha...\n",
       "156393      1  So I took my parents here during Christmas Day...\n",
       "211913      1  The only reason I keep going back to this loca...\n",
       "211693      1  This is not a terrible place, but it's far fro...\n",
       "187305      1  NOT THE LEAST BIT AUTHENTIC!!!!\\n\\nI can't get...\n",
       "269394      1  I went here recently for lunch. I had the Kung...\n",
       "53115       1  Once again, I came back and gave this place an..."
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_train.loc[df_train.label == 1].sample(10)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "YhN1TCoDsTXO"
   },
   "source": [
    "### 1.3 Get the values of DataFrame"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 34
    },
    "colab_type": "code",
    "id": "framCWCisS2C",
    "outputId": "8af33237-30e6-451d-9fba-1d424bd01c63"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "len(train_text) = 560000, len(test_text) = 38000\n"
     ]
    }
   ],
   "source": [
    "train_labels = df_train['label'].tolist()\n",
    "train_text = df_train['text'].tolist()\n",
    "\n",
    "test_labels = df_test['label'].tolist()\n",
    "test_text = df_test['text'].tolist()\n",
    "\n",
    "assert len(train_labels) == len(train_text)\n",
    "assert len(test_labels) == len(test_text)\n",
    "\n",
    "print(\"len(train_text) = {}, len(test_text) = {}\".format(len(train_text), len(test_text)))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "FaGNI_8nU2ZI"
   },
   "source": [
    "### 1.4 Get values indices"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The label in the dataset is `1` and `2`. When training in BERT, we should format the label to `0` and `1`."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "pLC4M3lhtrSY"
   },
   "outputs": [],
   "source": [
    "def format_label_value(label_list):\n",
    "    \"\"\"\n",
    "    It is a function to format label.\n",
    "    \"\"\"\n",
    "    format_label_list = []\n",
    "\n",
    "    for label in label_list:\n",
    "        format_label_list.append(label - 1)\n",
    "    \n",
    "    return format_label_list"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We just wanna learn BERT in the text classification task on our own computer, with the limitation of memory, we need to get the indices of dataset.\n",
    "\n",
    "**Orig**:\n",
    "- len(train_text) = 560000\n",
    "- len(test_text) = 38000\n",
    "\n",
    "**Indice according to the ratio `0.001`**:\n",
    "- len(train_text) = 560\n",
    "- len(test_text) = 38"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "E7IPdKcMU2ZI"
   },
   "outputs": [],
   "source": [
    "train_labels, train_text = train_labels[:560], train_text[:560]\n",
    "test_labels, test_text = test_labels[:38], test_text[:38]\n",
    "\n",
    "train_labels = format_label_value(train_labels)\n",
    "test_labels = format_label_value(test_labels)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "nRjPgJPls6Lp"
   },
   "source": [
    "## 2. BERT Tokenization & Input Formatting"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "ELfeERHEtc6z"
   },
   "source": [
    "### 2.1 BERT Tokenization"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 34
    },
    "colab_type": "code",
    "id": "MK4-dxRVtfXB",
    "outputId": "94dea51c-6c93-4dac-e977-b08173a82117"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Loading BERT tokenizer...\n"
     ]
    }
   ],
   "source": [
    "from transformers import BertTokenizer\n",
    "\n",
    "# Load the BERT tokenizer.\n",
    "print('Loading BERT tokenizer...')\n",
    "tokenizer = BertTokenizer.from_pretrained('bert-base-uncased', do_lower_case=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "wEfc5EzmtqDn"
   },
   "source": [
    "**Let's apply the tokenizer to one sentence just to see the output.**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 241
    },
    "colab_type": "code",
    "id": "LXQLHqiBtm06",
    "outputId": "639b8910-95ac-497b-a316-68a3ca2563a7"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Original:  Unfortunately, the frustration of being Dr. Goldberg's patient is a repeat of the experience I've had with so many other doctors in NYC -- good doctor, terrible staff.  It seems that his staff simply never answers the phone.  It usually takes 2 hours of repeated calling to get an answer.  Who has time for that or wants to deal with it?  I have run into this problem with many other doctors and I just don't get it.  You have office workers, you have patients with medical needs, why isn't anyone answering the phone?  It's incomprehensible and not work the aggravation.  It's with regret that I feel that I have to give Dr. Goldberg 2 stars.\n",
      "len(Original) =  643\n",
      "\n",
      "\n",
      "Tokenized:  ['unfortunately', ',', 'the', 'frustration', 'of', 'being', 'dr', '.', 'goldberg', \"'\", 's', 'patient', 'is', 'a', 'repeat', 'of', 'the', 'experience', 'i', \"'\", 've', 'had', 'with', 'so', 'many', 'other', 'doctors', 'in', 'nyc', '-', '-', 'good', 'doctor', ',', 'terrible', 'staff', '.', 'it', 'seems', 'that', 'his', 'staff', 'simply', 'never', 'answers', 'the', 'phone', '.', 'it', 'usually', 'takes', '2', 'hours', 'of', 'repeated', 'calling', 'to', 'get', 'an', 'answer', '.', 'who', 'has', 'time', 'for', 'that', 'or', 'wants', 'to', 'deal', 'with', 'it', '?', 'i', 'have', 'run', 'into', 'this', 'problem', 'with', 'many', 'other', 'doctors', 'and', 'i', 'just', 'don', \"'\", 't', 'get', 'it', '.', 'you', 'have', 'office', 'workers', ',', 'you', 'have', 'patients', 'with', 'medical', 'needs', ',', 'why', 'isn', \"'\", 't', 'anyone', 'answering', 'the', 'phone', '?', 'it', \"'\", 's', 'inc', '##omp', '##re', '##hen', '##sible', 'and', 'not', 'work', 'the', 'ag', '##gra', '##vation', '.', 'it', \"'\", 's', 'with', 'regret', 'that', 'i', 'feel', 'that', 'i', 'have', 'to', 'give', 'dr', '.', 'goldberg', '2', 'stars', '.']\n",
      "len(Tokenized) =  148\n",
      "\n",
      "\n",
      "Token IDs:  [6854, 1010, 1996, 9135, 1997, 2108, 2852, 1012, 18522, 1005, 1055, 5776, 2003, 1037, 9377, 1997, 1996, 3325, 1045, 1005, 2310, 2018, 2007, 2061, 2116, 2060, 7435, 1999, 16392, 1011, 1011, 2204, 3460, 1010, 6659, 3095, 1012, 2009, 3849, 2008, 2010, 3095, 3432, 2196, 6998, 1996, 3042, 1012, 2009, 2788, 3138, 1016, 2847, 1997, 5567, 4214, 2000, 2131, 2019, 3437, 1012, 2040, 2038, 2051, 2005, 2008, 2030, 4122, 2000, 3066, 2007, 2009, 1029, 1045, 2031, 2448, 2046, 2023, 3291, 2007, 2116, 2060, 7435, 1998, 1045, 2074, 2123, 1005, 1056, 2131, 2009, 1012, 2017, 2031, 2436, 3667, 1010, 2017, 2031, 5022, 2007, 2966, 3791, 1010, 2339, 3475, 1005, 1056, 3087, 10739, 1996, 3042, 1029, 2009, 1005, 1055, 4297, 25377, 2890, 10222, 19307, 1998, 2025, 2147, 1996, 12943, 17643, 21596, 1012, 2009, 1005, 1055, 2007, 9038, 2008, 1045, 2514, 2008, 1045, 2031, 2000, 2507, 2852, 1012, 18522, 1016, 3340, 1012]\n",
      "len(Token IDs) =  148\n",
      "\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# Print the original sentence.\n",
    "print('Original: ', train_text[0])\n",
    "print(\"len(Original) = \", len(train_text[0]))\n",
    "print(\"\\n\")\n",
    "\n",
    "# Print the sentence split into tokens.\n",
    "print('Tokenized: ', tokenizer.tokenize(train_text[0]))\n",
    "print(\"len(Tokenized) = \", len(tokenizer.tokenize(train_text[0])))\n",
    "print(\"\\n\")\n",
    "\n",
    "# Print the sentence mapped to token ids.\n",
    "print('Token IDs: ', tokenizer.convert_tokens_to_ids(tokenizer.tokenize(train_text[0])))\n",
    "print(\"len(Token IDs) = \", len(tokenizer.convert_tokens_to_ids(tokenizer.tokenize(train_text[0]))))\n",
    "print(\"\\n\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "RmjWYNTwuyDU"
   },
   "source": [
    "### 2.2 Input Formatting for BERT"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "oRoHLMIeu13j"
   },
   "source": [
    "**BERT needs adding special tokens --- [cls] and [sep]**\n",
    "\n",
    "The `tokenizer.encode` function combines multiple steps for us:\n",
    "1. Split the sentence into tokens.\n",
    "2. Add the special `[CLS]` and `[SEP]` tokens.\n",
    "3. Map the tokens to their IDs.\n",
    "\n",
    "Oddly, this function can perform truncating for us, but doesn't handle padding. "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "5gLHpQQrvwtC"
   },
   "source": [
    "**Encoding for text in training dataset**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "SQtMRAGRU2ZN"
   },
   "outputs": [],
   "source": [
    "import logging\n",
    "\n",
    "# Set logger to avoid warning `token indices sequence length is longer than the specified maximum sequence length for this model (1017 > 512)`\n",
    "logging.getLogger(\"transformers.tokenization_utils\").setLevel(logging.ERROR)\n",
    "\n",
    "\n",
    "def text_to_id(tokenizer, text_list):\n",
    "    \"\"\"\n",
    "    It is a function to transform text to id.\n",
    "    # `encode` will:\n",
    "    #   (1) Tokenize the sentence.\n",
    "    #   (2) Prepend the `[CLS]` token to the start.\n",
    "    #   (3) Append the `[SEP]` token to the end.\n",
    "    #   (4) Map tokens to their IDs.\n",
    "    \"\"\"\n",
    "    ids_list = []\n",
    "    \n",
    "    for item in text_list:\n",
    "        # Sentence to id and add [CLS] and [SEP]\n",
    "        encoded_item = tokenizer.encode(item, add_special_tokens=True)\n",
    "        ids_list.append(encoded_item)\n",
    "    \n",
    "    return ids_list"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 156
    },
    "colab_type": "code",
    "id": "LkGxQHFiu_zg",
    "outputId": "0fceb504-089e-44ba-df9b-a15ceceb3e4f"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Original: Unfortunately, the frustration of being Dr. Goldberg's patient is a repeat of the experience I've had with so many other doctors in NYC -- good doctor, terrible staff.  It seems that his staff simply never answers the phone.  It usually takes 2 hours of repeated calling to get an answer.  Who has time for that or wants to deal with it?  I have run into this problem with many other doctors and I just don't get it.  You have office workers, you have patients with medical needs, why isn't anyone answering the phone?  It's incomprehensible and not work the aggravation.  It's with regret that I feel that I have to give Dr. Goldberg 2 stars.\n",
      "\n",
      "Token IDs: [101, 6854, 1010, 1996, 9135, 1997, 2108, 2852, 1012, 18522, 1005, 1055, 5776, 2003, 1037, 9377, 1997, 1996, 3325, 1045, 1005, 2310, 2018, 2007, 2061, 2116, 2060, 7435, 1999, 16392, 1011, 1011, 2204, 3460, 1010, 6659, 3095, 1012, 2009, 3849, 2008, 2010, 3095, 3432, 2196, 6998, 1996, 3042, 1012, 2009, 2788, 3138, 1016, 2847, 1997, 5567, 4214, 2000, 2131, 2019, 3437, 1012, 2040, 2038, 2051, 2005, 2008, 2030, 4122, 2000, 3066, 2007, 2009, 1029, 1045, 2031, 2448, 2046, 2023, 3291, 2007, 2116, 2060, 7435, 1998, 1045, 2074, 2123, 1005, 1056, 2131, 2009, 1012, 2017, 2031, 2436, 3667, 1010, 2017, 2031, 5022, 2007, 2966, 3791, 1010, 2339, 3475, 1005, 1056, 3087, 10739, 1996, 3042, 1029, 2009, 1005, 1055, 4297, 25377, 2890, 10222, 19307, 1998, 2025, 2147, 1996, 12943, 17643, 21596, 1012, 2009, 1005, 1055, 2007, 9038, 2008, 1045, 2514, 2008, 1045, 2031, 2000, 2507, 2852, 1012, 18522, 1016, 3340, 1012, 102]\n",
      "\n",
      "len(train_text_ids) = 560\n",
      "\n",
      "len(test_text_ids) = 38\n"
     ]
    }
   ],
   "source": [
    "train_text_ids = text_to_id(tokenizer, train_text)\n",
    "test_text_ids = text_to_id(tokenizer, test_text)\n",
    "\n",
    "\n",
    "# Print sentence 0, now as a list of IDs.\n",
    "print('Original: {}\\n'.format(train_text[0]))\n",
    "print('Token IDs: {}\\n'.format(train_text_ids[0]))\n",
    "print(\"len(train_text_ids) = {}\\n\".format(len(train_text_ids)))\n",
    "print(\"len(test_text_ids) = {}\".format(len(test_text_ids)))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "YqQFZsj8U2ZP"
   },
   "source": [
    "**Calculate the maximum length of train_text_ids**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 85
    },
    "colab_type": "code",
    "id": "hlqsbXzrU2ZP",
    "outputId": "4905dc1c-0e21-49fa-a1b8-b1a582b8a308"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train: max sentence length:  1019\n",
      "Train: Min sentence length:  3\n",
      "Test: max sentence length:  661\n",
      "Test: Min sentence length:  20\n"
     ]
    }
   ],
   "source": [
    "print('Train: max sentence length: ', max([len(sen) for sen in train_text_ids]))\n",
    "print('Train: Min sentence length: ', min([len(sen) for sen in train_text_ids]))\n",
    "print('Test: max sentence length: ', max([len(sen) for sen in test_text_ids]))\n",
    "print('Test: Min sentence length: ', min([len(sen) for sen in test_text_ids]))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "From above results, we could find that the length of each sentence in `train_text_ids` is not the same, so we need to pad or truncate the text ids."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "v5v7RzzRU2ZR"
   },
   "source": [
    "### 2.3 Padding & Truncating"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "JGqIxB-cU2ZR"
   },
   "outputs": [],
   "source": [
    "def padding_truncating(input_ids_list, max_length):\n",
    "    \"\"\"\n",
    "    It is a function to perform padding and truncating\n",
    "    @param input_ids_list: <List> text_ids\n",
    "    @param max_length: <Integer> the number we wanna the sentence to be padding or truncating\n",
    "    @return: processed input_ids_list\n",
    "    \"\"\"\n",
    "    processed_input_ids_list = []\n",
    "    for item in input_ids_list:\n",
    "        seq_list = []\n",
    "        \n",
    "        if len(item) < max_length:\n",
    "            # Define a seq_list with the length of max_length\n",
    "            seq_list = [0] * (max_length - len(item))\n",
    "            item = item + seq_list\n",
    "        \n",
    "        elif len(item) >= max_length:\n",
    "            item = item[:max_length]\n",
    "            \n",
    "        processed_input_ids_list.append(item)\n",
    "    \n",
    "    return processed_input_ids_list"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Padding or truncating the `train_text_ids` and `test_text_ids`**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "gV6fn36hU2ZS"
   },
   "outputs": [],
   "source": [
    "train_padding_list = padding_truncating(train_text_ids, max_length=50)\n",
    "test_padding_list = padding_truncating(test_text_ids, max_length=50)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "XO3ZTKcbU2ZT"
   },
   "source": [
    "### 2.4 Attention Masks"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "Si3U82xDU2ZT"
   },
   "source": [
    "The attention mask simply makes it explicit which tokens are actual words versus which are padding. The `BERT` vocabulary does not use the ID 0, so **if a token ID is 0, it's a padded one, and otherwise it's a real token**."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "3h1qSkXeU2ZT"
   },
   "outputs": [],
   "source": [
    "def get_attention_masks(pad_input_ids_list):\n",
    "    \"\"\"\n",
    "    It is a function to get attention masks:\n",
    "    \n",
    "    - If a token ID is 0, then it's padding, set the mask to 0.\n",
    "    - If a token ID is > 0, then it's a real token, set the mask to 1.\n",
    "    \"\"\"\n",
    "    attention_masks_list = []\n",
    "    \n",
    "    for item in pad_input_ids_list:\n",
    "        \n",
    "        mask_list = []\n",
    "        for subitem in item:\n",
    "            if subitem > 0:\n",
    "                mask_list.append(1)\n",
    "            else:\n",
    "                mask_list.append(0)\n",
    "        attention_masks_list.append(mask_list)\n",
    "    \n",
    "    return attention_masks_list"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "xzUxl2IxU2ZV"
   },
   "outputs": [],
   "source": [
    "train_attention_masks = get_attention_masks(train_padding_list)\n",
    "test_attention_masks = get_attention_masks(test_padding_list)\n",
    "\n",
    "assert len(train_text) == len(train_labels) == len(train_attention_masks) == len(train_padding_list)\n",
    "assert len(test_text) == len(test_labels) == len(test_attention_masks) == len(test_padding_list)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "ceJlaSLAwZrg"
   },
   "source": [
    "### 2.5 Split train dataset into train_dataset and validation_dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "z9O38bJgwUAx"
   },
   "outputs": [],
   "source": [
    "from sklearn.model_selection import train_test_split\n",
    "\n",
    "# Use 90% for training and 10% for validation.\n",
    "train_padding_list, validation_padding_list, train_labels, validation_labels, train_attention_masks, validation_attention_masks = train_test_split(train_padding_list, train_labels, train_attention_masks, random_state=2020, test_size=0.1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "Q1s3xdtiyJTn"
   },
   "outputs": [],
   "source": [
    "assert len(train_labels) == len(train_attention_masks) == len(train_padding_list)\n",
    "assert len(validation_labels) == len(validation_attention_masks) == len(validation_padding_list)\n",
    "assert len(test_labels) == len(test_attention_masks) == len(test_padding_list)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 68
    },
    "colab_type": "code",
    "id": "UFloxwfxyc2Y",
    "outputId": "74072d2b-8157-4861-948b-b67cef21d50f"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "len(train_labels) = 504\n",
      "len(validation_labels) = 56\n",
      "len(test_labels) = 38\n"
     ]
    }
   ],
   "source": [
    "print(\"len(train_labels) = {}\\nlen(validation_labels) = {}\\nlen(test_labels) = {}\".format(len(train_labels), len(validation_labels), len(test_labels)))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "E92RWg2hU2ZW"
   },
   "source": [
    "### 2.6 Convert to Dataset"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### 2.6.1 Convert all the `List` objects to tensor"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "SVLLo2S_U2ZX"
   },
   "outputs": [],
   "source": [
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.optim as optim\n",
    "import torch.nn.functional as F\n",
    "\n",
    "# Convert all inputs and labels into torch tensors, the required datatype for our model.\n",
    "train_inputs = torch.tensor(train_padding_list)\n",
    "validation_inputs = torch.tensor(validation_padding_list)\n",
    "test_inputs = torch.tensor(test_padding_list)\n",
    "\n",
    "train_labels = torch.tensor(train_labels)\n",
    "validation_labels = torch.tensor(validation_labels)\n",
    "test_labels = torch.tensor(test_labels)\n",
    "\n",
    "train_masks = torch.tensor(train_attention_masks)\n",
    "validation_masks = torch.tensor(validation_attention_masks)\n",
    "test_masks = torch.tensor(test_attention_masks)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "I2BFxv_0U2ZY"
   },
   "source": [
    "#### 2.6.2 Form the Dataset with torch.tensor"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "PyNjxNPSU2ZY"
   },
   "outputs": [],
   "source": [
    "from torch.utils.data import TensorDataset, DataLoader, RandomSampler, SequentialSampler\n",
    "\n",
    "# The DataLoader needs to know our batch size for training, so we specify it \n",
    "# here.\n",
    "# For fine-tuning BERT on a specific task, the authors recommend a batch size of 16 or 32.\n",
    "\n",
    "batch_size = 16\n",
    "\n",
    "# Create the DataLoader for our training set.\n",
    "train_data = TensorDataset(train_inputs, train_masks, train_labels)\n",
    "train_sampler = RandomSampler(train_data)\n",
    "train_dataloader = DataLoader(train_data, sampler=train_sampler, batch_size=batch_size)\n",
    "\n",
    "# Create the DataLoader for our validation set.\n",
    "validation_data = TensorDataset(validation_inputs, validation_masks, validation_labels)\n",
    "validation_sampler = RandomSampler(validation_data)\n",
    "validation_dataloader = DataLoader(validation_data, sampler=validation_sampler, batch_size=batch_size)\n",
    "\n",
    "# Create the DataLoader for our test set.\n",
    "test_data = TensorDataset(test_inputs, test_masks, test_labels)\n",
    "test_sampler = SequentialSampler(test_data)\n",
    "test_dataloader = DataLoader(test_data, sampler=test_sampler, batch_size=batch_size)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "QdvMYZh9U2ZZ"
   },
   "source": [
    "## 3. Train BERT Text Classification Model"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "hISLv4phU2ZZ"
   },
   "source": [
    "### 3.1 BertForSequenceClassification"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Load `BertForSequenceClassification` from `transformers`**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 1000
    },
    "colab_type": "code",
    "id": "MyujoovlU2ZZ",
    "outputId": "f0ceed19-34bd-4b98-880f-eb7e2cc0cf74"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "BertForSequenceClassification(\n",
       "  (bert): BertModel(\n",
       "    (embeddings): BertEmbeddings(\n",
       "      (word_embeddings): Embedding(30522, 768, padding_idx=0)\n",
       "      (position_embeddings): Embedding(512, 768)\n",
       "      (token_type_embeddings): Embedding(2, 768)\n",
       "      (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "      (dropout): Dropout(p=0.1, inplace=False)\n",
       "    )\n",
       "    (encoder): BertEncoder(\n",
       "      (layer): ModuleList(\n",
       "        (0): BertLayer(\n",
       "          (attention): BertAttention(\n",
       "            (self): BertSelfAttention(\n",
       "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "            (output): BertSelfOutput(\n",
       "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "          )\n",
       "          (intermediate): BertIntermediate(\n",
       "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "          )\n",
       "          (output): BertOutput(\n",
       "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "        )\n",
       "        (1): BertLayer(\n",
       "          (attention): BertAttention(\n",
       "            (self): BertSelfAttention(\n",
       "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "            (output): BertSelfOutput(\n",
       "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "          )\n",
       "          (intermediate): BertIntermediate(\n",
       "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "          )\n",
       "          (output): BertOutput(\n",
       "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "        )\n",
       "        (2): BertLayer(\n",
       "          (attention): BertAttention(\n",
       "            (self): BertSelfAttention(\n",
       "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "            (output): BertSelfOutput(\n",
       "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "          )\n",
       "          (intermediate): BertIntermediate(\n",
       "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "          )\n",
       "          (output): BertOutput(\n",
       "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "        )\n",
       "        (3): BertLayer(\n",
       "          (attention): BertAttention(\n",
       "            (self): BertSelfAttention(\n",
       "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "            (output): BertSelfOutput(\n",
       "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "          )\n",
       "          (intermediate): BertIntermediate(\n",
       "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "          )\n",
       "          (output): BertOutput(\n",
       "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "        )\n",
       "        (4): BertLayer(\n",
       "          (attention): BertAttention(\n",
       "            (self): BertSelfAttention(\n",
       "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "            (output): BertSelfOutput(\n",
       "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "          )\n",
       "          (intermediate): BertIntermediate(\n",
       "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "          )\n",
       "          (output): BertOutput(\n",
       "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "        )\n",
       "        (5): BertLayer(\n",
       "          (attention): BertAttention(\n",
       "            (self): BertSelfAttention(\n",
       "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "            (output): BertSelfOutput(\n",
       "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "          )\n",
       "          (intermediate): BertIntermediate(\n",
       "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "          )\n",
       "          (output): BertOutput(\n",
       "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "        )\n",
       "        (6): BertLayer(\n",
       "          (attention): BertAttention(\n",
       "            (self): BertSelfAttention(\n",
       "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "            (output): BertSelfOutput(\n",
       "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "          )\n",
       "          (intermediate): BertIntermediate(\n",
       "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "          )\n",
       "          (output): BertOutput(\n",
       "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "        )\n",
       "        (7): BertLayer(\n",
       "          (attention): BertAttention(\n",
       "            (self): BertSelfAttention(\n",
       "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "            (output): BertSelfOutput(\n",
       "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "          )\n",
       "          (intermediate): BertIntermediate(\n",
       "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "          )\n",
       "          (output): BertOutput(\n",
       "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "        )\n",
       "        (8): BertLayer(\n",
       "          (attention): BertAttention(\n",
       "            (self): BertSelfAttention(\n",
       "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "            (output): BertSelfOutput(\n",
       "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "          )\n",
       "          (intermediate): BertIntermediate(\n",
       "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "          )\n",
       "          (output): BertOutput(\n",
       "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "        )\n",
       "        (9): BertLayer(\n",
       "          (attention): BertAttention(\n",
       "            (self): BertSelfAttention(\n",
       "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "            (output): BertSelfOutput(\n",
       "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "          )\n",
       "          (intermediate): BertIntermediate(\n",
       "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "          )\n",
       "          (output): BertOutput(\n",
       "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "        )\n",
       "        (10): BertLayer(\n",
       "          (attention): BertAttention(\n",
       "            (self): BertSelfAttention(\n",
       "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "            (output): BertSelfOutput(\n",
       "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "          )\n",
       "          (intermediate): BertIntermediate(\n",
       "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "          )\n",
       "          (output): BertOutput(\n",
       "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "        )\n",
       "        (11): BertLayer(\n",
       "          (attention): BertAttention(\n",
       "            (self): BertSelfAttention(\n",
       "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "            (output): BertSelfOutput(\n",
       "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "          )\n",
       "          (intermediate): BertIntermediate(\n",
       "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "          )\n",
       "          (output): BertOutput(\n",
       "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "        )\n",
       "      )\n",
       "    )\n",
       "    (pooler): BertPooler(\n",
       "      (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "      (activation): Tanh()\n",
       "    )\n",
       "  )\n",
       "  (dropout): Dropout(p=0.1, inplace=False)\n",
       "  (classifier): Linear(in_features=768, out_features=2, bias=True)\n",
       ")"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from transformers import BertForSequenceClassification, AdamW, BertConfig\n",
    "import torch\n",
    "\n",
    "# Load BertForSequenceClassification, the pretrained BERT model\n",
    "model = BertForSequenceClassification.from_pretrained(\n",
    "    \"bert-base-uncased\",  # Use the 12-layer BERT model, with an uncased vocab.\n",
    "     num_labels = 2,      # The number of output labels -- 2 for binary classification.\n",
    "                    # You can increase this for multi-class tasks.   \n",
    "     output_attentions = False, # Whether the model returns attentions weights.\n",
    "     output_hidden_states = False, # Whether the model returns all hidden-states.\n",
    ")\n",
    "\n",
    "model.to(device)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "AffsOomjU2Za"
   },
   "source": [
    "### 3.2 Optimizer & Learning Rate Scheduler"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "nLnToQI5U2Zb"
   },
   "source": [
    "Now that we have our model loaded we need to grab the training hyperparameters from within the stored model.\n",
    "\n",
    "For the purposes of fine-tuning, the authors recommend choosing from the following values:\n",
    "- Batch size: 16, 32  (We chose 32 when creating our DataLoaders).\n",
    "- Learning rate (Adam): 5e-5, 3e-5, 2e-5  (We'll use 2e-5).\n",
    "- Number of epochs: 2, 3, 4  (We'll use 4).\n",
    "\n",
    "The epsilon parameter `eps = 1e-8` is \"a very small number to prevent any division by zero in the implementation\" (from [here](https://machinelearningmastery.com/adam-optimization-algorithm-for-deep-learning/)).\n",
    "\n",
    "You can find the creation of the AdamW optimizer in `run_glue.py` [here](https://github.com/huggingface/transformers/blob/5bfcd0485ece086ebcbed2d008813037968a9e58/examples/run_glue.py#L109)."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "ypMAByuGU2Zb"
   },
   "outputs": [],
   "source": [
    "# Note: AdamW is a class from the huggingface library (as opposed to pytorch) \n",
    "# I believe the 'W' stands for 'Weight Decay fix\"\n",
    "optimizer = AdamW(model.parameters(),\n",
    "                  lr = 5e-5, # args.learning_rate - default is 5e-5, our notebook had 2e-5\n",
    "                  eps = 1e-8 # args.adam_epsilon  - default is 1e-8.\n",
    "                )\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 34
    },
    "colab_type": "code",
    "id": "nIQ1wqtXU2Zc",
    "outputId": "661fc47b-7910-4bc9-ec06-fae2a1918dbd"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "total_steps = 128\n"
     ]
    }
   ],
   "source": [
    "from transformers import get_linear_schedule_with_warmup\n",
    "\n",
    "# Number of training epochs (authors recommend between 2 and 4)\n",
    "epochs = 4\n",
    "\n",
    "# Total number of training steps is number of batches * number of epochs.\n",
    "total_steps = len(train_dataloader) * epochs\n",
    "print(\"total_steps = {}\".format(total_steps))\n",
    "\n",
    "# Create the learning rate scheduler.\n",
    "scheduler = get_linear_schedule_with_warmup(optimizer, \n",
    "                                            num_warmup_steps = 0, # Default value in run_glue.py\n",
    "                                            num_training_steps = total_steps)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "28HZBqHGU2Zd"
   },
   "source": [
    "### 3.3 Train"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "cBrQeDtBU2Zd"
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "\n",
    "# Function to calculate the accuracy of our predictions vs labels\n",
    "def flat_accuracy(preds, labels):\n",
    "    pred_flat = np.argmax(preds, axis=1).flatten()\n",
    "    labels_flat = labels.flatten()\n",
    "    return np.sum(pred_flat == labels_flat) / len(labels_flat)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "wxyEyFmBU2Zd"
   },
   "outputs": [],
   "source": [
    "import time\n",
    "import datetime\n",
    "\n",
    "def format_time(elapsed):\n",
    "    '''\n",
    "    Takes a time in seconds and returns a string hh:mm:ss\n",
    "    '''\n",
    "    # Round to the nearest second.\n",
    "    elapsed_rounded = int(round((elapsed)))\n",
    "    \n",
    "    # Format as hh:mm:ss\n",
    "    return str(datetime.timedelta(seconds=elapsed_rounded))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 935
    },
    "colab_type": "code",
    "id": "n4OdcNVeU2Ze",
    "outputId": "5092536d-f92d-4aab-b8c0-e4e24b142801"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "======== Epoch 1 / 4 ========\n",
      "Training...\n",
      "  Batch    10  of     32.    Elapsed: 0:00:46.\n",
      "  Batch    20  of     32.    Elapsed: 0:01:38.\n",
      "  Batch    30  of     32.    Elapsed: 0:02:26.\n",
      "\n",
      "  Average training loss: 0.60\n",
      "  Training epcoh took: 0:02:34\n",
      "\n",
      "Running Validation...\n",
      "  Accuracy: 0.86\n",
      "  Validation took: 0:00:03\n",
      "\n",
      "======== Epoch 2 / 4 ========\n",
      "Training...\n",
      "  Batch    10  of     32.    Elapsed: 0:00:46.\n",
      "  Batch    20  of     32.    Elapsed: 0:01:31.\n",
      "  Batch    30  of     32.    Elapsed: 0:02:15.\n",
      "\n",
      "  Average training loss: 0.31\n",
      "  Training epcoh took: 0:02:22\n",
      "\n",
      "Running Validation...\n",
      "  Accuracy: 0.86\n",
      "  Validation took: 0:00:03\n",
      "\n",
      "======== Epoch 3 / 4 ========\n",
      "Training...\n",
      "  Batch    10  of     32.    Elapsed: 0:00:45.\n",
      "  Batch    20  of     32.    Elapsed: 0:01:29.\n",
      "  Batch    30  of     32.    Elapsed: 0:02:12.\n",
      "\n",
      "  Average training loss: 0.12\n",
      "  Training epcoh took: 0:02:20\n",
      "\n",
      "Running Validation...\n",
      "  Accuracy: 0.86\n",
      "  Validation took: 0:00:03\n",
      "\n",
      "======== Epoch 4 / 4 ========\n",
      "Training...\n",
      "  Batch    10  of     32.    Elapsed: 0:00:43.\n",
      "  Batch    20  of     32.    Elapsed: 0:01:26.\n",
      "  Batch    30  of     32.    Elapsed: 0:02:08.\n",
      "\n",
      "  Average training loss: 0.03\n",
      "  Training epcoh took: 0:02:15\n",
      "\n",
      "Running Validation...\n",
      "  Accuracy: 0.84\n",
      "  Validation took: 0:00:03\n",
      "\n",
      "Training complete!\n"
     ]
    }
   ],
   "source": [
    "import random\n",
    "\n",
    "# Set the seed value all over the place to make this reproducible.\n",
    "seed_val = 12345\n",
    "\n",
    "random.seed(seed_val)\n",
    "np.random.seed(seed_val)\n",
    "torch.manual_seed(seed_val)\n",
    "\n",
    "# Store the average loss after each epoch so we can plot them.\n",
    "loss_values = []\n",
    "\n",
    "# For each epoch...\n",
    "for epoch_i in range(epochs):\n",
    "    \n",
    "    ##########################################\n",
    "    #               Training                 #\n",
    "    ##########################################\n",
    "    \n",
    "    # Perform one full pass over the training set.\n",
    "\n",
    "    print(\"\")\n",
    "    print('======== Epoch {:} / {:} ========'.format(epoch_i + 1, epochs))\n",
    "    print('Training...')\n",
    "\n",
    "    # Measure how long the training epoch takes.\n",
    "    t0 = time.time()\n",
    "\n",
    "    # Reset the total loss for this epoch.\n",
    "    total_loss = 0\n",
    "\n",
    "    # Put the model into training mode. Don't be mislead--the call to \n",
    "    # `train` just changes the *mode*, it doesn't *perform* the training.\n",
    "    # `dropout` and `batchnorm` layers behave differently during training\n",
    "    # vs. test (source: https://stackoverflow.com/questions/51433378/what-does-model-train-do-in-pytorch)\n",
    "    model.train()\n",
    "\n",
    "    # For each batch of training data...\n",
    "    for step, batch in enumerate(train_dataloader):\n",
    "\n",
    "        # Progress update every 10 batches.\n",
    "        if step % 10 == 0 and not step == 0:\n",
    "            # Calculate elapsed time in minutes.\n",
    "            elapsed = format_time(time.time() - t0)\n",
    "            \n",
    "            # Report progress.\n",
    "            print('  Batch {:>5,}  of  {:>5,}.    Elapsed: {:}.'.format(step, len(train_dataloader), elapsed))\n",
    "\n",
    "        # Unpack this training batch from our dataloader. \n",
    "        # `batch` contains three pytorch tensors:\n",
    "        #   [0]: input ids \n",
    "        #   [1]: attention masks\n",
    "        #   [2]: labels \n",
    "        b_input_ids = batch[0].to(device)\n",
    "        b_input_mask = batch[1].to(device)\n",
    "        b_labels = batch[2].to(device)\n",
    "\n",
    "        # Clear the gradients.\n",
    "        model.zero_grad()        \n",
    "\n",
    "        # Perform a forward pass (evaluate the model on this training batch).\n",
    "        # This will return the loss (rather than the model output) because we have provided the `labels`.\n",
    "        outputs = model(b_input_ids, \n",
    "                        token_type_ids=None, \n",
    "                        attention_mask=b_input_mask, \n",
    "                        labels=b_labels)\n",
    "        \n",
    "        # The call to `model` always returns a tuple, so we need to pull the \n",
    "        # loss value out of the tuple.\n",
    "        loss = outputs[0]\n",
    "\n",
    "        # Accumulate the training loss over all of the batches so that we can calculate the average loss at the end. \n",
    "        # `loss` is a Tensor containing a single value; the `.item()` function just returns the Python value from the tensor.\n",
    "        total_loss += loss.item()\n",
    "\n",
    "        # Perform a `backward` pass to calculate the gradients.\n",
    "        loss.backward()\n",
    "\n",
    "        # Clip the norm of the gradients to 1.0.\n",
    "        # This is to help prevent the \"exploding gradients\" problem.\n",
    "        torch.nn.utils.clip_grad_norm_(model.parameters(), 1.0)\n",
    "\n",
    "        # Update parameters and take a step using the computed gradient.\n",
    "        # The optimizer dictates the \"update rule\"--how the parameters are modified based on their gradients, the learning rate, etc.\n",
    "        optimizer.step()\n",
    "\n",
    "        # Update the learning rate.\n",
    "        scheduler.step()\n",
    "\n",
    "    # Calculate the average loss over the training data.\n",
    "    avg_train_loss = total_loss / len(train_dataloader)            \n",
    "    \n",
    "    # Store the loss value for plotting the learning curve.\n",
    "    loss_values.append(avg_train_loss)\n",
    "\n",
    "    print(\"\")\n",
    "    print(\"  Average training loss: {0:.2f}\".format(avg_train_loss))\n",
    "    print(\"  Training epcoh took: {:}\".format(format_time(time.time() - t0)))\n",
    "        \n",
    "        \n",
    "    ##########################################\n",
    "    #               Validation               #\n",
    "    ##########################################\n",
    "    # After the completion of each training epoch, measure our performance on our validation set.\n",
    "\n",
    "    print(\"\")\n",
    "    print(\"Running Validation...\")\n",
    "\n",
    "    t0 = time.time()\n",
    "\n",
    "    # Put the model in evaluation mode--the dropout layers behave differently during evaluation.\n",
    "    model.eval()\n",
    "\n",
    "    # Tracking variables \n",
    "    eval_loss, eval_accuracy = 0, 0\n",
    "    nb_eval_steps, nb_eval_examples = 0, 0\n",
    "\n",
    "    # Evaluate data for one epoch\n",
    "    for batch in validation_dataloader:\n",
    "        \n",
    "        # Add batch to device\n",
    "        batch = tuple(t.to(device) for t in batch)\n",
    "        \n",
    "        # Unpack the inputs from our dataloader\n",
    "        b_input_ids, b_input_mask, b_labels = batch\n",
    "        \n",
    "        # Telling the model not to compute or store gradients, saving memory and speeding up validation\n",
    "        with torch.no_grad():        \n",
    "\n",
    "            # Forward pass, calculate logit predictions.\n",
    "            # This will return the logits rather than the loss because we have not provided labels.\n",
    "            # token_type_ids is the same as the \"segment ids\", which differentiates sentence 1 and 2 in 2-sentence tasks.\n",
    "            # The documentation for this `model` function is here: \n",
    "            # https://huggingface.co/transformers/v2.2.0/model_doc/bert.html#transformers.BertForSequenceClassification\n",
    "            outputs = model(b_input_ids, \n",
    "                            token_type_ids=None, \n",
    "                            attention_mask=b_input_mask)\n",
    "        \n",
    "        # Get the \"logits\" output by the model. The \"logits\" are the output\n",
    "        # values prior to applying an activation function like the softmax.\n",
    "        logits = outputs[0]\n",
    "\n",
    "        # Move logits and labels to CPU\n",
    "        logits = logits.detach().cpu().numpy()\n",
    "        label_ids = b_labels.to('cpu').numpy()\n",
    "        \n",
    "        # Calculate the accuracy for this batch of test sentences.\n",
    "        # flat_accuracy(y_pred, y_true)\n",
    "        tmp_eval_accuracy = flat_accuracy(logits, label_ids)\n",
    "        \n",
    "        # Accumulate the total accuracy.\n",
    "        eval_accuracy += tmp_eval_accuracy\n",
    "\n",
    "        # Track the number of batches\n",
    "        nb_eval_steps += 1\n",
    "\n",
    "    # Report the final accuracy for this validation run.\n",
    "    print(\"  Accuracy: {0:.2f}\".format(eval_accuracy/nb_eval_steps))\n",
    "    print(\"  Validation took: {:}\".format(format_time(time.time() - t0)))\n",
    "\n",
    "print(\"\")\n",
    "print(\"Training complete!\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 3.4 Plot"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Plot the average loss in training**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAuIAAAGXCAYAAAD25DXQAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAAgAElEQVR4nOzdeVTV9b7/8efejMo8yrzBCQdEERFIyaFShkyyzMrUzNPobTj3lmbXc9KO93QardNgp+7JnFMbHEFNc0pDcTZNTUE24jwDjgi/P/zFjZxCge8GXo+1XCu+e3/3fu3eC3zx9bM/21RWVlaGiIiIiIjUKLPRAURERERE6iMVcRERERERA6iIi4iIiIgYQEVcRERERMQAKuIiIiIiIgZQERcRERERMYCKuIhINXr55ZeJjIy84Z+XX365Sp5vwIABdO/e/aZz1qQPPviAyMhI9u3bV6PPKyJiK+yNDiAiUpf169ePxMTE8q/Xr1/P9OnT6devH7GxseXHw8LCquT5nnrqKc6ePXvLOUVEpPqpiIuIVKOYmBhiYmLKv7506RLTp0+nXbt29O7du8qfr1OnTjd13u9ziohI9dPSFBERERERA6iIi4jYkO7duzNy5EheeeUVoqOjuf322zl+/DhlZWVMmzaN+++/n5iYGNq0aUNycjKffvopZWVl5ef/fo34gAEDGDJkCCtWrKBPnz60adOGLl268MEHH1BaWlp+v9+vEX/55ZdJTk5my5YtPPLII7Rt25bbbruNMWPGcO7cuQqZc3JyePrpp+nQoQPx8fGMGTOGGTNm3NT67xMnTjBq1CiSkpKIioqiZ8+efPrpp1y6dKnC/aZNm0avXr1o27Yt8fHxDB06lF9++aXCfRYuXMh9991HTEwMsbGxDB48mPXr11cqj4hIddLSFBERGzN//nwaN27MK6+8wtGjR/H29mbs2LF88skn3HvvvTzwwAMUFxcza9Ys3nnnHVxcXOjfv/81H2/Xrl288MIL9OvXj379+jFv3jw+/PBDvL29r3ve8ePHGTJkCCkpKdxzzz2sWLGCSZMm4ejoyLBhwwDYv38/Dz/8MACPPfYY9vb2TJkyhblz51b6dZ86dYoHH3yQgoICHnzwQSIiIli1ahXvvPMO27dv57333gNgzpw5jBo1ivT0dAYMGMDx48eZMGECAwYM4LvvvsPNzY21a9fy5z//mdtvv52+ffty9uxZJk+ezODBg5k/fz6hoaGVziciUtVUxEVEbMy5c+f4+OOPadSoEQAXL15k8uTJpKWl8Y9//KP8fn379iUxMZGVK1det1AfPnyYcePGlV8pT09PJykpiblz5173vFOnTjFy5EgGDBgAwAMPPEBqaipz584tL+IffvghhYWFzJkzhyZNmgDQu3dvkpOTK/26P/vsM/bu3ctHH33EnXfeCUD//v0ZPXo0U6dO5d5776VLly7MnTuXZs2a8cYbb5Sf27JlS95880127dpFbGwsGRkZODs7M27cOEwmEwC33XYbzz33HNu2bVMRFxGboKUpIiI2JiwsrLyEAzg4OLB69Wpee+21Cvc7ceIErq6unDlz5rqP16BBA7p27Vr+tZOTExERERw9evSGWVJSUip83aJFi/LzysrKWLJkCUlJSeUlHKBRo0bcc889N3zs3/v+++9p0qRJeQn/1TPPPAPAkiVLAAgICCAnJ4cPP/ywfOlLly5dmD9/fvlONAEBARQXFzNmzBj27NkDQGRkJAsXLrypXxJERKqDroiLiNgYHx+fK445ODiwbNkylixZQm5uLnl5eZw6dQqgwhrxq/H09MRsrnjdxdHRscIa8Wvx9va+5nknT57k5MmThIeHX3Fe48aNb/jYv7dv3z6SkpKuOO7n54e7uzsFBQUADB06lE2bNvHBBx/wwQcf0LRpU7p3707fvn3Lt4F85JFH+OGHH5g8eTKTJ08mJCSEbt26cf/999OiRYtKZxMRqQ66Ii4iYmPs7OwqfF1WVsYzzzzDc889x759+4iJiWHYsGEsWrSIwMDAGz7e70t4ZVzv3JKSEuByOf89JyenSj/X9X6hKC0txcHBAbh8tXv27Nl88cUXDBgwgJKSEj799FNSU1NZu3YtAK6urkyePJnp06fz+OOP4+LiwqRJk7j33ntvav26iEh10BVxEREbt27dOpYuXcozzzzD888/X368pKSEkydPGrbe2cfHh4YNG7J3794rbsvLy6v04wUHB5Obm3vF8SNHjlBUVFT+S8fOnTsBSExMLP8QovXr1zNo0CAmTZpEx44dyc3NpbCwkHbt2tGuXTtefPFFdu/eTf/+/Rk/fjy9evWqdD4RkaqmK+IiIjbu5MmTADRt2rTC8RkzZnD27NnyK9M1zWw20717d1asWEF+fn758VOnTjFv3rxKP163bt3Ys2cPixcvrnD8008/BShf5/78888zbNiwClsatmrVCgcHh/Ir+GPGjOGZZ56huLi4/D6NGzfG3d39lv6FQESkKumKuIiIjYuJicHV1ZXXX3+dgoICPDw8WLNmDRkZGTg5OVUomzXt+eefZ/ny5fTr148BAwbg6OjIl19+Wb5+/dcdS/6IJ598kkWLFvHCCy/w0EMPER4eTlZWFosWLaJHjx506dIFgCFDhjBy5EgeffRRkpOTKSsrY/bs2Zw/f758K8XBgwfz+OOP079/f9LT03FycmLx4sVYrdYKu62IiBhJRVxExMb5+vry6aef8vbbbzNu3DgcHR2JiIjg3XffZcuWLUycOJGjR4/i6+tb49nCwsKYPHkyb7zxBv/6179wcnIiPT0dOzs7/v3vf191/fi1eHp6Mn36dN577z0yMjI4ffo0oaGhDBs2jEcffbT8fn379sXBwYGJEyfy7rvvUlpaSlRUFJ999hnx8fEAdO7cmXHjxvGvf/2Ljz/+mPPnz9OsWTPeffdd0tLSqvp/g4jITTGV3ejt9iIiItdw7NgxvL29r7jy/be//Y1p06axefPm8jdZiohIRVooJyIiN+35558nLS2twlaIZ8+eZenSpbRo0UIlXETkOrQ0RUREblrv3r0ZOXIkTzzxBHfccQfnz59nzpw5HDx4kNGjRxsdT0TEpmlpioiI3JI5c+YwceJEcnJyMJvNREVF8cwzz9CxY0ejo4mI2DQVcRERERERA2iNuIiIiIiIAVTERUREREQMUK/frHniRDGlpTW7MsfHx5Vjx4pq9Dnl+jQT26S52B7NxDZpLrZHM7FNRszFbDbh5eVyzdvrdREvLS2r8SL+6/OKbdFMbJPmYns0E9ukudgezcQ22dpctDRFRERERMQAKuIiIiIiIgZQERcRERERMYCKuIiIiIiIAVTERUREREQMoCIuIiIiImIAFXEREREREQOoiIuIiIiIGMDwIj5v3jzS0tKIjo4mJSWFWbNmXff+paWljBs3jjvuuIPo6Gh69erF/PnzayitiIiIiEjVMPSTNTMyMnjxxRcZNGgQnTt3ZvHixQwfPhxnZ2eSk5Oves7f//53pk+fzn/+53/SokUL5s+fz3/913/h6upKly5davgV/HE/bjvIN8v3cPz0ebzdnejTpQmJrQOMjiUiIiIiBjG0iI8dO5aUlBRGjBgBQFJSEqdOneL999+/ahG3Wq1MmTKF1157jb59+wKQmJjI3r17Wblypc0W8R+3HWRC5g4ulJQCcOz0eSZk7gBQGRcRERGppwxbmpKfn4/VaqVHjx4Vjvfs2ZOcnBzy8/OvOGfx4sU4OzuTnp5e4fjkyZMZOXJktea9Fd8s31Newn91oaSUb5bvMSiRiIiIiBjNsCKek5MDQERERIXjFosFgNzc3CvO2blzJxEREaxevZp77rmHVq1a0aNHDzIyMqo/8C04dvp8pY6LiIiISN1n2NKUwsJCAFxdXSscd3FxAaCoqOiKc44fP86BAwd45ZVXeP755wkJCWHmzJn8+c9/xtvbm4SEhEpl8PFxvfGdqoCfVwOOnDh7xXF3F0f8/NxqJINcn+ZgmzQX26OZ2CbNxfZoJrbJ1uZiWBEvKyu77u1m85UX6y9evMjx48f55JNP6NatGwAJCQnk5OTw4YcfVrqIHztWRGnp9XNUhfTOERXWiAOYTHC6+ALvTl7Hg3c0xcHertpzyNX5+blx5Eih0THkdzQX26OZ2CbNxfZoJrbJiLmYzabrXvg1bGmKm9vl30iKi4srHP/1Svivt/+Wi4sLdnZ2dOrUqfyY2WzmtttuY+fOndWY9tYktg5gUEoLfNydMAE+7k48ltqS5Pgwlm4s4H8mrefwiTNGxxQRERGRGmTYFfFf14ZbrVYiIyPLj+fl5VW4/bcsFgulpaWUlJTg6OhYfvzixYuYTKZqTnxrElsHkNg64IrfxpqHePLv+dsZ/UU2g1Na0qGFv4EpRURERKSmGHZF3GKxEBISwoIFCyocX7RoEeHh4QQFBV1xTlJSEmVlZWRmZpYfKykpYeXKlcTGxlZ75urQrpkvrz4aR4C3Cx/P+ompi3dRcqn0xieKiIiISK1m6D7iQ4cOZcSIEXh4eNC1a1eWLFlCZmYmY8eOBS6/OdNqtdK0aVNcXV1JTEykS5cujBkzhjNnzhAeHs7UqVMpKCjgnXfeMfKl3BJfzwaMeKQ9M5buZvG6fewpOM3TvVvj69nA6GgiIiIiUk3sRo0aNcqoJ2/ZsiV+fn7Mnj2b6dOnU1RUxMsvv0yvXr0AyMzM5KmnnqJz586EhIQAcNddd1FcXMy0adP45ptvcHNz4x//+AcxMTGVfv6zZy9wg/eMVjkXFyfOnLlwxXGz2USbxj4E+7rww9b9LN+0n0DfhgT6uNRswHroWjMRY2kutkczsU2ai+3RTGyTEXMxmUw0bOh47dvLbrR9SR1WU7um/NYfecfu4RNnGDdrG3mHCknuGEafLo2xtzNsFVGdp3e32ybNxfZoJrZJc7E9molt0q4p8of4ezXklQHt6dY+mAVrrbw5dSPHT58zOpaIiIiIVCEVcRvlYG/HgB6RPNW7NflHihg1Ppste44ZHUtEREREqoiKuI3r2LIRrz4ah6erE+/N3MzXy/dwqVS7qoiIiIjUdiritUCAd0NGDozl9raBzP8xj7embeJE4XmjY4mIiIjILVARryUcHex4NKUlj9/dir0HTzNq/Fq27T1udCwRERERuUkq4rVMYlQAfxkUh1tDR979chOzVubU+M4vIiIiInLrVMRroWBfF/4ysAO3RQUwZ9Ve3pm+iVPF2q9UREREpDZREa+lnBztGHJ3KwantmB3wSlGfb6WHXknjI4lIiIiIn+QingtlxQdxF8GdqCBkz1vfbmReav3Ulp/P6NJREREpNZQEa8DQvxd+cugDnRs2YhvVuTw3szNFOqjdUVERERsmop4HdHAyZ4nerViYM9IduSdZNT4bH7Zd9LoWCIiIiJyDSridYjJZKJrTDD/PSAWBzszb0zZSOaaPC1VEREREbFBKuJ1kCXAjb8+Gkf75r7MXLqHD77aQtHZi0bHEhEREZHfUBGvoxo62/N0ehQP39mMn3KPM3r8WvbsP2V0LBERERH5/1TE6zCTycSdHUJ5ZUAsJpOJf0zewKLsfMq0VEVERETEcCri9UBEoDuvDo4juokPXy75hY++/Ykz57RURURERMRIKuL1hIuzA//Rpw39ujdl8+6jjBqfzd6Dp42OJSIiIlJvqYjXIyaTiZ4dwxjevz2lZWX8fdJ6lqzfp6UqIiIiIgZQEa+HmgZ7MGpwR1qFezPlu118MnsbZ8+XGB1LREREpF5REa+nXBs48Nz90dzftQnrdx7htS+ysR4qNDqWiIiISL2hIl6PmU0mUhMsDHs4hvMXL/E/k9azfFOBlqqIiIiI1AAVcaF5qCejBnekeYgHExbs5H/nbefcBS1VEREREalOKuICgLuLI39+oB3pSRFkbTvE3yaso+BIkdGxREREROosFXEpZzabuKdTBC8+2I7icyX8bcI6Vm09YHQsERERkTpJRVyu0DLcm9GD42gc5M6/5//M5/N/5vzFS0bHEhEREalTVMTlqjxcnXjxwRh63RbOqq0HGDNxHQeOFRsdS0RERKTOUBGXazKbTdx7e2P+3K8tp4ou8NoX68jadtDoWCIiIiJ1goq43FBUhA+jH+tIWCNXPp27nQkLdnCxREtVRERERG6Firj8IV5uTgx7OIaUhDCWb9rP/0xcz6ETZ4yOJSIiIlJrqYjLH2ZnNtO3a1Oevz+aY6fPMXp8Ntk7DhsdS0RERKRWUhGXSmvb1JdRgzsS7OvCuFk/MWXRLi6WlBodS0RERKRWURGXm+Lj4czw/u3pERfKkg37eH3yeo6cPGt0LBEREZFaQ0Vcbpq9nZkH72jGf/Rpw6ETZxk9PpuNu44YHUtERESkVlARl1vWvrkfowbH4e/VgA++2cqXS36h5JKWqoiIiIhcj4q4VAk/zwaMeCSWO9qHsCg7nzembODYqXNGxxIRERGxWSriUmUc7M3079Gcp9OjKDhazKjxa9m8+6jRsURERERskoq4VLm4Fv68+mgcPu7OvP/VFmYu282lUi1VEREREfktFXGpFo28G/LfA2Pp2i6IzCwrb07dyInC80bHEhEREbEZhhfxefPmkZaWRnR0NCkpKcyaNeu69589ezaRkZFX/HnttddqKLH8UQ72dgxMbsETvVphPVTEq5+v5afcY0bHEhEREbEJ9kY+eUZGBi+++CKDBg2ic+fOLF68mOHDh+Ps7ExycvJVz9mxYwcWi4U333yzwnFfX9+aiCw3IaF1AJYANz6e9RNjp28m7bZw0jtHYDabjI4mIiIiYhhDi/jYsWNJSUlhxIgRACQlJXHq1Cnef//9axbxnTt30rp1a9q1a1eTUeUWBfq4MHJgB6Ys2sW81XvZve8kT97TGg9XJ6OjiYiIiBjCsKUp+fn5WK1WevToUeF4z549ycnJIT8//6rn7dixg8jIyJqIKFXMycGOx9JaMiStJTn7T/Pq+Gx+3nvc6FgiIiIihjCsiOfk5AAQERFR4bjFYgEgNzf3inMOHz7MsWPH2L59O8nJybRu3ZqePXvecF252JZObQIZOagDLs72vD19E3NW5VJaWmZ0LBEREZEaZdjSlMLCQgBcXV0rHHdxcQGgqKjoinN27NgBwL59+3jppZdwcnJi1qxZDB8+nEuXLnHfffdVc2qpKiF+rvxlUAcmLdzJrJW5/JJ/ksd7tcbdxdHoaCIiIiI1wrAiXlZ2/SugZvOVF+ujoqL45JNPiIuLKy/wnTt35tixY7z//vuVLuI+Pq43vlM18PNzM+R5bdGIwfEsWmPlX99u4bUJ6xg2oAOtG/vUeA7NxDZpLrZHM7FNmovt0Uxsk63NxbAi7uZ2+X9EcXFxheO/Xgn/9fbf8vb2plu3blcc79KlC6tXr+b48eN4e3v/4QzHjhXV+JIIPz83jhwprNHntHXtm3jz3wNiGTfrJ175eBV9ujQmOT4Ms6lmdlXRTGyT5mJ7NBPbpLnYHs3ENhkxF7PZdN0Lv4atEf91bbjVaq1wPC8vr8Ltv7Vx40Zmzpx5xfHz589jb29/1fIutUNYIzf++mgcsZF+fLVsD//8agtFZy8aHUtERESk2hhWxC0WCyEhISxYsKDC8UWLFhEeHk5QUNAV52zatImRI0eWrxUHKC0tZeHChbRv3x4HB4dqzy3Vp4GTPU/1bs0jPZqzfe9xRo1fy+6CU0bHEhEREakWdqNGjRpl1JO7ubkxbtw4Tpw4gclk4vPPP2fWrFm8+uqrNGvWjOPHj7Nz505cXV1xdHSkcePGzJ8/nwULFuDt7U1BQQGvv/46mzdv5p133iEgIKBSz3/27AVusFS9yrm4OHHmzIWafdJaxGQyERHoTpsmPqzbcYTF6/bh5GBHkyB3TNW0VEUzsU2ai+3RTGyT5mJ7NBPbZMRcTCYTDRteeyMKQz/ivk+fPowePZoffviBoUOHkp2dzRtvvEFqaioAy5Yto1+/fmzbtg0ADw8PJk+eTHR0NK+//jovvPACZ86c4YsvvqBt27ZGvhSpYuEB7owaHEfbpr5M/343H36zleJzWqoiIiIidYep7Ebbl9RherOm7SsrK+O7dfuYuXQ3Xm5OPJ0eRUSge5U+h2ZimzQX26OZ2CbNxfZoJrZJb9YUqSSTyUSPuFBefqQ9ZWVl/H3Sehavy7/h9pciIiIitk5FXGqFJkEevDq4I1ER3kxd/AvjZv3EmXMlRscSERERuWkq4lJruDZw4Nn7o+nbrQkbdh3ltS+yyTuof/oTERGR2klFXGoVs8lESryF4f1juHiplP+ZtJ5lGwu0VEVERERqHRVxqZWahXjy6uA4WoR5MnHhTj6bu51zF7RURURERGoPFXGptdwbOvLCA23pc3tj1vx8iNe+WMe+w0VGxxIRERH5Q1TEpVYzm0zcfVs4Lz0Yw9nzJYyZuI6VW/YbHUtERETkhlTEpU5oYfFi1GMdaRLswfiMHfx73nbOX7hkdCwRERGRa1IRlzrDw8WR/+rXjns6hbP6p4P8beI69h8tNjqWiIiIyFWpiEudYjabSE9qzH/2a0fhmQu8NiGbH386aHQsERERkSuoiEud1DrCm1GDOxIe4M5n87bzRebPXLiopSoiIiJiO1TEpc7ycnPipYfakZZoYcXmA4yZuJ6Dx88YHUtEREQEUBGXOs7ObOa+Lk14oW9bThadZ/QX2az9+ZDRsURERERUxKV+iG7iw6jBcYT6ufLJ7G1MWriTiyVaqiIiIiLGURGXesPb3ZlhD8eQ3DGMpRsL+PukDRw+oaUqIiIiYgwVcalX7O3MPNC9Kc/dF83RU2cZ/UU2q/UBQCIiImIAFXGpl9o18+XVR+MI8Hbh9QnZTF28i5JLpUbHEhERkXpERVzqLV/PBox4pD33JDVm8bp9vD55A0dPnTU6loiIiNQTKuJSr9nbmXk8vQ3PpEdx8Hgxo8dns+mXo0bHEhERkXpARVwE6NDCn1cfjcPXowH//HoLM5bu1lIVERERqVYq4iL/n79XQ14Z0J5uMcEsWGPlzakbOX76nNGxREREpI5SERf5DQd7Owb0jOTJe1qTf6SIUeOz2ZpzzOhYIiIiUgepiItcRXyrRvx1UAc8XR0ZO2MzXy/fw6VSLVURERGRqqMiLnINgT4ujBzYgdvbBjL/xzzenraJE4XnjY4lIiIidYSKuMh1ODrY8WhKS/50d0tyD55m9Pi1bN973OhYIiIiUgeoiIv8AbdFBfKXQXG4NnTknS83MWtlDqWlZUbHEhERkVpMRVzkDwr2deEvAzuQGBXAnFV7eWf6Jk4VXzA6loiIiNRSKuIileDkaMeQtJYMTmnB7oJTjBq/lp3WE0bHEhERkVpIRVykkkwmE0ltgxg5sAPOjva8OW0j81bvpbRMS1VERETkj1MRF7lJof6u/HVQBzq2bMQ3K3J4b+ZmCs9oqYqIiIj8MSriIreggZM9T/RqxcCekezIO8mo8dn8su+k0bFERESkFlARF7lFJpOJrjHB/PeAWBzszLwxZSML1lgp01IVERERuQ4VcZEqYglw46+PxtG+uS8zlu7mg6+3UnT2otGxRERExEapiItUoYbO9jydHsXDdzZja84xRo/PZs/+U0bHEhERERukIi5SxUwmE3d2CGXEI7EA/GPyBr7LztdSFREREalARVykmjQOcmfUY3G0aezDtCW/8PG3P3HmnJaqiIiIyGUq4iLVyMXZgWfva8MD3ZqyafdRRn+RTd7BQqNjiYiIiA1QERepZiaTieT4MIb3b0/JpTL+Z9I6vt+wT0tVRERE6jkVcZEa0jTYg1GD42hp8Wbyol38a842zp4vMTqWiIiIGMTwIj5v3jzS0tKIjo4mJSWFWbNm/eFzDxw4QGxsLB9//HE1JhSpOm4NHXm+bzT3dWnMuh1HeG3COvIPFxkdS0RERAxgaBHPyMjgxRdfpHPnznz00Ud07NiR4cOHs2DBghueW1ZWxiuvvEJRkUqM1C5mk4m0xHBeeqgd5y6UMGbiOlZs3q+lKiIiIvWMvZFPPnbsWFJSUhgxYgQASUlJnDp1ivfff5/k5OTrnjt16lRycnJqIqZItYgM82L04I58NncbX2TuYKf1JAN7RuLkaGd0NBEREakBhl0Rz8/Px2q10qNHjwrHe/bsSU5ODvn5+dc99+233+Zvf/tbdccUqVbuLo78+YF2pCdFkLXtIK9NyKbgiP6VR0REpD4wrIj/ejU7IiKiwnGLxQJAbm7uVc8rLS3l5ZdfJiUlhdtvv716Q4rUALPZxD2dInjxwXYUnyvhbxPXsWrrAaNjiYiISDUzrIgXFl7eS9nV1bXCcRcXF4Brrv2eMGEC+/btK1/OIlJXtAz3ZvTgOBoHuvPv+T/zecbPnL94yehYIiIiUk0MWyN+ozemmc1X/o6wZ88e3nvvPf75z3/i5uZ2yxl8fFxvfKdq4Od369mlatnKTPz83PjHs7czbdEOZizeRf7hIl4eFEeIv23kq2m2Mhf5P5qJbdJcbI9mYptsbS6GFfFfi3RxcXGF479eCf990b506RIjRowgOTmZTp06UVLyf/svl5aWUlJSgr195V7OsWNFlJbW7E4Vfn5uHDmiT1a0JbY4k56xIQR7NeDTudt54d3lDEqOJKF1gNGxapQtzqW+00xsk+ZiezQT22TEXMxm03Uv/Bq2NOXXteFWq7XC8by8vAq3/+rAgQNs3ryZWbNm0bp16/I/AB988EH5f4vUFVGNfRg1OI6wRq58Onc7Exfs4GKJlqqIiIjUFYZdEbdYLISEhLBgwQLuuuuu8uOLFi0iPDycoKCgCvf39/fnq6++uuJx7r//fh566CHuu+++as8sUtO83Z156aEYvl2RQ+YaKzn7T/P0vVE08mpodDQRERG5RYbuIz506FBGjBiBh4cHXbt2ZcmSJWRmZjJ27FgAjh8/jtVqpWnTpri6utKmTZurPo6/v/81bxOp7eztzPTt1pRmoZ78e952Ro/P5rHUlnRo4W90NBEREbkFhn6yZp8+fRg9ejQ//PADQ4cOJTs7mzfeeIPU1FQAli1bRr9+/di2bZuRMUVsQrumvowa3JFgX2G2RCMAACAASURBVBc+nvUTU77bxcWSUqNjiYiIyE0yldXjz9XWmzUFat9MSi6V8tWyPSzKzici0I2nekfh59nA6FhVrrbNpT7QTGyT5mJ7NBPbpDdrisgts7cz8+AdzfiPPm04ePwso8dns3HXEaNjiYiISCWpiIvUUu2b+/Hq4Dj8vBrwwTdb+XLJL5Rc0lIVERGR2kJFXKQW8/dswCuPxHJH+xAWZefzxtQNHD99zuhYIiIi8geoiIvUcg72Zvr3aM7T6VEUHCnm1c/XsmXPUaNjiYiIyA2oiIvUEXEt/Hn10Th83J15b+YWvlq2h0ulWqoiIiJiq1TEReqQRt4N+e+BsXRtF0RGVh5vTd3IicLzRscSERGRq1ARF6ljHOztGJjcgid6tSLvUBGjxq/lp9xjRscSERGR31ERF6mjEloH8NdHO+De0JGx0zfz7YqcGt83X0RERK5NRVykDgv0cWHkoA50ahPI3NV7efvLjZwq0lIVERERW6AiLlLHOTnY8VhaSx5LbUnO/tO8Oj6bn/NOGB1LRESk3lMRF6knOkcHMnJQB1yc7Xn7y43MXZVLaZmWqoiIiBhFRVykHgnxc+UvgzoQ36oR367MZeyMzZw+c8HoWCIiIvWSirhIPePsaM/jd7diUHIkO60nGT0+m135J42OJSIiUu+oiIvUQyaTiS7tghk5MBYHezNvTt1IRlaelqqIiIjUIBVxkXosrJEbrz4aR2ykH18t28M/v9pC0dmLRscSERGpF1TEReq5Bk72PNW7NY/0aM72vccZNX4tewpOGR1LRESkzlMRFxFMJhPd24fwyoBYzCYT/5iygUVrrZRpqYqIiEi1UREXkXLhAe6MGhxH26a+fPn9bj78ZivF57RURUREpDqoiItIBQ2dHRh6bxQP3tGMLXuOMXp8NrkHThsdS0REpM5REReRK5hMJnrEhfJy//aUlZXx+uT1LFm/T0tVREREqtBNFfGysjLy8/PLv87NzeWNN97gnXfeITc3t8rCiYixmgR78OrgjrQO92bKd7sYN3sbZ86VGB1LRESkTrCv7AkHDx5kyJAhODo68u2333L06FEeeOABCgsLAZg8eTJTpkyhVatWVR5WRGqeawMHnr0/moVrrHy9PAfroUKeSY8irJGb0dFERERqtUpfEX/33Xc5cOAADz30EAAzZsygsLCQ9957jyVLlhAYGMg///nPKg8qIsYxm0ykJFgY9nAMF0tKGTNxPcs2FWipioiIyC2odBFftWoVgwYN4oEHHgDg+++/JzAwkOTkZIKDg3nggQfYsGFDlQcVEeM1D/Xk1cFxtAjzZOKCnXw2dzvnLmipioiIyM2odBEvLCwkJCQEgGPHjrFt2zaSkpLKb2/QoAElJfqLWaSucm/oyAsPtOXe2xuz5udD/G3COvYdKTI6loiISK1T6SIeFBTErl27AJg/fz4A3bp1K7995cqV5UVdROoms8lEr9vCeenBGM6cK2HMhHWs3LLf6FgiIiK1SqWL+N13382kSZN46qmnGDt2LIGBgSQlJWG1WnnqqadYsmQJ9913X3VkFREb08LixajHOtIk2IPxGTv49/ztnL94yehYIiIitUKld035j//4D+zs7Jg3bx7t27dn2LBh2NvbU1RUxLp163jqqacYNGhQdWQVERvk4eLIf/Vrx5xVucxdtZe9Bwp5Oj2KIF8Xo6OJiIjYNFNZFW17UFZWRklJCQ4ODlXxcDXi2LEiSktrdtcHPz83jhwprNHnlOvTTKrOttzjfDp3GxculjKwZySJUQE3/Viai+3RTGyT5mJ7NBPbZMRczGYTPj6u1779Zh/47Nmz5f994sQJpk6dytdff83Jkydv9iFFpJZrHeHNqMEdsQS48dm87XyRuYMLWqoiIiJyVZVemnL69Gn+/Oc/c/r0aWbOnElRURH33XcfBw4coKysjI8++oipU6cSGhpaHXlFxMZ5uTnx0kPtmLUyl/k/5pF74DRPp0cR4N3Q6GgiIiI2pdJXxN977z3WrFlTvmXhV199xf79+3nppZeYOHEiZrOZ9957r8qDikjtYWc2c1+XJrzQty0nCs/z2hfZrP35kNGxREREbEqli/j333/PI488wnPPPQfA4sWL8fHx4bHHHqNjx47079+f1atXV3lQEal9opv4MGpwHMF+LnwyexuTFu3kYkmp0bFERERsQqWL+LFjx2jWrBlw+cN9Nm3aRKdOncpv9/LyqrB+XETqN293Z4Y/3J6eHUNZuqGAv09ez+GT+hkhIiJS6SLeqFEj8vPzgctXwy9dukTXrl3Lb9+wYQOBgYFVFlBEaj97OzP9ujfj2fvacOTEWUaPz2b9ziNGxxIRETFUpd+s2a1bNyZMmEBRURHz58/Hw8OD7t27c+jQIT777DNmz57NM888Ux1ZRaSWi2nmx6jBroyb/RMffbuVuzqE0rdbE+ztbnoDJxERkVqr0n/7vfTSS6SlpfHVV1/h7u7O2LFjcXZ25tChQ0yZMoVevXrxxBNPVEdWEakDfD0bMOKRWO6MDeG7dfn8Y8oGjp7SUhUREal/quwDfS5cuMDJkyfx9/evioerEfpAHwHNxEjrdhxmfObPmE0mhtzdinZNfctv01xsj2ZimzQX26OZ2CZb/ECfSi9N+dXJkydZvXo1BQUFODg4EBgYWOFNm3/UvHnzGDduHPn5+QQHB/Pkk0+Snp5+zfsfPnyYN998k1WrVnH+/HkSEhIYPnw4FovlZl+KiBikQwt/whq58vGsn/jnV1tIjg8j2NeFWStzOH76PN7uTvTp0oTE1jf/CZ0iIiK26qaK+NSpU3nrrbc4d+4cv72g7uTkxLBhw+jfv/8fepyMjAxefPFFBg0aROfOnVm8eDHDhw/H2dmZ5OTkK+5//vx5/vSnP3H+/Hn++te/4uzszEcffcQjjzzC/PnzcXd3v5mXIyIG8vdqyH8PiOXLJbtZsMaKyQS//lg5dvo8EzJ3AKiMi4hInVPpIr548WJee+01WrVqxZ/+9CcaN25MWVkZOTk5jB8/njFjxhAUFES3bt1u+Fhjx44lJSWFESNGAJCUlMSpU6d4//33r1rEly5dys6dO/n666+JiooCoFmzZtxxxx0sXLiQvn37VvbliIgNcLC3Y0DPSLJ3HKbo7MUKt10oKeWb5XtUxEVEpM6p9Js1P/vsM1q1asWXX35JamoqLVq0oGXLlqSlpTFt2jRatmzJ//7v/97wcfLz87FarfTo0aPC8Z49e5KTk1O+ReJvde7cmWnTppWXcAAHBwfg8hp1Eandfl/Cf3Xs9PkaTiIiIlL9Kl3Ed+zYQe/evXF0dLziNgcHB3r37s3PP/98w8fJyckBICIiosLxX9d65+bmXnGOq6sr7du3B+DixYvs2LGDl19+GS8vL+66667KvhQRsTE+7k5XPW42wcK1Vs6eL6nhRCIiItWn0kXc0dHxup+cWVxcjJ2d3Q0fp7Dw8rtWXV0rvpPUxcUFgKKiouue/+yzz9K7d2+ysrIYNmxYrdqtRUSurk+XJjjaV/yxZG9nopFXA6Z/v5th41bz7YocTp/Rv4CJiEjtV+k14nFxcUyZMoU+ffpcUX4PHTrE1KlTiY2NveHj3GjXRLP5+r8jPP744wwaNIg5c+aUrzHv06fPDZ/3t663nUx18vNzM+R55do0E9twT1c33N2cmZj5M0dPnMXXqwEDU1rSNTaUnXnH+Xrpbuau3svC7Hx6JlhI79IEf6+GRseuV/S9Yps0F9ujmdgmW5tLpfcR37VrF/369cNsNpOenk54eDhweanJnDlzuHTpUvla8etZtmwZTz75JHPmzCEyMrL8+LZt2+jTpw+ff/75H94OceDAgRw6dIiFCxdW5qVoH3EBNBNbda257D9aTOaaPLK2HQIgvlUjUuLDCPYz5hfr+kTfK7ZJc7E9moltqhP7iDdv3pwJEyYwZswYpkyZUuG2qKgoRo4cecMSDv+3NtxqtVYo4nl5eRVu/63t27eTm5tLWlpaheOtW7dm69atlX0pIlILBfm6MCStFfcmNWbh2nyWby5g9U8HiWnmS2qChSbBHkZHFBER+UNuah/x6OhoZsyYwbFjxygoKKCsrIzg4GB8fX3Jyspi4sSJDBw48LqPYbFYCAkJYcGCBRXeaLlo0SLCw8MJCgq64pysrCzefPNN2rRpQ1hYGACXLl0iKyuL5s2b38xLEZFaytvdmYfubMbdt1lYsn4fS9bvY+MvR4kM9SQ10UJUhDcmk8nomCIiItd005+sCeDj44OPj0+FY5mZmcyYMeOGRRxg6NChjBgxAg8PD7p27cqSJUvIzMxk7NixABw/fhyr1UrTpk1xdXWlT58+TJo0iaeffppnn30WZ2dnpkyZwq5du/j8889v5aWISC3l1tCR9KTGJMeHsWLzARautTJ2xmbC/F1JTbTQIdIfs1mFXEREbM8tFfFb1adPHy5cuMDnn3/OzJkzCQ0N5Y033iA1NRW4vI58xIgRTJw4kfj4eDw9PZk8eTJvv/02r732GsXFxURHRzNhwgQ6dOhg5EsREYM5O9rTIy6U7u2D+XHbQTKzrHwyexv+njkkx4fRqU0ADvY33tFJRESkplT6zZo38uqrrzJjxow/tJe40fRmTQHNxFbd6lxKy8rYuOsoGVl7yT1QiIeLIz3iQukaE0wDJ0OvQdRa+l6xTZqL7dFMbFOdeLOmiEhtYDaZiI30o31zX3bknSAjK4+Zy/Yw78c8urcP5s4OoXi4XPnBZCIiIjVFRVxE6jSTyUTLcG9ahnuz9+BpMn7MI+PHPBZl59M5OpDkjmH4eTYwOqaIiNRDNyzi+/fvr9QDFhcX33QYEZHqFB7gzjP3tuHg8TMsWJPHik37Wb5xPx1b+pOaYCHEX3uRi4hIzblhEe/evXultgArKyvTlmEiYtMCvBvyaEpLenduzKJsK8s27Sdr+yGim/iQmmCheain0RFFRKQeuGERT09PV7EWkTrJy82Jft2bkZYYztIN+/hu3T7+MWUDTUM8SEuwEN3ERz//RESk2lT5rim1iXZNEdBMbJURczl/8RIrN+9n4Vorx06fJ8TPhZQECx1b+mNnNtdoFluk7xXbpLnYHs3ENmnXFBERG+bkYMedHS5vcbj250NkZFn5bO52vl1xeS/yzm0CcXTQXuQiIlI1VMRFRH7H3s7MbVGBJLQOYPPuo2T8mMfkRbuY/UMud3W4/KFBDZ0djI4pIiK1nIq4iMg1mE0mYpr50a6pL7vyT5KRZeWbFTlkZOXRLSaYu+JC8XR1MjqmiIjUUiriIiI3YDKZiAzzIjLMC+uhQjKy8liw1sp36/Lp1CaQ5PgwGnk1NDqmiIjUMiriIiKVENbIjad6R9Hn9jMsWJvPD1sOsGLzfuJa+JMSb8ES4GZ0RBERqSVUxEVEboK/V0MG9ozknk7hfLcun6UbClj782GiIrxJTbAQGeaprQ9FROS6VMRFRG6Bp6sTfbs2JS3BwtKNBXyXnc+b0zbSJMid1AQLbZv5YlYhFxGRq1ARFxGpAg2dHUhLDOeuDqGs2nqAzDVWPvhmK4E+DUlNsBDfqhH2dtqLXERE/o+KuIhIFXJ0sKNb+xBubxdE9o7DZPxo5d/zf+bblTn07BjG7dFBODlqL3IREVERFxGpFnZmMwmtAohv2YitOcfI+DGPaYt/Ye6qvdwZG0L32BBcG2gvchGR+kxFXESkGplMJqKb+BLdxJdf9p0k48c8Zv2QS+YaK13aBdEjLhRvd2ejY4qIiAFUxEVEakizEE+e7+vJvsNFZK7JY/G6fSxZv4/EqABS4sMI9HExOqKIiNQgFXERkRoW4u/K471ac29SYxastbJyywFWbTlA+0g/UhMsRAS6Gx1RRERqgIq4iIhBfD0b8EiPSO7pFMHi9fksWV/A+p1HaGnxIi3RQkuLl/YiFxGpw1TERUQM5u7iSJ/bm5ASb2HZpgIWrc3n7S83ER7gRmqChfbN/TCbVchFROoaFXERERvRwMmelHgLd8aGsPqng2SusfLxrJ8I8G5ISnwYiVEB2otcRKQOUREXEbExDvZ2dGkXTFJ0EOt2HiYjK4/xmTuY9UMuPeJC6dIuCGdH/fgWEant9JNcRMRGmc0mOrZsRFwLf7btPU7Gj3lM/34381bv5Y7YEO6IDcGtoaPRMUVE5CapiIuI2DiTyURUhA9RET7sKThFRlYec1btZcEaK7e3DaJnxzB8PLQXuYhIbaMiLiJSizQJ9uDZ+6LZf7SYzDV5LN1YwNKNBSS0akRygoVgX+1FLiJSW6iIi4jUQkG+LgxJa0V658YszLayYvN+Vv10kJhmvqQmWGgS7GF0RBERuQEVcRGRWszHw5mH72xOr9vCWbL+8id1bvzlKC3CPElNsNA6wlt7kYuI2CgVcRGROsCtoSPpSY1Jjg9jxab9LMzO590ZmwnzdyU10UKHSH/tRS4iYmNUxEVE6hBnR3t6dAyje2wIP247SGaWlU9mb8PfM4fkhDA6RQXgYG9ndEwREUFFXESkTrK3M5MUHUSnqEA2/nKE+T/mMXHBTmavvLwXedeYYBo46a8AEREj6aewiEgdZjabiI30p31zP3bknWB+Vh4zl+1h3o95dG8fzF0dQnF30V7kIiJGUBEXEakHTCYTLcO9aRnuTe6B02Rm5ZHxYx6LsvPpHB1Icscw/DwbGB1TRKReUREXEalnIgLdeebeNhw8fobMrDxWbNrP8o376djKn9R4CyH+rkZHFBGpF1TERUTqqQDvhgxObUl6UmMWZVtZtnE/WdsOEd3Eh7REC81CPI2OKCJSp6mIi4jUc15uTvTr3oy0xHC+37CPxev28frkDTQL8SA1wUJ0Ex/tRS4iUg1UxEVEBADXBg7c0ymCnnFhrNyyn4Vrrbz/1RZC/FxITbCQmuRidEQRkTpFRVxERCpwcrTjzg6Xtzhcs/0QmWusfDp3O7NW7aVHhxA6twnE0UF7kYuI3CoVcRERuSp7OzOd2gSSGBXA5t1HWbRuH5MX7WLOD7ncFRdKt5hgGjo7GB1TRKTWMhsdYN68eaSlpREdHU1KSgqzZs267v2PHDnCyJEj6datGzExMfTp04fMzMwaSisiUv+YTSZimvnx1rNJDH84hrAAN75ensOLH69m5tLdnCw6b3REEZFaydAr4hkZGbz44osMGjSIzp07s3jxYoYPH46zszPJyclX3P/ChQv86U9/orCwkOeeew5/f38WLlzICy+8wKVLl7j77rsNeBUiIvWDyWQiMsyLyDAvrIcKycjKY8FaK9+t20fnNgH0jA+jkVdDo2OKiNQahhbxsWPHkpKSwogRIwBISkri1KlTvP/++1ct4itWrGDHjh3MnDmT6OhoADp16sT+/fv57LPPVMRFRGpIWCM3nuodxb23n2HhGis/bD3A8s37iWvhT0q8BUuAm9ERRURsnmFLU/Lz87FarfTo0aPC8Z49e5KTk0N+fv4V57i4uNCvXz/atGlT4Xjjxo2xWq3VmldERK7UyKshA5Nb8ObTt5EcH8aWPccY/UU2787YxE7rCcrKyoyOKCJiswy7Ip6TkwNAREREheMWiwWA3NxcQkNDK9yWmJhIYmJihWMXL15k+fLlNGvWrBrTiojI9Xi6OtG3a1PSEiws3VjAd9n5vDF1I02C3ElNsNC2mS9m7UUuIlKBYUW8sLAQAFfXih+l7OJyeZ/aoqKiP/Q4b731Fnv37uWjjz6q2oAiIlJpDZ0dSEsM564OoazaeoDMNVY++GYrQb4upMSHEd+qEfZ2hu8TICJiEwwr4jf650qz+fo/qMvKynjrrbeYMGECQ4YM4c4776x0Bh8f1xvfqRr4+WntpK3RTGyT5mJ7KjOTB4I8ue/OSFZu3s/X3//Cv+f/zJzVe0nv0oQe8RacHbWDblXR94rt0Uxsk63NxbCfgm5ul/9HFBcXVzj+65XwX2+/mgsXLvDyyy8zf/58hgwZwrBhw24qw7FjRZSW1uz6RT8/N44cKazR55Tr00xsk+Zie252Jq1DPWg1MJatOceY/2Men836iWkLd3JnhxC6tw/BtYH2Ir8V+l6xPZqJbTJiLmaz6boXfg0r4r+uDbdarURGRpYfz8vLq3D77xUVFfHkk0+yYcMGXnnlFQYNGlT9YUVE5JaYTCaim/gS3cSXXfknyczKY9bKXDKzrHRpF0TPjmF4uTkZHVNEpEYZVsQtFgshISEsWLCAu+66q/z4okWLCA8PJygo6IpzLl26xNNPP83mzZsZO3bsVbc4FBER29Y81JPmoZ7sO1xExpo8Fq/bx5L1+7gtKoDk+DACfVyMjigiUiMMXaA3dOhQRowYgYeHB127dmXJkiVkZmYyduxYAI4fP47VaqVp06a4urry5ZdfsnbtWvr160dAQACbNm0qfyyTyUTbtm2NeikiIlJJIf6uPNGrNfcmNWbhWisrtxzghy0HaB/pR2qChYhAd6MjiohUK1OZwZu8fvnll3z++eccOHCA0NBQnnjiCdLT0wH45ptvGDFiBBMnTiQ+Pp6BAweyZs2aqz6OnZ0d27dvr9Rza424gGZiqzQX21PdMzldfIHv1uXz/YYCzp4voVW4F6kJFlpavDBp68Nr0veK7dFMbJMtrhE3vIgbSUVcQDOxVZqL7ampmZw9X8KyTQUsWpvPqeILRAS6kZpgIaa5n/Yivwp9r9gezcQ22WIR195RIiJiUxo42ZMSb+HO2BBW/XSQBVlWPvr2JwK8G5ISH0ZiVID2IheROkFFXEREbJKDvR1d2wVze3QQ63YeJiMrj/GZO5j1Qy4940K5vV2Q9iIXkVpNP8FERMSmmc0mOrZsRFwLf7blHicjK48vv9/N3NV7uSM2hDtiQ3Br6Gh0TBGRSlMRFxGRWsFkMhHV2Ieoxj7sKThFRlYec1btZcFaK7e3DaJnXBg+Hs5GxxQR+cNUxEVEpNZpEuzBs/dFU3C0mAVZeSzdUMDSDQUktGpEcoKFYF/tRS4itk9FXEREaq1gXxeG3N2K9KTGLMy2smLzflb9dJCYZr6kJlpoEuRhdEQRkWtSERcRkVrPx8OZh+9sTq/bwlmy/vIndW785SgtwjxJTbDQOsJbe5GLiM1RERcRkTrDraEj6UmNSY4PY8Wm/SzMzufdGZsJa+RKaoKFDpH+mM0q5CJiG1TERUSkznF2tKdHxzC6tQ8ha9tBMtZY+WT2Nvy9ckiOD6NTVCAO9tqLXESMpSIuIiJ1loO9maS2QXRqE8jGX44w/8c8Ji7YyewfcukRF0rXdsE0cNJfhSJiDP30ERGROs9sNhEb6U/75n78nHeCjKw8Zi7dw7zVeXRvH8xdHUJxd9Fe5CJSs1TERUSk3jCZTLQK96ZVuDe5B06TkZVHxo95LMrOJyk6kOSOYfh6NjA6pojUEyriIiJSL0UEujP03jYcOFbMgjVWlm/az7KN++nYyp/UeAsh/q5GRxSROk5FXERE6rVAHxcGp7a8vBf52suFPGvbIdo28SE10UKzEE+jI4pIHaUiLiIiAni5OfHgHc24+7Zwvt+wj8Xr9vH65A00C/EgLdFCm8Y+2otcRKqUiriIiMhvuDZw4J5OEfSMC2PFlv0sXGvlvZlbCPFzITXBQlxLf+zM2vpQRG6diriIiMhVODnacVeHULrFBLNm+yEy11j5dO52vlmRQ0p8GJ3aBOLoYGd0TBGpxVTERURErsPezkynNoEkRgWw+ZejzM/KY9KiXcz+IZe74i4X9YbODkbHFJFaSEVcRETkDzCbTMQ096NdM1925Z9kflYeXy/PISMrj64xwfToEIqHq5PRMUWkFlERFxERqQSTyURkmBeRYV7kHSwkc00eC9ZY+S57H53bBJAcH4a/V0OjY4pILaAiLiIicpMsAW481TuKe28/w8I1Vn7YeoDlm/cT18Kf1AQLYY3cjI4oIjZMRVxEROQWNfJqyMDkFtzTOYLvsvNZurGAtT8fJqqxN2kJFpqHemrrQxG5goq4iIhIFfF0daJvt6akJVpYurGA77LzeWPqRpoEu5OaYKFtU1/MKuQi8v+piIuIiFSxhs4OpCWGc1eHUH7YeoAFa6x88PVWgnxdSIkPI75VI+zttBe5SH2nIi4iIlJNHB3s6N4+hC7tgsj++TAZWXn8e/7PzFqZQ8+OYSS1DcJJe5GL1Fsq4iIiItXMzmwmoXUA8a0asWXPMTKy8pi6+BfmrNrLnR1CuCM2BBftRS5S76iIi4iI1BCTyUTbpr60bXp5L/KMrDxmrcwlc42Vru2C6BEXhpeb9iIXqS9UxEVERAzQPNST5qGe5B8uInNNHt9l72Pxun3cFhVASoKFAG/tRS5S16mIi4iIGCjU35UnerXm3qTGLFhr5YctB/hhywFiI/1ITbQQHuBudEQRqSYq4iIiIjbAz7MBA3pEck+nCBavy+f7DQWs23mEVuFepCVYaGHx0l7kInXM/2vvzqOjru7/jz9nJplsk0xWsk0yZCEIpEiQVVApoAG1btRfW2tVLGA9XSyKC2orcooL1qNyRCsqKFCoX4pEZZOGb5VvRQpYQRRZE5LJBpKQhKAkJPn8/phkNCYsoUlmSF6PczhH79xL7idvLp93Przv/SgRFxER8SH2ECuTrkjj6hFOPvi0mA3bXDzztx2kxIdy9QgnWRkxOotcpJtQIi4iIuKDggL8mDjCyfghDj76vIz1WwqZv+pz4iKDmTgiGbPJRM7/5VFRXUtkWAA3XZHGyAFx3p62iLSDEnEREREf5u9nYcygRC4fmMD2vUdY+3EBi9buadGnvLqWN9e525SMi1w49FovERGRC4DZbGJYv1gemzyU0ODWZ47X1TeyPHcfRyu/wTAML8xQRNpLT8RFREQuICaTieNfn2rzs5pv6nngLx8TERpAH4edPg73EYmJ0SGYzaorF/E1SsRFREQuMFFhAZRXm746gwAAHeNJREFU17Zqt4dYufbS3uwvqmR/URVbvzwCuOvN3Ym5OzlPiQ/F38/S1dMWke9RIi4iInKBuemKNN5ct4e6+kZPm9XPzP8bm87IAXGMu8SBYRiUV51kX1NSvs9VyWcHywHws5joHR9GhiPck6AHB7YudxGRzqVEXERE5ALTvCHz7Q8PnvbUFJPJRHR4ENHhQVyaGQ/A8a/rOFBU5U7Miyp5f2sha7cYmIDEGBt9ktxJeYYjnMiwQG9cmkiP4vVEfPXq1bz88su4XC4SExO56667uOGGG85p7NNPP82XX37JG2+80bmTFBER8TEjB8QxckAcMTGhfPXV8XMaExpsJSsjhqyMGABqTzWQX1LteWq++fMy/vmfYgCiwgLJSHKXsvRJCic+Kljnl4t0MK8m4mvXrmXGjBncfvvtjB49mtzcXB588EECAwOZMGHCGccuWbKEhQsXMnLkyC6arYiISPcS4G/hImcEFzkjAGhobKToyAn2uSrZX1TJF4eO8fEXhwEICfRrSsrdT8ydcaH4WXT4msh/w6uJ+HPPPcfEiROZOXMmAJdddhlVVVW88MILp03EDx8+zNy5c1m7di2hoaFdOV0REZFuzWI244wLxRkXypVDkzAMgyOV3zQl5u6Slh0HjgLg72cmNT6MPknhZCTZSUuwExTg9X9oF7mgeG3FuFwuCgsLuffee1u0Z2dns27dOlwuF0lJSa3GPffcc+zevZtFixYxf/78rpquiIhIj2MymYiNCCY2IpjLBiYAUHWijgNFlexzuevM13x8iNWbwWSC5F6h7s2fSeFkOOzYbQHevQARH+e1RDwvLw+AlJSUFu1OpxOA/Pz8NhPxKVOmkJqaitlsViIuIiLSxewhVi7p24tL+vYC4JvaevJKq9nf9NR802cl5H5SBECv8KCmDaDu88xjI4Iwqc5cxMNrifjx4+6NJTabrUV7SEgIADU1NW2OS09P79yJiYiIyDkLCvBjQO9IBvSOBKC+oZHCwzWeOvOdB8r5aFcZAGHB/u4686an5smxNixm1ZlLz+W1RPxsr981d8HCjIqynb1TJ4iJUW27r1FMfJPi4nsUE9/ka3GJj7Mz/OJEwH2/LzpSw+78cnbnV7A7v5xP9n0FQKDVwkXOSPqnRNI/NYq+yREEdpM6c1+Libj5Wly89qe9eaPliRMnWrQ3Pwnvio2Y5eU1NDae+QeCjtaeY6akaygmvklx8T2KiW+6EOISaIbBaVEMTosC+nDseK377Z9NdebLN+zFACxmE8mxoZ43gPZJshMWbPX29NvtQohJT+SNuJjNpjM++PVaIt5cG15YWEjfvn097QUFBS0+FxERke4lIjSAYf1iGdYvFoCvT9ZzsKTKczrL//6nmA3bXADERQa3OM88xh6oOnPpNryWiDudThwOB+vXr+fKK6/0tG/YsIHevXuTkJDgramJiIhIFwoO9OMHqVH8IDUKgFP1jRSUHXe/aMhVyfY9X7FpZykA4Tarp848IykcR4wNs1mJuVyYvFqI9etf/5qZM2dit9sZM2YMGzduZN26dTz33HMAVFRUUFhYSHp6eqtNnSIiItI9+fuZSXfYSXfYYYSTRsOg5OgJz8ks+4oq2bbnCABBARbSEptOZnHYSYkPw+pv8fIViJwbrybiN910E3V1dSxcuJAVK1aQlJTE008/zdVXXw3ABx98wMyZM1m8eDHDhw/35lRFRETES8wmE44YG44YGz8c7ACgvOok+4sq2VdUxf6iSlZtch+LbDGbSIkP89SZpzvs2IL8vTl9kdMyGWc7vqQb02ZNAcXEVykuvkcx8U2Ki1vNN6c4UFzleWqeX1pNQ9M9PjE6hD5JTeUsjnCi7IGdOhfFxDdps6aIiIhIJ7AF+TMoPZpB6dEA1J1qIL+02vPE/N+7y/jg02IAIsMCyPjOeeYJ0SGYtQFUvECJuIiIiHQ7Vn8LfZMj6JscAUBjo0HRVzWek1m+LDzGlt2HAQgJ9CM90Z2UZzjCccaF4u+nFw1J51MiLiIiIt2euemM8uTYUMYPScIwDL6qOtlUyuJOznceLAfcm0Wb68wzksJJS7ATHKiUSTqe/lSJiIhIj2MymegVHkSv8CBG/SAegOqv6zhQ9O155uu2FLLm4wJMJkiKsXleMtTHEU5EaICXr0C6AyXiIiIiIkBYsJXBGTEMzogBoLaugbySKk+d+b92lbLxP0UARNsDyUj69jzzuMhgvWhI2k2JuIiIiEgbAqwW+vWOpF/vSADqGxpxHanxnMyyK6+czZ+XAe7Nos1HJg4fmECo1YyfRXXmcmZKxEVERETOgZ/FXTueEh/GVcPAMAwOH/uG/a5K91tAi6r4dP9R/uefB7D6m0lLsHtOZklLCCPQqrRLWtKfCBEREZHzYDKZiIsMJi4ymMsuTgCgsqaWI9V1bP+ilP1FVby3+RCG4X4pUXKsu848I8lOuiMce4jVy1cg3qZEXERERKSDhNsC6JMSTUZCKADf1NZzsKSKfa4qDhRV8sGOYv6x3QVAbETQty8aSgqnV3iQ6sx7GCXiIiIiIp0kKMCPzJQoMlOiAHedeUHZcXcpi6uKT/d9xb8+KwXAHmL11JlnJIXj6BWCxaw68+5MibiIiIhIF/GzmElLtJOWaGficGg0DErLv3afZd60CXT73q8ACLRaSEt015lnOMJJSQgjwN/i5SuQjqREXERERMRLzCYTidEhJEaHMGZQIgAV1SfZX1TleWr+zv/lYwAWs4necaEtzjO3Bfl79wLkv6JEXERERMSHRIYFMrx/IMP7xwJw4uQpDha768z3F1WS+4mL9VsLAYiPCv72PHNHOFH2QNWZX0CUiIuIiIj4sJBAfwamRTMwLRqAU/UN5Jced5ezFFWx9csjfLijBICI0IAWdeaJMSGYlZj7LCXiIiIiIhcQfz8LGUnuRBugsdGg+OgJ9hdVss/1bXIO7s2i7sTcnZynxIfi76c6c1+hRFxERETkAmY2m0jqZSOpl42xgx0YhkF51XfqzIuq+OxgOQB+FhMp8WHfnmeeaCc4UHXm3qJEXERERKQbMZlMRIcHER0exMjMOACOf13HgeIq9jfVmb+/tZC1WwxMQGKMjT5J7hrzPg47kWGB3r2AHkSJuIiIiEg3FxpsJatPDFl9YgCoPdVAfkm154n55s/L+Od/igGItgd6Sln6JIUTHxWsOvNOokRcREREpIcJ8LdwkTOCi5wRADQ0NlJ05ETTkYmVfHHoGB9/cRgAW5A/6Yl2z1NzZ1wofha9aKgjKBEXERER6eEsZjPOuFCccaFcOSQJwzA4UvkN+13f1pnvOHAUAKufmdSEMNKb6szTEuwEBSilPB/6romIiIhICyaTidiIYGIjghk9MB6AqhN1HCiq9JxnvvbjAlZvNjCZILlXqPss86Yzze22AC9fwYVBibiIiIiInJU9xMolfXtxSd9eAJysq+dgSTX7m45M3PRZCbmfFAHQKyLI85KhPknhxEYE6UVDbVAiLiIiIiLtFmj1Y0DvSAb0jgSgvqGRwsM1TWeZV7LzQDkf7SoDICzY373502GnT1I4ybE2LGbVmSsRFxEREZH/mp/FXTuemhDGhOHJGIZBWcXX7vPMm5LzT/Z9Bbg3i6YlNp1n7rCTmmAnwNrzXjSkRFxEREREOpzJZCI+KoT4qBAuvzgBgGPHa9lfVOk5z/zdf+VjABazieTYb+vM0x12woKt3r2ALqBEXERERES6RERoAMP6xTKsXywAX5+s52BJ8xPzKv73P8Vs2OYCID4quMV55jH2wG5XZ65EXERERES8IjjQjx+kRvGD1CgATtU3UlB23HOe+Sd7v2LTzlIAwm1WT515RlI4jhgbZvOFnZgrERcRERERn+DvZybdYSfdYYcRThoNg5KjJ9hfVNV0Oksl2/YcASAowEJaot1TZ54SH4bVv3Wd+cdflPH2hwepqK4lMiyAm65IY+SAuK6+tDYpERcRERERn2Q2mXDE2HDE2PhhViIA5VUn2V9Uyb4id535qk15APhZTPSOC/OczJKeaGdXXjlvrttDXX2je2x1LW+u2wPgE8m4EnERERERuWBE2QOJsscxoimRrvnmFAeKqzybQDdsc7Hu34WAexNoQ6PRYnxdfSNvf3hQibiIiIiIyH/DFuTPoPRoBqVHA1B3qoH80mr2F1XxdtPT8u8rr67tyimelk5SFxEREZFuw+pvoW9yBNde2puosIA2+5yuvaspERcRERGRbummK9Kw+rVMd61+Zm66Is1LM2pJpSkiIiIi0i0114Hr1BQRERERkS42ckAcIwfEERMTyldfHff2dFpQaYqIiIiIiBcoERcRERER8QIl4iIiIiIiXqBEXERERETEC7yeiK9evZprrrmGgQMHMnHiRHJycs7Y/8SJEzz++OOMGjWKrKwspk6dyqFDh7pmsiIiIiIiHcSrifjatWuZMWMGo0ePZv78+QwbNowHH3yQ9evXn3bM9OnTWb9+PTNmzODpp5/m8OHD3HbbbRw/7lu7YEVEREREzsSrxxc+99xzTJw4kZkzZwJw2WWXUVVVxQsvvMCECRNa9d++fTsffvghr776KpdffjkAQ4YMYdy4cSxfvpxp06Z16fxFRERERM6X156Iu1wuCgsLueqqq1q0Z2dnk5eXh8vlajXmo48+IiQkhFGjRnnaIiMjGTp0KJs2ber0OYuIiIiIdBSvJeJ5eXkApKSktGh3Op0A5OfntznG6XRisVhatCcnJ7fZX0RERETEV3mtNKW5pttms7VoDwkJAaCmpqbVmJqamlb9m8e01f9szGZTu8d0BG99XTk9xcQ3KS6+RzHxTYqL71FMfFNXx+VsX89ribhhGGf83Gxu/bD+TGPa6n82EREh7R7TEaKiWv8wId6lmPgmxcX3KCa+SXHxPYqJb/K1uHitNCU0NBRwH0f4Xc1Ptps//y6bzdaqf/Pv0daTchERERERX+W1RLy5NrywsLBFe0FBQYvPvz/G5XK1ejJeUFDQZn8REREREV/ltUTc6XTicDhanRm+YcMGevfuTUJCQqsxo0ePprq6ms2bN3vaKioq2L59O5deemmnz1lEREREpKNYZs2aNctbXzw0NJSXX36ZY8eOYTKZWLhwITk5OTz22GP06dOHiooK9u7di81mw2q1kpiYyNatW1m2bBnh4eGUlJTw8MMPYxgGTzzxBIGBgd66FBERERGRdjEZZ9s12cn+9re/sXDhQkpLS0lKSmLatGnccMMNALz99tvMnDmTxYsXM3z4cACqqqp46qmnyM3NpbGxkUsuuYSHHnqI1NRUb16GiIiIiEi7eD0RFxERERHpibxWIy4iIiIi0pMpERcRERER8QIl4iIiIiIiXqBEvAOtXr2aa665hoEDBzJx4kRycnLO2P/EiRM8/vjjjBo1iqysLKZOncqhQ4e6ZrI9SHvj8s4779C3b99Wv2bPnt1FM+45vvzySwYMGEBZWdkZ+2mtdK1zjYvWSudqbGxk+fLl/OhHPyIrK4vx48fz5JNPel5815b6+nqef/55rrjiCi6++GJuueUWPvvssy6cdfd2PjHZvn17m+vkrrvu6sKZd2+GYfDGG2+QnZ3NwIEDue6663jvvffOOMZX7itee8V9d7N27VpmzJjB7bffzujRo8nNzeXBBx8kMDCQCRMmtDlm+vTp7Nq1iwceeICQkBBefPFFbrvtNtasWdPmm0Wl/c4nLnv27MHpdDJ37twW7dHR0V0x5R7j4MGD3HXXXdTX15+1r9ZK12lPXLRWOtdrr73G888/zy9/+UtGjhxJfn4+8+bN48CBA7z++uttjpkzZw6rVq1ixowZJCQksGjRIu644w7eeecdkpKSuvgKup/zicnevXsJDg5m0aJFLdrDwsK6Yso9wiuvvMK8efP47W9/y6BBg9i0aRMzZszAYrFw9dVXtznGZ+4rhnSI8ePHG7///e9btN1zzz3GhAkT2uy/bds2IyMjw/jwww89beXl5cagQYOMV155pVPn2pO0Ny6GYRiTJ09uNUY6zqlTp4ylS5caWVlZxrBhw4yMjAyjtLT0tP21VrpGe+NiGFornamxsdEYOnSoMWvWrBbta9asMTIyMozdu3e3GuNyuYx+/foZy5Yt87TV1tYaY8aMMf74xz92+py7u/OJiWEYxqOPPmrcfPPNXTHFHqmurs4YOnSoMXv27Bbtt956q/Gzn/2szTG+dF9RaUoHcLlcFBYWctVVV7Voz87OJi8vD5fL1WrMRx99REhICKNGjfK0RUZGMnToUDZt2tTpc+4Jzicu4H7K17dv366YYo/0ySef8Oc//5k777yTGTNmnLW/1krXaG9cQGulM504cYLrrruOa6+9tkV78zszCgsLW43ZsmULDQ0NZGdne9qsVitjxozRWukA5xMTcJd6aZ10HovFwpIlS5g2bVqLdn9/f2pra9sc40v3FSXiHSAvLw+AlJSUFu1OpxOA/Pz8Nsc4nU4sFkuL9uTk5Db7S/udT1yOHDlCeXk5u3fvZsKECQwYMIDs7Oyz1pXLuUtLSyM3N5ff/OY3rf78t0VrpWu0Ny5aK53LZrPx6KOPcskll7Roz83NBSA9Pb3VmLy8POx2O5GRkS3anU4nJSUlnDx5svMm3AOcT0waGhrYv38/ZWVl3HjjjWRmZjJmzBgWLlyIode4dAiz2Uzfvn2JjY3FMAyOHj3KggUL2Lx5Mz/5yU/aHONL9xXViHeA48ePA+5F+l0hISEAbW7iqKmpadW/ecyZNn3IuTufuOzZsweAoqIi7r//fgICAsjJyeHBBx+koaGBSZMmdfKsu7/21g9rrXSN9sZFa6Xr7dy5kwULFjB+/HjS0tJafX6mtQLuJ7qBgYGdPs+e5GwxOXToECdPniQ/P597772XiIgINm7cyNy5c6mpqeF3v/udF2bdfW3YsMHzPR0zZgzXXXddm/186b6iRLwDnO2nWrO59T88nGlMW/2l/c4nLpmZmfzlL39h6NChnkU6evRoysvLeeGFF5RceIHWim/SWulan3zyCb/61a9wOBz86U9/arPP2f7OM5lMnTG1HutcYhIbG8urr75Kv379iImJAWDkyJGcPHmSV199lTvvvLPNhFDOT//+/Vm6dCl79+7lhRdeYNq0abz55put/uz70n1Fd7EO0Ly79sSJEy3am3+qamv3rc1ma9W/+ffQouwY5xOXyMhIfvjDH7aKwRVXXMHhw4epqKjopNnK6Wit+Catla6zdu1aJk+eTHx8PG+88QYRERFt9jvdWmn+O0/rpeO0JyaXX365JwlvNmbMGOrq6lRe18GSkpIYOnQot956K4888gj//ve/+fTTT1v186X7ihLxDtBcg/z9jRoFBQUtPv/+GJfL1eqnsoKCgjb7S/udT1w+/fRTVqxY0aq9trYWPz8/HZXnBVorvklrpWssWrSIe++9l0GDBvHXv/6VXr16nbZvamoqlZWVVFVVtWgvKCjA4XBgtVo7e7o9QntisnfvXpYtW8apU6datDfX658ugZdzV1lZSU5ODocPH27R3r9/f8C9n+X7fOm+okS8AzidThwOB+vXr2/RvmHDBnr37k1CQkKrMaNHj6a6uprNmzd72ioqKti+fTuXXnppp8+5JzifuOzYsYNHH33UU/8K7hc4vP/++wwePBh/f/9On7e0pLXim7RWOt+KFSt46qmnmDhxIq+99tpZf7hpXg/vv/++p62uro4PPvhAa6WDtDcmBQUFPP74461O4li7di0Oh4PExMTOnG6P0NjYyEMPPcRbb73Vov2jjz4CICMjo9UYX7qvWGbNmjWrS79iNxUaGsrLL7/MsWPHMJlMLFy4kJycHB577DH69OlDRUUFe/fuxWazYbVaSUxMZOvWrSxbtozw8HBKSkp4+OGHMQyDJ554QhtqOkh745KamsqaNWtYv349kZGRFBcX8+STT7Jz506effZZ4uLivH1J3cqXX37Jxo0bmTx5suefA7VWvO9c4qK10rnKy8uZMmUKsbGx3HfffZSXl1NWVub5ZbVaaWhoYPfu3VitVoKCgggLC6O4uJjXX3+doKAgjh07xuzZs3G5XMydO5fw8HBvX9YF7XxikpSUxKZNm3jvvfcICwujoqKC+fPns379ep544ok2N3hK+wQFBVFRUcHixYvx8/Ojrq6Od955hxdffJGbbrqJSZMm+fZ9pUtPLe/mli9fblx55ZVGZmamMXHiRGPVqlWez1auXGlkZGQYW7Zs8bRVVlYaDz30kDFkyBBj8ODBxtSpU42DBw96Y+rdWnvjUlRUZEyfPt249NJLjYEDBxq33HKLsW3bNm9Mvdtr/v5/98UxWived65x0VrpPKtWrTIyMjJO+ysnJ8fYsmWLkZGRYaxcudIzrra21pgzZ44xcuRI4+KLLzZuueUWY8eOHV68ku7jfGNSXl5u/OEPfzAuv/xyIzMz07jxxhuNf/zjH168ku6nrq7OWLBggXHVVVcZmZmZxvjx440FCxYYDQ0NhmH49n3FZBg6yFJEREREpKupRlxERERExAuUiIuIiIiIeIEScRERERERL1AiLiIiIiLiBUrERURERES8QIm4iIiIiIgX+Hl7AiIi0jEeeughVq1adcY+48aN46WXXuqiGbU0duxYEhMTWbJkiVe+voiIr1EiLiLSzcycOZOIiIg2P4uPj+/i2YiIyOkoERcR6WbGjx+Pw+Hw9jREROQsVCMuIiIiIuIFSsRFRHqgsWPH8sgjj7BixQrGjRvHoEGD+OlPf8qWLVta9d2+fTt33HEHWVlZZGVlcdttt7Ft27ZW/Xbu3MnUqVMZMmQIw4cPZ9q0aezdu7dVv3fffZdrrrmGzMxMsrOzWb58eadco4iIr1MiLiLSzVRXV1NRUdHmr4aGBk+/zZs3M3v2bLKzs7nnnnuoqKhgypQpbN261dNn48aN/OIXv6C0tJS7776bu+++m9LSUu644w42btzo6bd9+3Z+/vOfc/DgQaZMmcLdd9/NgQMHuO222ygqKvL027VrF3PmzGHChAnMnDkTq9XKrFmzyM3N7ZpvjoiIDzEZhmF4exIiIvLfO5dTU3JycujXrx9jx46luLiY+fPnM378eAAqKirIzs4mNTWVt956i/r6esaNG4fJZGL16tXYbDbAnehfe+21gDtR9/f35+abb6a0tJT33nvPs1E0Pz+fq6++msmTJ/PAAw8wduxYSkpKWLlyJQMGDACguLiYcePGcd111zF37tzO+taIiPgkbdYUEelmnnnmGaKjo9v8LDk52fPfqampniQcIDIykuuvv56lS5dSXl5OcXExZWVlzJgxw5OEA4SFhXHrrbfy7LPP8vnnn5OcnMxnn33GnXfe2eK0lpSUFFauXNnipJbevXt7knCAxMREIiMjOXr0aIdcu4jIhUSJuIhINzN48OBzOjUlPT29VZvT6cQwDIqLiz0lJSkpKa36paamAlBSUoLFYvGM/b7+/fu3+P+oqKhWfQIDAzl16tRZ5ysi0t2oRlxEpIfy9/dv1dZcQ26xWDhT5WLzZ/7+/jQ2NgJgMpnO+jXNZt12RESa6Ym4iEgPVVhY2KqtoKAAi8WCw+HwPKXOy8tr1S8/Px+AuLg4YmNjT/v7PfPMM9jtdqZNm9aRUxcR6Rb0aEJEpIfatWsXO3bs8Pz/0aNHeffddxkxYgR2u50BAwYQExPD8uXLqamp8fSrqalh2bJlxMTEkJmZSWxsLBdddBFr1qxp0c/lcrF48WLVf4uInIaeiIuIdDO5ubmnfcU9wPXXXw+A1Wpl6tSp3H777QQGBrJs2TIaGxt54IEHAHfZyaOPPsr06dOZNGkSP/7xjwH4+9//zpEjR5g3b56n1GTmzJlMmTKFSZMmcfPNN2M2m1m6dClhYWFMnTq1k69YROTCpERcRKSbefLJJ8/4eXMiPmjQIK655hpeeukljh8/zpAhQ7jvvvu46KKLPH0nTJiA3W7npZdeYv78+fj5+XHxxRczZ84chgwZ4uk3YsQI3nzzTebNm8f8+fMJCAhg6NCh3H///cTExHTOhYqIXOB0jriISA80duxYEhMTWbJkibenIiLSY6lGXERERETEC5SIi4iIiIh4gRJxEREREREvUI24iIiIiIgX6Im4iIiIiIgXKBEXEREREfECJeIiIiIiIl6gRFxERERExAuUiIuIiIiIeIEScRERERERL/j/hG0eCbeY4agAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 864x432 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "\n",
    "import seaborn as sns\n",
    "\n",
    "# Use plot styling from seaborn.\n",
    "sns.set(style='darkgrid')\n",
    "\n",
    "# Increase the plot size and font size.\n",
    "sns.set(font_scale=1.5)\n",
    "plt.rcParams[\"figure.figsize\"] = (12,6)\n",
    "\n",
    "# Plot the learning curve.\n",
    "plt.plot(loss_values, 'b-o')\n",
    "\n",
    "# Label the plot.\n",
    "plt.title(\"Training loss\")\n",
    "plt.xlabel(\"Epoch\")\n",
    "plt.ylabel(\"Loss\")\n",
    "\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "BweZTU3f0Yql"
   },
   "source": [
    "### 3.5 Evaluation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 51
    },
    "colab_type": "code",
    "id": "eFbpwZgrU2Zf",
    "outputId": "62752494-fe13-4dc3-e823-73a00a23b0d6"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Predicting labels for 38 test sentences...\n",
      "Batch 1\n",
      "correct = 14\n",
      "\n",
      "Batch 2\n",
      "correct = 28\n",
      "\n",
      "Batch 3\n",
      "correct = 31\n",
      "\n",
      "DONE.\n",
      "Total correct =  31\n",
      "Test accuracy = 0.82\n"
     ]
    }
   ],
   "source": [
    "# Prediction on test set\n",
    "\n",
    "print('Predicting labels for {:,} test sentences...'.format(len(test_inputs)))\n",
    "\n",
    "# Put model in evaluation mode\n",
    "model.eval()\n",
    "\n",
    "# Tracking variables \n",
    "predictions , true_labels = [], []\n",
    "\n",
    "# Predict \n",
    "idx = 0\n",
    "correct = 0\n",
    "for batch in test_dataloader:\n",
    "    \n",
    "    print(\"Batch {}\".format(idx + 1))\n",
    "    idx += 1\n",
    "    \n",
    "    # Add batch to device\n",
    "    batch = tuple(t.to(device) for t in batch)\n",
    "  \n",
    "    # Unpack the inputs from our dataloader\n",
    "    b_input_ids, b_input_mask, b_labels = batch\n",
    "  \n",
    "    # Telling the model not to compute or store gradients, saving memory and \n",
    "    # speeding up prediction\n",
    "    with torch.no_grad():\n",
    "        # Forward pass, calculate logit predictions\n",
    "        outputs = model(b_input_ids, token_type_ids=None, \n",
    "                        attention_mask=b_input_mask)\n",
    "\n",
    "    # Get the prediction probability\n",
    "    logits = outputs[0]\n",
    "    \n",
    "    # Get the prediction label\n",
    "    pred = np.argmax(logits, axis=1)\n",
    "   \n",
    "    # Get the total correct number between pred and true labels in each batch\n",
    "    correct += (pred == b_labels).sum().item()\n",
    "    print(\"correct = {}\\n\".format(correct))\n",
    "    \n",
    "\n",
    "print('DONE.')\n",
    "print(\"Total correct = \", correct)\n",
    "print(\"Test accuracy = {0:.2f}\".format(correct / len(test_inputs)))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 3.6 Saving Trained Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "('./saved_models/vocab.txt',\n",
       " './saved_models/special_tokens_map.json',\n",
       " './saved_models/added_tokens.json')"
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import os\n",
    "\n",
    "saved_model_dir = \"./saved_models/\"\n",
    "\n",
    "if not os.path.exists(saved_model_dir):\n",
    "    os.makedirs(saved_model_dir)\n",
    "\n",
    "# Save model to the saved_model_dir\n",
    "model.save_pretrained(saved_model_dir)\n",
    "tokenizer.save_pretrained(saved_model_dir)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 3.7 Load Trained Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Load a trained model and vocabulary that you have fine-tuned\n",
    "\n",
    "# Model_class: BertForSequenceClassification, XLNetForSequenceClassification, XLMForSequenceClassification, RobertaForSequenceClassification\n",
    "load_model = BertForSequenceClassification.from_pretrained(saved_model_dir)       \n",
    "\n",
    "# Tokenizer_class: BertTokenizer, XLNetTokenizer, XLMTokenizer, RobertaTokenizer\n",
    "load_tokenizer = BertTokenizer.from_pretrained(saved_model_dir)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "accelerator": "GPU",
  "colab": {
   "collapsed_sections": [],
   "name": "Colab_Praparation_CPU.ipynb",
   "provenance": [],
   "toc_visible": true
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}