BertTextClassificationBC3AST.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Bert text classification on SST2 using PyTorch\n",
    "\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Requirement already satisfied: sagemaker==2.9.1 in /Users/aeg/envs/bert-sst2-bc3/lib/python3.7/site-packages (from -r requirements_notebook.txt (line 1)) (2.9.1)\n",
      "Requirement already satisfied: boto3==1.16.1 in /Users/aeg/envs/bert-sst2-bc3/lib/python3.7/site-packages (from -r requirements_notebook.txt (line 2)) (1.16.1)\n",
      "Requirement already satisfied: scikit-learn==0.23.1 in /Users/aeg/envs/bert-sst2-bc3/lib/python3.7/site-packages (from -r requirements_notebook.txt (line 3)) (0.23.1)\n",
      "Requirement already satisfied: s3transfer<0.4.0,>=0.3.0 in /Users/aeg/envs/bert-sst2-bc3/lib/python3.7/site-packages (from boto3==1.16.1->-r requirements_notebook.txt (line 2)) (0.3.4)\n",
      "Requirement already satisfied: botocore<1.20.0,>=1.19.1 in /Users/aeg/envs/bert-sst2-bc3/lib/python3.7/site-packages (from boto3==1.16.1->-r requirements_notebook.txt (line 2)) (1.19.56)\n",
      "Requirement already satisfied: jmespath<1.0.0,>=0.7.1 in /Users/aeg/envs/bert-sst2-bc3/lib/python3.7/site-packages (from boto3==1.16.1->-r requirements_notebook.txt (line 2)) (0.10.0)\n",
      "Requirement already satisfied: protobuf>=3.1 in /Users/aeg/envs/bert-sst2-bc3/lib/python3.7/site-packages (from sagemaker==2.9.1->-r requirements_notebook.txt (line 1)) (3.14.0)\n",
      "Requirement already satisfied: numpy>=1.9.0 in /Users/aeg/envs/bert-sst2-bc3/lib/python3.7/site-packages (from sagemaker==2.9.1->-r requirements_notebook.txt (line 1)) (1.19.5)\n",
      "Requirement already satisfied: importlib-metadata>=1.4.0 in /Users/aeg/envs/bert-sst2-bc3/lib/python3.7/site-packages (from sagemaker==2.9.1->-r requirements_notebook.txt (line 1)) (3.4.0)\n",
      "Requirement already satisfied: smdebug-rulesconfig==0.1.5 in /Users/aeg/envs/bert-sst2-bc3/lib/python3.7/site-packages (from sagemaker==2.9.1->-r requirements_notebook.txt (line 1)) (0.1.5)\n",
      "Requirement already satisfied: google-pasta in /Users/aeg/envs/bert-sst2-bc3/lib/python3.7/site-packages (from sagemaker==2.9.1->-r requirements_notebook.txt (line 1)) (0.2.0)\n",
      "Requirement already satisfied: protobuf3-to-dict>=0.1.5 in /Users/aeg/envs/bert-sst2-bc3/lib/python3.7/site-packages (from sagemaker==2.9.1->-r requirements_notebook.txt (line 1)) (0.1.5)\n",
      "Requirement already satisfied: packaging>=20.0 in /Users/aeg/envs/bert-sst2-bc3/lib/python3.7/site-packages (from sagemaker==2.9.1->-r requirements_notebook.txt (line 1)) (20.8)\n",
      "Requirement already satisfied: threadpoolctl>=2.0.0 in /Users/aeg/envs/bert-sst2-bc3/lib/python3.7/site-packages (from scikit-learn==0.23.1->-r requirements_notebook.txt (line 3)) (2.1.0)\n",
      "Requirement already satisfied: joblib>=0.11 in /Users/aeg/envs/bert-sst2-bc3/lib/python3.7/site-packages (from scikit-learn==0.23.1->-r requirements_notebook.txt (line 3)) (1.0.0)\n",
      "Requirement already satisfied: scipy>=0.19.1 in /Users/aeg/envs/bert-sst2-bc3/lib/python3.7/site-packages (from scikit-learn==0.23.1->-r requirements_notebook.txt (line 3)) (1.6.0)\n",
      "Requirement already satisfied: urllib3<1.27,>=1.25.4 in /Users/aeg/envs/bert-sst2-bc3/lib/python3.7/site-packages (from botocore<1.20.0,>=1.19.1->boto3==1.16.1->-r requirements_notebook.txt (line 2)) (1.26.2)\n",
      "Requirement already satisfied: python-dateutil<3.0.0,>=2.1 in /Users/aeg/envs/bert-sst2-bc3/lib/python3.7/site-packages (from botocore<1.20.0,>=1.19.1->boto3==1.16.1->-r requirements_notebook.txt (line 2)) (2.8.1)\n",
      "Requirement already satisfied: zipp>=0.5 in /Users/aeg/envs/bert-sst2-bc3/lib/python3.7/site-packages (from importlib-metadata>=1.4.0->sagemaker==2.9.1->-r requirements_notebook.txt (line 1)) (3.4.0)\n",
      "Requirement already satisfied: typing-extensions>=3.6.4 in /Users/aeg/envs/bert-sst2-bc3/lib/python3.7/site-packages (from importlib-metadata>=1.4.0->sagemaker==2.9.1->-r requirements_notebook.txt (line 1)) (3.7.4.3)\n",
      "Requirement already satisfied: pyparsing>=2.0.2 in /Users/aeg/envs/bert-sst2-bc3/lib/python3.7/site-packages (from packaging>=20.0->sagemaker==2.9.1->-r requirements_notebook.txt (line 1)) (2.4.7)\n",
      "Requirement already satisfied: six>=1.9 in /Users/aeg/envs/bert-sst2-bc3/lib/python3.7/site-packages (from protobuf>=3.1->sagemaker==2.9.1->-r requirements_notebook.txt (line 1)) (1.15.0)\n"
     ]
    }
   ],
   "source": [
    "!pip install -r requirements_notebook.txt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import sys, os, shutil\n",
    "import logging\n",
    "\n",
    "sys.path.append(\"src\")\n",
    "\n",
    "logging.basicConfig(level=\"INFO\", handlers=[logging.StreamHandler(sys.stdout)],\n",
    "                        format='%(asctime)s - %(name)s - %(levelname)s - %(message)s')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "resume_from_checkpoint=False"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import boto3\n",
    "\n",
    "def delete_s3_objects(s3_uri):\n",
    "    client = boto3.client('s3')\n",
    "    s3_uri = s3_uri.replace(\"s3://\",\"\")\n",
    "    bucket, prefix = s3_uri.split(\"/\")[0], \"/\".join( s3_uri.split(\"/\")[1:])\n",
    "    \n",
    "    response = client.list_objects(\n",
    "    Bucket=bucket,\n",
    "    Delimiter='|',\n",
    "    Prefix=prefix,\n",
    "    MaxKeys = 20\n",
    ")\n",
    "    s3 = boto3.resource('s3')\n",
    "    for item in response.get(\"Contents\", []):\n",
    "        print(\"Deleting {}\".format(item[\"Key\"]))\n",
    "        obj = s3.Object(bucket, item[\"Key\"] )\n",
    "        obj.delete()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "if not resume_from_checkpoint:\n",
    "    delete_s3_objects(s3_checkpoint)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Bucket and role set up"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2021-01-20 20:26:47,687 - sagemaker.analytics - WARNING - pandas failed to import. Analytics features will be impaired or broken.\n",
      "2021-01-20 20:26:47,819 - botocore.credentials - INFO - Found credentials in shared credentials file: ~/.aws/credentials\n",
      "2021-01-20 20:26:48,084 - botocore.credentials - INFO - Found credentials in shared credentials file: ~/.aws/credentials\n",
      "2021-01-20 20:26:48,248 - botocore.credentials - INFO - Found credentials in shared credentials file: ~/.aws/credentials\n"
     ]
    }
   ],
   "source": [
    "import sagemaker, boto3\n",
    "from sagemaker import get_execution_role\n",
    "sm_session = sagemaker.session.Session()\n",
    "\n",
    "\n",
    "sagemaker_session = sagemaker.Session()\n",
    "account_id =  boto3.client('sts').get_caller_identity().get('Account')\n",
    "region = boto3.session.Session().region_name\n",
    "\n",
    "\n",
    "#role = sagemaker.get_execution_role()\n",
    "role=\"arn:aws:iam::{}:role/service-role/AmazonSageMaker-ExecutionRole-20190118T115449\".format(account_id)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "data_bucket = sm_session.default_bucket()\n",
    "\n",
    "data_bucket_prefix = \"bert-bc3ast-classify\"\n",
    "\n",
    "s3_uri_data = \"s3://{}/{}/data\".format(data_bucket, data_bucket_prefix)\n",
    "s3_uri_train = \"{}/{}\".format(s3_uri_data, \"train.csv\")\n",
    "s3_uri_val = \"{}/{}\".format(s3_uri_data, \"dev.csv\")\n",
    "\n",
    "s3_uri_test = \"{}/{}\".format(s3_uri_data, \"test.csv\")\n",
    "\n",
    "s3_output_path = \"s3://{}/{}/output\".format(data_bucket, data_bucket_prefix)\n",
    "s3_code_path = \"s3://{}/{}/code\".format(data_bucket, data_bucket_prefix)\n",
    "s3_checkpoint = \"s3://{}/{}/checkpoint\".format(data_bucket, data_bucket_prefix)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "prepare_dataset = True"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Prepare dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "raw_data_dir = \"tmp\"\n",
    "processed_out_dir = os.path.join(raw_data_dir, \"processd\")\n",
    "\n",
    "if os.path.exists(processed_out_dir):\n",
    "    shutil.rmtree(processed_out_dir)\n",
    "\n",
    "os.makedirs(processed_out_dir, exist_ok=True)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2021-01-20 20:26:53,613 - utils.bc3ast_preprocess - INFO - Writing to <_io.TextIOWrapper name='tmp/processd/train_full.tsv' mode='w' encoding='UTF-8'>\n",
      "2021-01-20 20:26:54,228 - utils.bc3ast_preprocess - INFO - Writing to <_io.TextIOWrapper name='tmp/processd/test.tsv' mode='w' encoding='UTF-8'>\n",
      "Uploading file tmp/processd/train.tsv to s3://sagemaker-us-east-2-324346001917/bert-bc3ast-classify/data/train.csv in 9.874878 seconds\n",
      "Uploading file tmp/processd/val.tsv to s3://sagemaker-us-east-2-324346001917/bert-bc3ast-classify/data/dev.csv in 7.431086 seconds\n",
      "Uploading file tmp/processd/test.tsv to s3://sagemaker-us-east-2-324346001917/bert-bc3ast-classify/data/test.csv in 14.291215 seconds\n"
     ]
    }
   ],
   "source": [
    "from utils.bc3ast_preprocess import BC3ASTPreprocess\n",
    "from s3_util import S3Util\n",
    "\n",
    "if prepare_dataset:\n",
    "    train_file = os.path.join(raw_data_dir , \"bc3_act_all_records.tsv\")\n",
    "    train_label_file = os.path.join(raw_data_dir , \"bc3_act_gold_standard.tsv\")\n",
    "\n",
    "    test_file = os.path.join(raw_data_dir , \"bc3_act_all_records_test.tsv\")\n",
    "    test_label_file = os.path.join(raw_data_dir , \"bc3_act_gold_standard_test.tsv\")\n",
    "\n",
    "\n",
    "    processed_train_val_file = os.path.join(processed_out_dir , \"train_full.tsv\")\n",
    "    processed_train_file = os.path.join(processed_out_dir , \"train.tsv\")\n",
    "    processed_val_file = os.path.join(processed_out_dir , \"val.tsv\")\n",
    "    processed_test_file = os.path.join(processed_out_dir , \"test.tsv\")\n",
    "        \n",
    "    BC3ASTPreprocess().process( train_file, train_label_file,  processed_train_val_file)\n",
    "    BC3ASTPreprocess().split(processed_train_val_file, processed_train_file, processed_val_file,  split=0.8)\n",
    "\n",
    "    BC3ASTPreprocess().process(test_file, test_label_file, processed_test_file)\n",
    "\n",
    "    S3Util().upload_file(processed_train_file, s3_uri_train )\n",
    "    S3Util().upload_file(processed_val_file, s3_uri_val )\n",
    "    S3Util().upload_file(processed_test_file, s3_uri_test )\n",
    "    \n",
    "    "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Train\n",
    "\n",
    "This shows you how to train BERT on SageMaker using SPOT instances"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "inputs_full =  {\n",
    "    \"train\" : s3_uri_train,\n",
    "    \"val\" : s3_uri_val,\n",
    "\n",
    "}\n",
    "\n",
    "\n",
    "\n",
    "inputs = inputs_full"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "sm_localcheckpoint_dir=\"/opt/ml/checkpoints/\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "instance_type = \"ml.p3.2xlarge\"\n",
    "instance_type_gpu_map = {\"ml.p3.8xlarge\":4, \"ml.p3.2xlarge\": 1, \"ml.p3.16xlarge\":8}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "hp = {\n",
    "\"epochs\" : 30,\n",
    "\"earlystoppingpatience\" : 3,\n",
    "# Increasing batch size might end up with CUDA OOM error, increase grad accumulation instead\n",
    "\"batch\" : 8 * instance_type_gpu_map[instance_type],\n",
    "\"trainfile\" :s3_uri_train.split(\"/\")[-1],\n",
    "\"valfile\" : s3_uri_val.split(\"/\")[-1],\n",
    "\"datasetfactory\":\"datasets.bc3ast_dataset_factory.BC3ASTDatasetFactory\",\n",
    "# The number of steps to accumulate gradients for\n",
    "\"gradaccumulation\" : 4,\n",
    "\"log-level\":\"INFO\",\n",
    "# This param depends on your model max pos embedding size or when large you might end up with CUDA OOM error    \n",
    "\"maxseqlen\" : 512,\n",
    "# Make sure the lr is quite small, as this is a pretrained model..\n",
    "\"lr\":0.00001,\n",
    "# Use finetuning (set to 1), if you only want to change the weights in the final classification layer.. \n",
    "\"finetune\": 0,\n",
    "\"checkpointdir\" : sm_localcheckpoint_dir,\n",
    "# Checkpoints once every n epochs\n",
    "\"checkpointfreq\": 2\n",
    "}\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'epochs': 30,\n",
       " 'earlystoppingpatience': 3,\n",
       " 'batch': 8,\n",
       " 'trainfile': 'train.csv',\n",
       " 'valfile': 'dev.csv',\n",
       " 'datasetfactory': 'datasets.bc3ast_dataset_factory.BC3ASTDatasetFactory',\n",
       " 'gradaccumulation': 4,\n",
       " 'log-level': 'INFO',\n",
       " 'maxseqlen': 512,\n",
       " 'lr': 1e-05,\n",
       " 'finetune': 0,\n",
       " 'checkpointdir': '/opt/ml/checkpoints/',\n",
       " 'checkpointfreq': 2}"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "hp"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'train': 's3://sagemaker-us-east-2-324346001917/bert-bc3ast-classify/data/train.csv',\n",
       " 'val': 's3://sagemaker-us-east-2-324346001917/bert-bc3ast-classify/data/dev.csv'}"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "inputs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "metric_definitions = [{\"Name\": \"TrainLoss\",\n",
    "                     \"Regex\": \"###score: train_loss### (\\d*[.]?\\d*)\"}\n",
    "                    ,{\"Name\": \"ValidationLoss\",\n",
    "                     \"Regex\": \"###score: val_loss### (\\d*[.]?\\d*)\"}\n",
    "                    ,{\"Name\": \"TrainScore\",\n",
    "                     \"Regex\": \"###score: train_score### (\\d*[.]?\\d*)\"}\n",
    "                   ,{\"Name\": \"ValidationScore\",\n",
    "                     \"Regex\": \"###score: val_score### (\\d*[.]?\\d*)\"}\n",
    "                    ]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "# set True if you need spot instance\n",
    "use_spot = True\n",
    "train_max_run_secs =   2*24 * 60 * 60\n",
    "spot_wait_sec =  5 * 60\n",
    "max_wait_time_secs = train_max_run_secs +  spot_wait_sec\n",
    "\n",
    "if not use_spot:\n",
    "    max_wait_time_secs = None\n",
    "    s3_checkpoint=None\n",
    "    sm_localcheckpoint_dir=None\n",
    "    hp.pop(\"checkpointdir\")\n",
    "    \n",
    "# During local mode, no spot.., use smaller dataset\n",
    "if instance_type == 'local':\n",
    "    use_spot = False\n",
    "    max_wait_time_secs = 0\n",
    "    wait = True\n",
    "    # Use smaller dataset to run locally\n",
    "    inputs = inputs_sample\n",
    "    \n",
    "    \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "job_type = \"bert-sst2-classification\"\n",
    "base_name = \"{}\".format(job_type)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2021-01-20 20:27:30,161 - sagemaker - INFO - Creating training-job with name: bert-sst2-classification-2021-01-20-09-27-26-442\n",
      "2021-01-20 09:27:32 Starting - Starting the training job...\n",
      "2021-01-20 09:27:33 Starting - Launching requested ML instances...\n",
      "2021-01-20 09:28:37 Starting - Preparing the instances for training......\n",
      "2021-01-20 09:29:45 Downloading - Downloading input data...\n",
      "2021-01-20 09:30:20 Training - Downloading the training image......\n",
      "2021-01-20 09:31:35 Training - Training image download completed. Training in progress..\u001b[34mbash: cannot set terminal process group (-1): Inappropriate ioctl for device\u001b[0m\n",
      "\u001b[34mbash: no job control in this shell\u001b[0m\n",
      "\u001b[34m2021-01-20 09:31:37,557 sagemaker-containers INFO     Imported framework sagemaker_pytorch_container.training\u001b[0m\n",
      "\u001b[34m2021-01-20 09:31:37,582 sagemaker_pytorch_container.training INFO     Block until all host DNS lookups succeed.\u001b[0m\n",
      "\u001b[34m2021-01-20 09:31:37,586 sagemaker_pytorch_container.training INFO     Invoking user training script.\u001b[0m\n",
      "\u001b[34m2021-01-20 09:31:37,956 sagemaker-containers INFO     Module default_user_module_name does not provide a setup.py. \u001b[0m\n",
      "\u001b[34mGenerating setup.py\u001b[0m\n",
      "\u001b[34m2021-01-20 09:31:37,956 sagemaker-containers INFO     Generating setup.cfg\u001b[0m\n",
      "\u001b[34m2021-01-20 09:31:37,956 sagemaker-containers INFO     Generating MANIFEST.in\u001b[0m\n",
      "\u001b[34m2021-01-20 09:31:37,956 sagemaker-containers INFO     Installing module with the following command:\u001b[0m\n",
      "\u001b[34m/opt/conda/bin/python -m pip install . -r requirements.txt\u001b[0m\n",
      "\u001b[34mProcessing /tmp/tmpny6qvk64/module_dir\u001b[0m\n",
      "\u001b[34mCollecting transformers==2.3.0\n",
      "  Downloading transformers-2.3.0-py3-none-any.whl (447 kB)\u001b[0m\n",
      "\u001b[34mCollecting sentencepiece==0.1.91\n",
      "  Downloading sentencepiece-0.1.91-cp36-cp36m-manylinux1_x86_64.whl (1.1 MB)\u001b[0m\n",
      "\u001b[34mCollecting scikit-learn==0.23.1\n",
      "  Downloading scikit_learn-0.23.1-cp36-cp36m-manylinux1_x86_64.whl (6.8 MB)\u001b[0m\n",
      "\u001b[34mCollecting sacremoses\n",
      "  Downloading sacremoses-0.0.43.tar.gz (883 kB)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: requests in /opt/conda/lib/python3.6/site-packages (from transformers==2.3.0->-r requirements.txt (line 1)) (2.22.0)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: numpy in /opt/conda/lib/python3.6/site-packages (from transformers==2.3.0->-r requirements.txt (line 1)) (1.16.4)\u001b[0m\n",
      "\u001b[34mCollecting regex!=2019.12.17\n",
      "  Downloading regex-2020.11.13-cp36-cp36m-manylinux2014_x86_64.whl (723 kB)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: boto3 in /opt/conda/lib/python3.6/site-packages (from transformers==2.3.0->-r requirements.txt (line 1)) (1.13.23)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: tqdm in /opt/conda/lib/python3.6/site-packages (from transformers==2.3.0->-r requirements.txt (line 1)) (4.42.1)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: joblib>=0.11 in /opt/conda/lib/python3.6/site-packages (from scikit-learn==0.23.1->-r requirements.txt (line 3)) (0.15.1)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: scipy>=0.19.1 in /opt/conda/lib/python3.6/site-packages (from scikit-learn==0.23.1->-r requirements.txt (line 3)) (1.2.2)\u001b[0m\n",
      "\u001b[34mCollecting threadpoolctl>=2.0.0\n",
      "  Downloading threadpoolctl-2.1.0-py3-none-any.whl (12 kB)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: six in /opt/conda/lib/python3.6/site-packages (from sacremoses->transformers==2.3.0->-r requirements.txt (line 1)) (1.14.0)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: click in /opt/conda/lib/python3.6/site-packages (from sacremoses->transformers==2.3.0->-r requirements.txt (line 1)) (7.1.2)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: chardet<3.1.0,>=3.0.2 in /opt/conda/lib/python3.6/site-packages (from requests->transformers==2.3.0->-r requirements.txt (line 1)) (3.0.4)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /opt/conda/lib/python3.6/site-packages (from requests->transformers==2.3.0->-r requirements.txt (line 1)) (1.25.8)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: certifi>=2017.4.17 in /opt/conda/lib/python3.6/site-packages (from requests->transformers==2.3.0->-r requirements.txt (line 1)) (2020.4.5.1)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: idna<2.9,>=2.5 in /opt/conda/lib/python3.6/site-packages (from requests->transformers==2.3.0->-r requirements.txt (line 1)) (2.8)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: jmespath<1.0.0,>=0.7.1 in /opt/conda/lib/python3.6/site-packages (from boto3->transformers==2.3.0->-r requirements.txt (line 1)) (0.10.0)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: botocore<1.17.0,>=1.16.23 in /opt/conda/lib/python3.6/site-packages (from boto3->transformers==2.3.0->-r requirements.txt (line 1)) (1.16.23)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: s3transfer<0.4.0,>=0.3.0 in /opt/conda/lib/python3.6/site-packages (from boto3->transformers==2.3.0->-r requirements.txt (line 1)) (0.3.3)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: docutils<0.16,>=0.10 in /opt/conda/lib/python3.6/site-packages (from botocore<1.17.0,>=1.16.23->boto3->transformers==2.3.0->-r requirements.txt (line 1)) (0.15.2)\u001b[0m\n",
      "\u001b[34mRequirement already satisfied: python-dateutil<3.0.0,>=2.1 in /opt/conda/lib/python3.6/site-packages (from botocore<1.17.0,>=1.16.23->boto3->transformers==2.3.0->-r requirements.txt (line 1)) (2.8.1)\u001b[0m\n",
      "\u001b[34mBuilding wheels for collected packages: default-user-module-name, sacremoses\n",
      "  Building wheel for default-user-module-name (setup.py): started\u001b[0m\n",
      "\u001b[34m  Building wheel for default-user-module-name (setup.py): finished with status 'done'\n",
      "  Created wheel for default-user-module-name: filename=default_user_module_name-1.0.0-py2.py3-none-any.whl size=32248 sha256=b355b23f43d675e58620ae41a697c7b3838ca35a1f584528b92cbed1e350e8b4\n",
      "  Stored in directory: /tmp/pip-ephem-wheel-cache-udt96nz1/wheels/39/a7/bd/bec63015864429a3485993ae226fcddd196a2d3d3d9b158bbc\n",
      "  Building wheel for sacremoses (setup.py): started\n",
      "  Building wheel for sacremoses (setup.py): finished with status 'done'\n",
      "  Created wheel for sacremoses: filename=sacremoses-0.0.43-py3-none-any.whl size=893259 sha256=fcd8282e38ba35cb83fb46b821da6f53b3134ca363fdfa8c790e73c328207ee7\n",
      "  Stored in directory: /root/.cache/pip/wheels/49/25/98/cdea9c79b2d9a22ccc59540b1784b67f06b633378e97f58da2\u001b[0m\n",
      "\u001b[34mSuccessfully built default-user-module-name sacremoses\u001b[0m\n",
      "\u001b[34mInstalling collected packages: regex, sacremoses, sentencepiece, transformers, threadpoolctl, scikit-learn, default-user-module-name\n",
      "  Attempting uninstall: scikit-learn\n",
      "    Found existing installation: scikit-learn 0.21.2\u001b[0m\n",
      "\u001b[34m    Uninstalling scikit-learn-0.21.2:\n",
      "      Successfully uninstalled scikit-learn-0.21.2\u001b[0m\n",
      "\u001b[34mSuccessfully installed default-user-module-name-1.0.0 regex-2020.11.13 sacremoses-0.0.43 scikit-learn-0.23.1 sentencepiece-0.1.91 threadpoolctl-2.1.0 transformers-2.3.0\u001b[0m\n",
      "\u001b[34mWARNING: You are using pip version 20.1.1; however, version 20.3.3 is available.\u001b[0m\n",
      "\u001b[34mYou should consider upgrading via the '/opt/conda/bin/python -m pip install --upgrade pip' command.\u001b[0m\n",
      "\u001b[34m2021-01-20 09:31:47,363 sagemaker-containers INFO     Invoking user script\n",
      "\u001b[0m\n",
      "\u001b[34mTraining Env:\n",
      "\u001b[0m\n",
      "\u001b[34m{\n",
      "    \"additional_framework_parameters\": {},\n",
      "    \"channel_input_dirs\": {\n",
      "        \"val\": \"/opt/ml/input/data/val\",\n",
      "        \"train\": \"/opt/ml/input/data/train\"\n",
      "    },\n",
      "    \"current_host\": \"algo-1\",\n",
      "    \"framework_module\": \"sagemaker_pytorch_container.training:main\",\n",
      "    \"hosts\": [\n",
      "        \"algo-1\"\n",
      "    ],\n",
      "    \"hyperparameters\": {\n",
      "        \"checkpointfreq\": 2,\n",
      "        \"lr\": 1e-05,\n",
      "        \"batch\": 8,\n",
      "        \"trainfile\": \"train.csv\",\n",
      "        \"gradaccumulation\": 4,\n",
      "        \"datasetfactory\": \"datasets.bc3ast_dataset_factory.BC3ASTDatasetFactory\",\n",
      "        \"finetune\": 0,\n",
      "        \"log-level\": \"INFO\",\n",
      "        \"maxseqlen\": 512,\n",
      "        \"valfile\": \"dev.csv\",\n",
      "        \"epochs\": 30,\n",
      "        \"earlystoppingpatience\": 3,\n",
      "        \"checkpointdir\": \"/opt/ml/checkpoints/\"\n",
      "    },\n",
      "    \"input_config_dir\": \"/opt/ml/input/config\",\n",
      "    \"input_data_config\": {\n",
      "        \"val\": {\n",
      "            \"TrainingInputMode\": \"File\",\n",
      "            \"S3DistributionType\": \"FullyReplicated\",\n",
      "            \"RecordWrapperType\": \"None\"\n",
      "        },\n",
      "        \"train\": {\n",
      "            \"TrainingInputMode\": \"File\",\n",
      "            \"S3DistributionType\": \"FullyReplicated\",\n",
      "            \"RecordWrapperType\": \"None\"\n",
      "        }\n",
      "    },\n",
      "    \"input_dir\": \"/opt/ml/input\",\n",
      "    \"is_master\": true,\n",
      "    \"job_name\": \"bert-sst2-classification-2021-01-20-09-27-26-442\",\n",
      "    \"log_level\": 20,\n",
      "    \"master_hostname\": \"algo-1\",\n",
      "    \"model_dir\": \"/opt/ml/model\",\n",
      "    \"module_dir\": \"s3://sagemaker-us-east-2-324346001917/bert-bc3ast-classify/code/bert-sst2-classification-2021-01-20-09-27-26-442/source/sourcedir.tar.gz\",\n",
      "    \"module_name\": \"main\",\n",
      "    \"network_interface_name\": \"eth0\",\n",
      "    \"num_cpus\": 8,\n",
      "    \"num_gpus\": 1,\n",
      "    \"output_data_dir\": \"/opt/ml/output/data\",\n",
      "    \"output_dir\": \"/opt/ml/output\",\n",
      "    \"output_intermediate_dir\": \"/opt/ml/output/intermediate\",\n",
      "    \"resource_config\": {\n",
      "        \"current_host\": \"algo-1\",\n",
      "        \"hosts\": [\n",
      "            \"algo-1\"\n",
      "        ],\n",
      "        \"network_interface_name\": \"eth0\"\n",
      "    },\n",
      "    \"user_entry_point\": \"main.py\"\u001b[0m\n",
      "\u001b[34m}\n",
      "\u001b[0m\n",
      "\u001b[34mEnvironment variables:\n",
      "\u001b[0m\n",
      "\u001b[34mSM_HOSTS=[\"algo-1\"]\u001b[0m\n",
      "\u001b[34mSM_NETWORK_INTERFACE_NAME=eth0\u001b[0m\n",
      "\u001b[34mSM_HPS={\"batch\":8,\"checkpointdir\":\"/opt/ml/checkpoints/\",\"checkpointfreq\":2,\"datasetfactory\":\"datasets.bc3ast_dataset_factory.BC3ASTDatasetFactory\",\"earlystoppingpatience\":3,\"epochs\":30,\"finetune\":0,\"gradaccumulation\":4,\"log-level\":\"INFO\",\"lr\":1e-05,\"maxseqlen\":512,\"trainfile\":\"train.csv\",\"valfile\":\"dev.csv\"}\u001b[0m\n",
      "\u001b[34mSM_USER_ENTRY_POINT=main.py\u001b[0m\n",
      "\u001b[34mSM_FRAMEWORK_PARAMS={}\u001b[0m\n",
      "\u001b[34mSM_RESOURCE_CONFIG={\"current_host\":\"algo-1\",\"hosts\":[\"algo-1\"],\"network_interface_name\":\"eth0\"}\u001b[0m\n",
      "\u001b[34mSM_INPUT_DATA_CONFIG={\"train\":{\"RecordWrapperType\":\"None\",\"S3DistributionType\":\"FullyReplicated\",\"TrainingInputMode\":\"File\"},\"val\":{\"RecordWrapperType\":\"None\",\"S3DistributionType\":\"FullyReplicated\",\"TrainingInputMode\":\"File\"}}\u001b[0m\n",
      "\u001b[34mSM_OUTPUT_DATA_DIR=/opt/ml/output/data\u001b[0m\n",
      "\u001b[34mSM_CHANNELS=[\"train\",\"val\"]\u001b[0m\n",
      "\u001b[34mSM_CURRENT_HOST=algo-1\u001b[0m\n",
      "\u001b[34mSM_MODULE_NAME=main\u001b[0m\n",
      "\u001b[34mSM_LOG_LEVEL=20\u001b[0m\n",
      "\u001b[34mSM_FRAMEWORK_MODULE=sagemaker_pytorch_container.training:main\u001b[0m\n",
      "\u001b[34mSM_INPUT_DIR=/opt/ml/input\u001b[0m\n",
      "\u001b[34mSM_INPUT_CONFIG_DIR=/opt/ml/input/config\u001b[0m\n",
      "\u001b[34mSM_OUTPUT_DIR=/opt/ml/output\u001b[0m\n",
      "\u001b[34mSM_NUM_CPUS=8\u001b[0m\n",
      "\u001b[34mSM_NUM_GPUS=1\u001b[0m\n",
      "\u001b[34mSM_MODEL_DIR=/opt/ml/model\u001b[0m\n",
      "\u001b[34mSM_MODULE_DIR=s3://sagemaker-us-east-2-324346001917/bert-bc3ast-classify/code/bert-sst2-classification-2021-01-20-09-27-26-442/source/sourcedir.tar.gz\u001b[0m\n",
      "\u001b[34mSM_TRAINING_ENV={\"additional_framework_parameters\":{},\"channel_input_dirs\":{\"train\":\"/opt/ml/input/data/train\",\"val\":\"/opt/ml/input/data/val\"},\"current_host\":\"algo-1\",\"framework_module\":\"sagemaker_pytorch_container.training:main\",\"hosts\":[\"algo-1\"],\"hyperparameters\":{\"batch\":8,\"checkpointdir\":\"/opt/ml/checkpoints/\",\"checkpointfreq\":2,\"datasetfactory\":\"datasets.bc3ast_dataset_factory.BC3ASTDatasetFactory\",\"earlystoppingpatience\":3,\"epochs\":30,\"finetune\":0,\"gradaccumulation\":4,\"log-level\":\"INFO\",\"lr\":1e-05,\"maxseqlen\":512,\"trainfile\":\"train.csv\",\"valfile\":\"dev.csv\"},\"input_config_dir\":\"/opt/ml/input/config\",\"input_data_config\":{\"train\":{\"RecordWrapperType\":\"None\",\"S3DistributionType\":\"FullyReplicated\",\"TrainingInputMode\":\"File\"},\"val\":{\"RecordWrapperType\":\"None\",\"S3DistributionType\":\"FullyReplicated\",\"TrainingInputMode\":\"File\"}},\"input_dir\":\"/opt/ml/input\",\"is_master\":true,\"job_name\":\"bert-sst2-classification-2021-01-20-09-27-26-442\",\"log_level\":20,\"master_hostname\":\"algo-1\",\"model_dir\":\"/opt/ml/model\",\"module_dir\":\"s3://sagemaker-us-east-2-324346001917/bert-bc3ast-classify/code/bert-sst2-classification-2021-01-20-09-27-26-442/source/sourcedir.tar.gz\",\"module_name\":\"main\",\"network_interface_name\":\"eth0\",\"num_cpus\":8,\"num_gpus\":1,\"output_data_dir\":\"/opt/ml/output/data\",\"output_dir\":\"/opt/ml/output\",\"output_intermediate_dir\":\"/opt/ml/output/intermediate\",\"resource_config\":{\"current_host\":\"algo-1\",\"hosts\":[\"algo-1\"],\"network_interface_name\":\"eth0\"},\"user_entry_point\":\"main.py\"}\u001b[0m\n",
      "\u001b[34mSM_USER_ARGS=[\"--batch\",\"8\",\"--checkpointdir\",\"/opt/ml/checkpoints/\",\"--checkpointfreq\",\"2\",\"--datasetfactory\",\"datasets.bc3ast_dataset_factory.BC3ASTDatasetFactory\",\"--earlystoppingpatience\",\"3\",\"--epochs\",\"30\",\"--finetune\",\"0\",\"--gradaccumulation\",\"4\",\"--log-level\",\"INFO\",\"--lr\",\"1e-05\",\"--maxseqlen\",\"512\",\"--trainfile\",\"train.csv\",\"--valfile\",\"dev.csv\"]\u001b[0m\n",
      "\u001b[34mSM_OUTPUT_INTERMEDIATE_DIR=/opt/ml/output/intermediate\u001b[0m\n",
      "\u001b[34mSM_CHANNEL_VAL=/opt/ml/input/data/val\u001b[0m\n",
      "\u001b[34mSM_CHANNEL_TRAIN=/opt/ml/input/data/train\u001b[0m\n",
      "\u001b[34mSM_HP_CHECKPOINTFREQ=2\u001b[0m\n",
      "\u001b[34mSM_HP_LR=1e-05\u001b[0m\n",
      "\u001b[34mSM_HP_BATCH=8\u001b[0m\n",
      "\u001b[34mSM_HP_TRAINFILE=train.csv\u001b[0m\n",
      "\u001b[34mSM_HP_GRADACCUMULATION=4\u001b[0m\n",
      "\u001b[34mSM_HP_DATASETFACTORY=datasets.bc3ast_dataset_factory.BC3ASTDatasetFactory\u001b[0m\n",
      "\u001b[34mSM_HP_FINETUNE=0\u001b[0m\n",
      "\u001b[34mSM_HP_LOG-LEVEL=INFO\u001b[0m\n",
      "\u001b[34mSM_HP_MAXSEQLEN=512\u001b[0m\n",
      "\u001b[34mSM_HP_VALFILE=dev.csv\u001b[0m\n",
      "\u001b[34mSM_HP_EPOCHS=30\u001b[0m\n",
      "\u001b[34mSM_HP_EARLYSTOPPINGPATIENCE=3\u001b[0m\n",
      "\u001b[34mSM_HP_CHECKPOINTDIR=/opt/ml/checkpoints/\u001b[0m\n",
      "\u001b[34mPYTHONPATH=/opt/ml/code:/opt/conda/bin:/opt/conda/lib/python36.zip:/opt/conda/lib/python3.6:/opt/conda/lib/python3.6/lib-dynload:/opt/conda/lib/python3.6/site-packages\n",
      "\u001b[0m\n",
      "\u001b[34mInvoking script with the following command:\n",
      "\u001b[0m\n",
      "\u001b[34m/opt/conda/bin/python main.py --batch 8 --checkpointdir /opt/ml/checkpoints/ --checkpointfreq 2 --datasetfactory datasets.bc3ast_dataset_factory.BC3ASTDatasetFactory --earlystoppingpatience 3 --epochs 30 --finetune 0 --gradaccumulation 4 --log-level INFO --lr 1e-05 --maxseqlen 512 --trainfile train.csv --valfile dev.csv\n",
      "\n",
      "\u001b[0m\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[34m{'trainfile': 'train.csv', 'traindir': '/opt/ml/input/data/train', 'valfile': 'dev.csv', 'valdir': '/opt/ml/input/data/val', 'datasetfactory': 'datasets.bc3ast_dataset_factory.BC3ASTDatasetFactory', 'outdir': '/opt/ml/output/data', 'modeldir': '/opt/ml/model', 'checkpointdir': '/opt/ml/checkpoints/', 'checkpointfreq': '2', 'earlystoppingpatience': 3, 'epochs': 30, 'gradaccumulation': 4, 'batch': 8, 'lr': 1e-05, 'finetune': 0, 'maxseqlen': 512, 'log_level': 'INFO'}\u001b[0m\n",
      "\u001b[34m2021-01-20 09:31:50,435 - builder - INFO - Retrieving Tokeniser\u001b[0m\n",
      "\u001b[34m2021-01-20 09:31:50,518 - transformers.file_utils - INFO - https://s3.amazonaws.com/models.huggingface.co/bert/bert-base-cased-vocab.txt not found in cache or force_download set to True, downloading to /tmp/tmpgor2xbls\u001b[0m\n",
      "\u001b[34m2021-01-20 09:31:50,636 - transformers.file_utils - INFO - copying /tmp/tmpgor2xbls to cache at /root/.cache/torch/transformers/5e8a2b4893d13790ed4150ca1906be5f7a03d6c4ddf62296c383f6db42814db2.e13dbb970cb325137104fb2e5f36fe865f27746c6b526f6352861b1980eb80b1\u001b[0m\n",
      "\u001b[34m2021-01-20 09:31:50,637 - transformers.file_utils - INFO - creating metadata file for /root/.cache/torch/transformers/5e8a2b4893d13790ed4150ca1906be5f7a03d6c4ddf62296c383f6db42814db2.e13dbb970cb325137104fb2e5f36fe865f27746c6b526f6352861b1980eb80b1\u001b[0m\n",
      "\u001b[34m2021-01-20 09:31:50,637 - transformers.file_utils - INFO - removing temp file /tmp/tmpgor2xbls\u001b[0m\n",
      "\u001b[34m2021-01-20 09:31:50,638 - transformers.tokenization_utils - INFO - loading file https://s3.amazonaws.com/models.huggingface.co/bert/bert-base-cased-vocab.txt from cache at /root/.cache/torch/transformers/5e8a2b4893d13790ed4150ca1906be5f7a03d6c4ddf62296c383f6db42814db2.e13dbb970cb325137104fb2e5f36fe865f27746c6b526f6352861b1980eb80b1\u001b[0m\n",
      "\u001b[34m2021-01-20 09:31:50,724 - builder - INFO - Completed retrieving Tokeniser\u001b[0m\n",
      "\u001b[34m2021-01-20 09:31:50,738 - builder - INFO - Retrieving Tokeniser\u001b[0m\n",
      "\u001b[34m2021-01-20 09:31:50,738 - builder - INFO - Completed retrieving Tokeniser\u001b[0m\n",
      "\u001b[34m2021-01-20 09:31:50,738 - datasets.bc3ast_dataset - INFO - loading /opt/ml/input/data/train/train.csv\u001b[0m\n",
      "\u001b[34m2021-01-20 09:31:50,760 - datasets.bc3ast_dataset - INFO - Loaded file /opt/ml/input/data/train/train.csv with 1824 records Counter({'0': 912, '1': 912})\u001b[0m\n",
      "\u001b[34m2021-01-20 09:31:50,760 - builder - INFO - Retrieving Tokeniser\u001b[0m\n",
      "\u001b[34m2021-01-20 09:31:50,760 - builder - INFO - Completed retrieving Tokeniser\u001b[0m\n",
      "\u001b[34m2021-01-20 09:31:50,760 - datasets.bc3ast_dataset - INFO - loading /opt/ml/input/data/val/dev.csv\u001b[0m\n",
      "\u001b[34m2021-01-20 09:31:50,766 - datasets.bc3ast_dataset - INFO - Loaded file /opt/ml/input/data/val/dev.csv with 456 records Counter({'1': 228, '0': 228})\u001b[0m\n",
      "\u001b[34m2021-01-20 09:31:50,766 - builder - INFO - Retrieving model\u001b[0m\n",
      "\u001b[34m2021-01-20 09:31:50,837 - transformers.file_utils - INFO - https://s3.amazonaws.com/models.huggingface.co/bert/bert-base-cased-config.json not found in cache or force_download set to True, downloading to /tmp/tmpvh09kt98\u001b[0m\n",
      "\u001b[34m2021-01-20 09:31:50,919 - transformers.file_utils - INFO - copying /tmp/tmpvh09kt98 to cache at /root/.cache/torch/transformers/b945b69218e98b3e2c95acf911789741307dec43c698d35fad11c1ae28bda352.9da767be51e1327499df13488672789394e2ca38b877837e52618a67d7002391\u001b[0m\n",
      "\u001b[34m2021-01-20 09:31:50,919 - transformers.file_utils - INFO - creating metadata file for /root/.cache/torch/transformers/b945b69218e98b3e2c95acf911789741307dec43c698d35fad11c1ae28bda352.9da767be51e1327499df13488672789394e2ca38b877837e52618a67d7002391\u001b[0m\n",
      "\u001b[34m2021-01-20 09:31:50,920 - transformers.file_utils - INFO - removing temp file /tmp/tmpvh09kt98\u001b[0m\n",
      "\u001b[34m2021-01-20 09:31:50,920 - transformers.configuration_utils - INFO - loading configuration file https://s3.amazonaws.com/models.huggingface.co/bert/bert-base-cased-config.json from cache at /root/.cache/torch/transformers/b945b69218e98b3e2c95acf911789741307dec43c698d35fad11c1ae28bda352.9da767be51e1327499df13488672789394e2ca38b877837e52618a67d7002391\u001b[0m\n",
      "\u001b[34m2021-01-20 09:31:50,920 - transformers.configuration_utils - INFO - Model config {\n",
      "  \"architectures\": [\n",
      "    \"BertForMaskedLM\"\n",
      "  ],\n",
      "  \"attention_probs_dropout_prob\": 0.1,\n",
      "  \"finetuning_task\": null,\n",
      "  \"hidden_act\": \"gelu\",\n",
      "  \"hidden_dropout_prob\": 0.1,\n",
      "  \"hidden_size\": 768,\n",
      "  \"id2label\": {\n",
      "    \"0\": \"LABEL_0\",\n",
      "    \"1\": \"LABEL_1\"\n",
      "  },\n",
      "  \"initializer_range\": 0.02,\n",
      "  \"intermediate_size\": 3072,\n",
      "  \"is_decoder\": false,\n",
      "  \"label2id\": {\n",
      "    \"LABEL_0\": 0,\n",
      "    \"LABEL_1\": 1\n",
      "  },\n",
      "  \"layer_norm_eps\": 1e-12,\n",
      "  \"max_position_embeddings\": 512,\n",
      "  \"model_type\": \"bert\",\n",
      "  \"num_attention_heads\": 12,\n",
      "  \"num_hidden_layers\": 12,\n",
      "  \"num_labels\": 2,\n",
      "  \"output_attentions\": false,\n",
      "  \"output_hidden_states\": false,\n",
      "  \"output_past\": true,\n",
      "  \"pad_token_id\": 0,\n",
      "  \"pruned_heads\": {},\n",
      "  \"torchscript\": false,\n",
      "  \"type_vocab_size\": 2,\n",
      "  \"use_bfloat16\": false,\n",
      "  \"vocab_size\": 28996\u001b[0m\n",
      "\u001b[34m}\n",
      "\u001b[0m\n",
      "\u001b[34m2021-01-20 09:31:50,996 - transformers.file_utils - INFO - https://s3.amazonaws.com/models.huggingface.co/bert/bert-base-cased-pytorch_model.bin not found in cache or force_download set to True, downloading to /tmp/tmp0m8hwkga\u001b[0m\n",
      "\u001b[34m2021-01-20 09:31:59,314 - transformers.file_utils - INFO - copying /tmp/tmp0m8hwkga to cache at /root/.cache/torch/transformers/35d8b9d36faaf46728a0192d82bf7d00137490cd6074e8500778afed552a67e5.3fadbea36527ae472139fe84cddaa65454d7429f12d543d80bfc3ad70de55ac2\u001b[0m\n",
      "\u001b[34m2021-01-20 09:31:59,811 - transformers.file_utils - INFO - creating metadata file for /root/.cache/torch/transformers/35d8b9d36faaf46728a0192d82bf7d00137490cd6074e8500778afed552a67e5.3fadbea36527ae472139fe84cddaa65454d7429f12d543d80bfc3ad70de55ac2\u001b[0m\n",
      "\u001b[34m2021-01-20 09:31:59,811 - transformers.file_utils - INFO - removing temp file /tmp/tmp0m8hwkga\u001b[0m\n",
      "\u001b[34m2021-01-20 09:31:59,880 - transformers.modeling_utils - INFO - loading weights file https://s3.amazonaws.com/models.huggingface.co/bert/bert-base-cased-pytorch_model.bin from cache at /root/.cache/torch/transformers/35d8b9d36faaf46728a0192d82bf7d00137490cd6074e8500778afed552a67e5.3fadbea36527ae472139fe84cddaa65454d7429f12d543d80bfc3ad70de55ac2\u001b[0m\n",
      "\u001b[34m2021-01-20 09:32:03,170 - transformers.modeling_utils - INFO - Weights of BertForSequenceClassification not initialized from pretrained model: ['classifier.weight', 'classifier.bias']\u001b[0m\n",
      "\u001b[34m2021-01-20 09:32:03,171 - transformers.modeling_utils - INFO - Weights from pretrained model not used in BertForSequenceClassification: ['cls.predictions.bias', 'cls.predictions.transform.dense.weight', 'cls.predictions.transform.dense.bias', 'cls.predictions.decoder.weight', 'cls.seq_relationship.weight', 'cls.seq_relationship.bias', 'cls.predictions.transform.LayerNorm.weight', 'cls.predictions.transform.LayerNorm.bias']\u001b[0m\n",
      "\u001b[34m2021-01-20 09:32:03,172 - builder - INFO - Retrieving model complete\u001b[0m\n",
      "\u001b[34m2021-01-20 09:33:11,226 - bert_train - INFO - Train set result details:\u001b[0m\n",
      "\u001b[34m2021-01-20 09:33:11,231 - bert_train - INFO - Train set result details: 0.5608552631578947\u001b[0m\n",
      "\u001b[34m2021-01-20 09:33:11,231 - bert_train - INFO - Validation set result details:\u001b[0m\n",
      "\u001b[34m2021-01-20 09:33:16,639 - bert_train - INFO - Validation set result details: 0.6140350877192983 \u001b[0m\n",
      "\u001b[34m2021-01-20 09:33:16,640 - bert_train - INFO - Snapshotting because the current score 0.6140350877192983 is greater than None \u001b[0m\n",
      "\u001b[34m2021-01-20 09:33:16,640 - bert_train - INFO - Snapshot model to /opt/ml/model/best_snaphsotmodel.pt\u001b[0m\n",
      "\u001b[34m2021-01-20 09:33:17,130 - bert_train - INFO - Checkpoint model to /opt/ml/checkpoints/checkpoint.pt\u001b[0m\n",
      "\u001b[34m2021-01-20 09:33:17,552 - bert_train - INFO - Run     74     0       228     9/228         4% 0.085485 0.082031       0.5609       0.6140\u001b[0m\n",
      "\u001b[34m###score: train_loss### 0.08548496242024396\u001b[0m\n",
      "\u001b[34m###score: val_loss### 0.08203068454014628\u001b[0m\n",
      "\u001b[34m###score: train_score### 0.5608552631578947\u001b[0m\n",
      "\u001b[34m###score: val_score### 0.6140350877192983\u001b[0m\n",
      "\u001b[34m2021-01-20 09:34:19,325 - bert_train - INFO - Train set result details:\u001b[0m\n",
      "\u001b[34m2021-01-20 09:34:19,328 - bert_train - INFO - Train set result details: 0.7357456140350878\u001b[0m\n",
      "\u001b[34m2021-01-20 09:34:19,328 - bert_train - INFO - Validation set result details:\u001b[0m\n",
      "\u001b[34m2021-01-20 09:34:24,691 - bert_train - INFO - Validation set result details: 0.7850877192982456 \u001b[0m\n",
      "\u001b[34m2021-01-20 09:34:24,692 - bert_train - INFO - Snapshotting because the current score 0.7850877192982456 is greater than 0.6140350877192983 \u001b[0m\n",
      "\u001b[34m2021-01-20 09:34:24,692 - bert_train - INFO - Snapshot model to /opt/ml/model/best_snaphsotmodel.pt\u001b[0m\n",
      "\u001b[34m2021-01-20 09:34:25,270 - bert_train - INFO - Checkpoint model to /opt/ml/checkpoints/checkpoint.pt\u001b[0m\n",
      "\u001b[34m2021-01-20 09:34:25,823 - bert_train - INFO - Run    142     1       456     9/228         4% 0.070161 0.059670       0.7357       0.7851\u001b[0m\n",
      "\u001b[34m###score: train_loss### 0.0701612898785817\u001b[0m\n",
      "\u001b[34m###score: val_loss### 0.05967047143923609\u001b[0m\n",
      "\u001b[34m###score: train_score### 0.7357456140350878\u001b[0m\n",
      "\u001b[34m###score: val_score### 0.7850877192982456\u001b[0m\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[34m2021-01-20 09:35:27,935 - bert_train - INFO - Train set result details:\u001b[0m\n",
      "\u001b[34m2021-01-20 09:35:27,937 - bert_train - INFO - Train set result details: 0.831140350877193\u001b[0m\n",
      "\u001b[34m2021-01-20 09:35:27,937 - bert_train - INFO - Validation set result details:\u001b[0m\n",
      "\u001b[34m2021-01-20 09:35:33,316 - bert_train - INFO - Validation set result details: 0.8157894736842105 \u001b[0m\n",
      "\u001b[34m2021-01-20 09:35:33,316 - bert_train - INFO - Snapshotting because the current score 0.8157894736842105 is greater than 0.7850877192982456 \u001b[0m\n",
      "\u001b[34m2021-01-20 09:35:33,316 - bert_train - INFO - Snapshot model to /opt/ml/model/best_snaphsotmodel.pt\u001b[0m\n",
      "\u001b[34m2021-01-20 09:35:33,890 - bert_train - INFO - Checkpoint model to /opt/ml/checkpoints/checkpoint.pt\u001b[0m\n",
      "\u001b[34m2021-01-20 09:35:34,440 - bert_train - INFO - Run    211     2       684     9/228         4% 0.051172 0.052492       0.8311       0.8158\u001b[0m\n",
      "\u001b[34m###score: train_loss### 0.05117241614253113\u001b[0m\n",
      "\u001b[34m###score: val_loss### 0.05249242824420594\u001b[0m\n",
      "\u001b[34m###score: train_score### 0.831140350877193\u001b[0m\n",
      "\u001b[34m###score: val_score### 0.8157894736842105\u001b[0m\n",
      "\u001b[34m2021-01-20 09:36:36,844 - bert_train - INFO - Train set result details:\u001b[0m\n",
      "\u001b[34m2021-01-20 09:36:36,846 - bert_train - INFO - Train set result details: 0.8618421052631579\u001b[0m\n",
      "\u001b[34m2021-01-20 09:36:36,846 - bert_train - INFO - Validation set result details:\u001b[0m\n",
      "\u001b[34m2021-01-20 09:36:42,261 - bert_train - INFO - Validation set result details: 0.8004385964912281 \u001b[0m\n",
      "\u001b[34m2021-01-20 09:36:42,261 - bert_train - INFO - Checkpoint model to /opt/ml/checkpoints/checkpoint.pt\u001b[0m\n",
      "\u001b[34m2021-01-20 09:36:42,793 - bert_train - INFO - Run    279     3       912     9/228         4% 0.042978 0.056773       0.8618       0.8004\u001b[0m\n",
      "\u001b[34m###score: train_loss### 0.042977805780410244\u001b[0m\n",
      "\u001b[34m###score: val_loss### 0.05677278318622133\u001b[0m\n",
      "\u001b[34m###score: train_score### 0.8618421052631579\u001b[0m\n",
      "\u001b[34m###score: val_score### 0.8004385964912281\u001b[0m\n",
      "\u001b[34m2021-01-20 09:37:45,456 - bert_train - INFO - Train set result details:\u001b[0m\n",
      "\u001b[34m2021-01-20 09:37:45,459 - bert_train - INFO - Train set result details: 0.8887061403508771\u001b[0m\n",
      "\u001b[34m2021-01-20 09:37:45,459 - bert_train - INFO - Validation set result details:\u001b[0m\n",
      "\u001b[34m2021-01-20 09:37:50,863 - bert_train - INFO - Validation set result details: 0.8640350877192983 \u001b[0m\n",
      "\u001b[34m2021-01-20 09:37:50,863 - bert_train - INFO - Snapshotting because the current score 0.8640350877192983 is greater than 0.8157894736842105 \u001b[0m\n",
      "\u001b[34m2021-01-20 09:37:50,863 - bert_train - INFO - Snapshot model to /opt/ml/model/best_snaphsotmodel.pt\u001b[0m\n",
      "\u001b[34m2021-01-20 09:37:51,452 - bert_train - INFO - Checkpoint model to /opt/ml/checkpoints/checkpoint.pt\u001b[0m\n",
      "\u001b[34m2021-01-20 09:37:52,094 - bert_train - INFO - Run    348     4      1140     9/228         4% 0.035088 0.042852       0.8887       0.8640\u001b[0m\n",
      "\u001b[34m###score: train_loss### 0.03508794521749543\u001b[0m\n",
      "\u001b[34m###score: val_loss### 0.0428517754271365\u001b[0m\n",
      "\u001b[34m###score: train_score### 0.8887061403508771\u001b[0m\n",
      "\u001b[34m###score: val_score### 0.8640350877192983\u001b[0m\n",
      "\u001b[34m2021-01-20 09:38:54,972 - bert_train - INFO - Train set result details:\u001b[0m\n",
      "\u001b[34m2021-01-20 09:38:54,974 - bert_train - INFO - Train set result details: 0.9106359649122807\u001b[0m\n",
      "\u001b[34m2021-01-20 09:38:54,974 - bert_train - INFO - Validation set result details:\u001b[0m\n",
      "\u001b[34m2021-01-20 09:39:00,394 - bert_train - INFO - Validation set result details: 0.8333333333333334 \u001b[0m\n",
      "\u001b[34m2021-01-20 09:39:00,395 - bert_train - INFO - Checkpoint model to /opt/ml/checkpoints/checkpoint.pt\u001b[0m\n",
      "\u001b[34m2021-01-20 09:39:00,929 - bert_train - INFO - Run    417     5      1368     9/228         4% 0.029005 0.052126       0.9106       0.8333\u001b[0m\n",
      "\u001b[34m###score: train_loss### 0.02900508189524867\u001b[0m\n",
      "\u001b[34m###score: val_loss### 0.05212556137785054\u001b[0m\n",
      "\u001b[34m###score: train_score### 0.9106359649122807\u001b[0m\n",
      "\u001b[34m###score: val_score### 0.8333333333333334\u001b[0m\n",
      "\u001b[34m2021-01-20 09:40:04,021 - bert_train - INFO - Train set result details:\u001b[0m\n",
      "\u001b[34m2021-01-20 09:40:04,023 - bert_train - INFO - Train set result details: 0.9391447368421053\u001b[0m\n",
      "\u001b[34m2021-01-20 09:40:04,023 - bert_train - INFO - Validation set result details:\u001b[0m\n",
      "\u001b[34m2021-01-20 09:40:09,425 - bert_train - INFO - Validation set result details: 0.8596491228070176 \u001b[0m\n",
      "\u001b[34m2021-01-20 09:40:09,425 - bert_train - INFO - Checkpoint model to /opt/ml/checkpoints/checkpoint.pt\u001b[0m\n",
      "\u001b[34m2021-01-20 09:40:09,955 - bert_train - INFO - Run    486     6      1596     9/228         4% 0.022262 0.042166       0.9391       0.8596\u001b[0m\n",
      "\u001b[34m###score: train_loss### 0.02226239123806488\u001b[0m\n",
      "\u001b[34m###score: val_loss### 0.04216620591550804\u001b[0m\n",
      "\u001b[34m###score: train_score### 0.9391447368421053\u001b[0m\n",
      "\u001b[34m###score: val_score### 0.8596491228070176\u001b[0m\n",
      "\u001b[34m2021-01-20 09:41:13,364 - bert_train - INFO - Train set result details:\u001b[0m\n",
      "\u001b[34m2021-01-20 09:41:13,366 - bert_train - INFO - Train set result details: 0.9533991228070176\u001b[0m\n",
      "\u001b[34m2021-01-20 09:41:13,366 - bert_train - INFO - Validation set result details:\u001b[0m\n",
      "\u001b[34m2021-01-20 09:41:18,795 - bert_train - INFO - Validation set result details: 0.8640350877192983 \u001b[0m\n",
      "\u001b[34m2021-01-20 09:41:18,796 - bert_train - INFO - Checkpoint model to /opt/ml/checkpoints/checkpoint.pt\u001b[0m\n",
      "\u001b[34m2021-01-20 09:41:19,336 - bert_train - INFO - Run    556     7      1824     9/228         4% 0.018372 0.046388       0.9534       0.8640\u001b[0m\n",
      "\u001b[34m###score: train_loss### 0.018372292315848825\u001b[0m\n",
      "\u001b[34m###score: val_loss### 0.04638764031819607\u001b[0m\n",
      "\u001b[34m###score: train_score### 0.9533991228070176\u001b[0m\n",
      "\u001b[34m###score: val_score### 0.8640350877192983\u001b[0m\n",
      "\u001b[34m2021-01-20 09:42:22,411 - bert_train - INFO - Train set result details:\u001b[0m\n",
      "\u001b[34m2021-01-20 09:42:22,413 - bert_train - INFO - Train set result details: 0.9682017543859649\u001b[0m\n",
      "\u001b[34m2021-01-20 09:42:22,413 - bert_train - INFO - Validation set result details:\u001b[0m\n",
      "\u001b[34m2021-01-20 09:42:27,796 - bert_train - INFO - Validation set result details: 0.875 \u001b[0m\n",
      "\u001b[34m2021-01-20 09:42:27,797 - bert_train - INFO - Snapshotting because the current score 0.875 is greater than 0.8640350877192983 \u001b[0m\n",
      "\u001b[34m2021-01-20 09:42:27,797 - bert_train - INFO - Snapshot model to /opt/ml/model/best_snaphsotmodel.pt\u001b[0m\n",
      "\u001b[34m2021-01-20 09:42:28,375 - bert_train - INFO - Checkpoint model to /opt/ml/checkpoints/checkpoint.pt\u001b[0m\n",
      "\u001b[34m2021-01-20 09:42:29,014 - bert_train - INFO - Run    625     8      2052     9/228         4% 0.013735 0.045747       0.9682       0.8750\u001b[0m\n",
      "\u001b[34m###score: train_loss### 0.013734728592170174\u001b[0m\n",
      "\u001b[34m###score: val_loss### 0.0457466386683416\u001b[0m\n",
      "\u001b[34m###score: train_score### 0.9682017543859649\u001b[0m\n",
      "\u001b[34m###score: val_score### 0.875\u001b[0m\n",
      "\u001b[34m2021-01-20 09:43:32,091 - bert_train - INFO - Train set result details:\u001b[0m\n",
      "\u001b[34m2021-01-20 09:43:32,093 - bert_train - INFO - Train set result details: 0.9791666666666666\u001b[0m\n",
      "\u001b[34m2021-01-20 09:43:32,093 - bert_train - INFO - Validation set result details:\u001b[0m\n",
      "\u001b[34m2021-01-20 09:43:37,529 - bert_train - INFO - Validation set result details: 0.8771929824561403 \u001b[0m\n",
      "\u001b[34m2021-01-20 09:43:37,529 - bert_train - INFO - Snapshotting because the current score 0.8771929824561403 is greater than 0.875 \u001b[0m\n",
      "\u001b[34m2021-01-20 09:43:37,529 - bert_train - INFO - Snapshot model to /opt/ml/model/best_snaphsotmodel.pt\u001b[0m\n",
      "\u001b[34m2021-01-20 09:43:38,103 - bert_train - INFO - Checkpoint model to /opt/ml/checkpoints/checkpoint.pt\u001b[0m\n",
      "\u001b[34m2021-01-20 09:43:38,657 - bert_train - INFO - Run    695     9      2280     9/228         4% 0.009500 0.048376       0.9792       0.8772\u001b[0m\n",
      "\u001b[34m###score: train_loss### 0.009500106132850704\u001b[0m\n",
      "\u001b[34m###score: val_loss### 0.04837599267627586\u001b[0m\n",
      "\u001b[34m###score: train_score### 0.9791666666666666\u001b[0m\n",
      "\u001b[34m###score: val_score### 0.8771929824561403\u001b[0m\n",
      "\u001b[34m2021-01-20 09:44:41,666 - bert_train - INFO - Train set result details:\u001b[0m\n",
      "\u001b[34m2021-01-20 09:44:41,668 - bert_train - INFO - Train set result details: 0.9841008771929824\u001b[0m\n",
      "\u001b[34m2021-01-20 09:44:41,668 - bert_train - INFO - Validation set result details:\u001b[0m\n",
      "\u001b[34m2021-01-20 09:44:47,097 - bert_train - INFO - Validation set result details: 0.868421052631579 \u001b[0m\n",
      "\u001b[34m2021-01-20 09:44:47,097 - bert_train - INFO - Checkpoint model to /opt/ml/checkpoints/checkpoint.pt\u001b[0m\n",
      "\u001b[34m2021-01-20 09:44:47,622 - bert_train - INFO - Run    764    10      2508     9/228         4% 0.007580 0.049678       0.9841       0.8684\u001b[0m\n",
      "\u001b[34m###score: train_loss### 0.007579522188997974\u001b[0m\n",
      "\u001b[34m###score: val_loss### 0.04967789708130192\u001b[0m\n",
      "\u001b[34m###score: train_score### 0.9841008771929824\u001b[0m\n",
      "\u001b[34m###score: val_score### 0.868421052631579\u001b[0m\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[34m2021-01-20 09:45:50,566 - bert_train - INFO - Train set result details:\u001b[0m\n",
      "\u001b[34m2021-01-20 09:45:50,568 - bert_train - INFO - Train set result details: 0.9906798245614035\u001b[0m\n",
      "\u001b[34m2021-01-20 09:45:50,568 - bert_train - INFO - Validation set result details:\u001b[0m\n",
      "\u001b[34m2021-01-20 09:45:55,991 - bert_train - INFO - Validation set result details: 0.8706140350877193 \u001b[0m\n",
      "\u001b[34m2021-01-20 09:45:55,991 - bert_train - INFO - Checkpoint model to /opt/ml/checkpoints/checkpoint.pt\u001b[0m\n",
      "\u001b[34m2021-01-20 09:45:56,534 - bert_train - INFO - Run    833    11      2736     9/228         4% 0.005537 0.057184       0.9907       0.8706\u001b[0m\n",
      "\u001b[34m###score: train_loss### 0.00553677411619247\u001b[0m\n",
      "\u001b[34m###score: val_loss### 0.05718375396865763\u001b[0m\n",
      "\u001b[34m###score: train_score### 0.9906798245614035\u001b[0m\n",
      "\u001b[34m###score: val_score### 0.8706140350877193\u001b[0m\n",
      "\u001b[34m2021-01-20 09:46:59,589 - bert_train - INFO - Train set result details:\u001b[0m\n",
      "\u001b[34m2021-01-20 09:46:59,592 - bert_train - INFO - Train set result details: 0.9906798245614035\u001b[0m\n",
      "\u001b[34m2021-01-20 09:46:59,592 - bert_train - INFO - Validation set result details:\u001b[0m\n",
      "\u001b[34m2021-01-20 09:47:04,969 - bert_train - INFO - Validation set result details: 0.8618421052631579 \u001b[0m\n",
      "\u001b[34m2021-01-20 09:47:04,970 - bert_train - INFO - Checkpoint model to /opt/ml/checkpoints/checkpoint.pt\u001b[0m\n",
      "\u001b[34m2021-01-20 09:47:05,527 - bert_train - INFO - Run    902    12      2964     9/228         4% 0.004689 0.058613       0.9907       0.8618\u001b[0m\n",
      "\u001b[34m###score: train_loss### 0.004689004882892365\u001b[0m\n",
      "\u001b[34m###score: val_loss### 0.0586130006663632\u001b[0m\n",
      "\u001b[34m###score: train_score### 0.9906798245614035\u001b[0m\n",
      "\u001b[34m###score: val_score### 0.8618421052631579\u001b[0m\n",
      "\u001b[34m2021-01-20 09:48:08,572 - bert_train - INFO - Train set result details:\u001b[0m\n",
      "\u001b[34m2021-01-20 09:48:08,575 - bert_train - INFO - Train set result details: 0.9939692982456141\u001b[0m\n",
      "\u001b[34m2021-01-20 09:48:08,575 - bert_train - INFO - Validation set result details:\u001b[0m\n",
      "\u001b[34m2021-01-20 09:48:14,014 - bert_train - INFO - Validation set result details: 0.8706140350877193 \u001b[0m\n",
      "\u001b[34m2021-01-20 09:48:14,014 - bert_train - INFO - Checkpoint model to /opt/ml/checkpoints/checkpoint.pt\u001b[0m\n",
      "\u001b[34m2021-01-20 09:48:14,544 - bert_train - INFO - Run    971    13      3192     9/228         4% 0.003807 0.057086       0.9940       0.8706\u001b[0m\n",
      "\u001b[34m###score: train_loss### 0.003806662035027617\u001b[0m\n",
      "\u001b[34m###score: val_loss### 0.05708566285146956\u001b[0m\n",
      "\u001b[34m###score: train_score### 0.9939692982456141\u001b[0m\n",
      "\u001b[34m###score: val_score### 0.8706140350877193\u001b[0m\n",
      "\u001b[34m2021-01-20 09:48:14,544 - bert_train - INFO - Early stopping.. with no improvement in 4\u001b[0m\n",
      "\u001b[34m2021-01-20 09:48:15,237 sagemaker-containers INFO     Reporting training SUCCESS\u001b[0m\n",
      "\n",
      "2021-01-20 09:48:17 Uploading - Uploading generated training model\n",
      "2021-01-20 09:49:15 Completed - Training job completed\n",
      "Training seconds: 1170\n",
      "Billable seconds: 351\n",
      "Managed Spot Training savings: 70.0%\n"
     ]
    }
   ],
   "source": [
    "from sagemaker.pytorch import PyTorch\n",
    "\n",
    "estimator = PyTorch(\n",
    "                    entry_point='main.py',\n",
    "                    source_dir = 'src',\n",
    "                    role=role,\n",
    "                    framework_version =\"1.4.0\",\n",
    "                    py_version='py3',\n",
    "                    instance_count=1,\n",
    "                    instance_type=instance_type,\n",
    "                    hyperparameters = hp,\n",
    "                    output_path=s3_output_path,\n",
    "                    metric_definitions=metric_definitions,\n",
    "                    volume_size=30,\n",
    "                    code_location=s3_code_path,\n",
    "                    debugger_hook_config=False,\n",
    "                    base_job_name =base_name,  \n",
    "                    use_spot_instances = use_spot,\n",
    "                    max_run =  train_max_run_secs,\n",
    "                    max_wait = max_wait_time_secs,   \n",
    "                    checkpoint_s3_uri=s3_checkpoint,\n",
    "                    checkpoint_local_path=sm_localcheckpoint_dir)\n",
    "\n",
    "estimator.fit(inputs, wait=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Deploy BERT model"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Inference container\n",
    "Ideally the server containing should already have all the required dependencies installed to reduce start up time and ensure that the runtime enviornment is consistent. This can be implemented using a custom docker image.\n",
    "\n",
    "But for this demo, to simplify, we will let the Pytorch container script model install the dependencies during start up. As a result, you will see some of the initial ping requests fail, until all dependencies are installed.\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2021-01-20 21:05:28,168 - sagemaker - INFO - Creating model with name: pytorch-inference-2021-01-20-10-05-28-167\n",
      "2021-01-20 21:05:32,255 - sagemaker - INFO - Creating endpoint with name pytorch-inference-2021-01-20-10-05-29-534\n",
      "----------------!"
     ]
    }
   ],
   "source": [
    "from sagemaker.pytorch import PyTorchModel\n",
    "from sagemaker import get_execution_role\n",
    "\n",
    "model_uri = estimator.model_data\n",
    "\n",
    "model = PyTorchModel(model_data=model_uri,\n",
    "                     role=role,\n",
    "                     py_version = \"py3\",\n",
    "                     framework_version='1.4.0',\n",
    "                     entry_point='serve.py',\n",
    "                     source_dir='src')\n",
    "\n",
    "predictor = model.deploy(initial_instance_count=1, instance_type='ml.p3.2xlarge')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Invoke API"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "import json\n",
    "\n",
    "\n",
    "class TextSerDes:\n",
    "    \n",
    "     def serialize(self, x):\n",
    "        data_bytes=\"\\n\".join(x).encode(\"utf-8\")\n",
    "        return data_bytes\n",
    "    \n",
    "     def deserialize(self, x, content_type):\n",
    "        payload =   x.read().decode(\"utf-8\")\n",
    "        return json.loads(payload) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "def chunk_predict(predictor, data, chunk_size=50):\n",
    "    predictor.serializer = TextSerDes()\n",
    "    predictor.deserializer = TextSerDes()\n",
    "    \n",
    "    result = []\n",
    "    for i in range(0, len(data), chunk_size):\n",
    "        \n",
    "        re = predictor.predict(data[i:i+chunk_size],  initial_args={ \"Accept\":\"text/json\", \"ContentType\" : \"text/csv\" })\n",
    "        result.extend(re)\n",
    "    return result"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "from s3_util import S3Util\n",
    "import csv, io\n",
    "\n",
    "def load_test_csv(s3_uri):\n",
    "    os.path.join(processed_out_dir, \"test.csv\")\n",
    "    data = S3Util().download_object(s3_uri).decode(\"utf-8\")\n",
    "    \n",
    "    csv_reader = csv.reader(io.StringIO(data), delimiter='\\t',\n",
    "                            quotechar='\"', quoting=csv.QUOTE_MINIMAL)\n",
    "    \n",
    "    actuals =[]\n",
    "    inputs = []\n",
    "    ids = []\n",
    "    for r in csv_reader:\n",
    "        text = r[0]\n",
    "        label = r[1]\n",
    "        id = r[2]\n",
    "        \n",
    "        inputs.append(text)\n",
    "        actuals.append(label)\n",
    "        ids.append(id)\n",
    "    return inputs, actuals, ids\n",
    "        \n",
    "\n",
    "def write_predictions_csv(test_data, predictions, output_file):\n",
    "    \n",
    "    with open(output_file, \"w\") as f:\n",
    "        csv_writer = csv.writer(f, delimiter='\\t',\n",
    "                            quotechar='\"', quoting=csv.QUOTE_MINIMAL)\n",
    "            \n",
    "        for d,p in zip(test_data, predictions):\n",
    "            csv_writer.writerow([d,p])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "\n",
    "test_data, test_labels, test_ids = load_test_csv(s3_uri_test)\n",
    "response  = chunk_predict(predictor, test_data )\n",
    "predictions_label = [ list(l.keys())[0] for l in response ]\n",
    "\n",
    "write_predictions_csv(test_ids, predictions_label, \"bc3act-output.csv\" )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.8206666666666667"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.metrics import accuracy_score\n",
    "\n",
    "accuracy_score(test_labels, predictions_label)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.5871066768994628"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.metrics import f1_score\n",
    "\n",
    "f1_score(test_labels, predictions_label, pos_label=\"1\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Delete endpoint"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2021-01-20 21:17:38,808 - sagemaker - INFO - Deleting endpoint configuration with name: pytorch-inference-2021-01-20-10-05-29-534\n",
      "2021-01-20 21:17:41,506 - sagemaker - INFO - Deleting endpoint with name: pytorch-inference-2021-01-20-10-05-29-534\n"
     ]
    }
   ],
   "source": [
    "predictor.delete_endpoint()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}