Search Engine Bert-Final.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Collecting pytorch-pretrained-bert\n",
      "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/95/68/84de54aea460eb5b2e90bf47a429aacc1ce97ff052ec40874ea38ae2331d/pytorch_pretrained_bert-0.4.0-py3-none-any.whl (45kB)\n",
      "\u001b[K    100% |████████████████████████████████| 51kB 18.7MB/s ta 0:00:01\n",
      "\u001b[?25hRequirement already satisfied: torch>=0.4.1 in /home/ec2-user/anaconda3/envs/pytorch_p36/lib/python3.6/site-packages (from pytorch-pretrained-bert) (0.4.1)\n",
      "Collecting tqdm (from pytorch-pretrained-bert)\n",
      "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/91/55/8cb23a97301b177e9c8e3226dba45bb454411de2cbd25746763267f226c2/tqdm-4.28.1-py2.py3-none-any.whl (45kB)\n",
      "\u001b[K    100% |████████████████████████████████| 51kB 29.2MB/s ta 0:00:01\n",
      "\u001b[?25hRequirement already satisfied: numpy in /home/ec2-user/anaconda3/envs/pytorch_p36/lib/python3.6/site-packages (from pytorch-pretrained-bert) (1.14.5)\n",
      "Requirement already satisfied: requests in /home/ec2-user/anaconda3/envs/pytorch_p36/lib/python3.6/site-packages (from pytorch-pretrained-bert) (2.20.1)\n",
      "Requirement already satisfied: boto3 in /home/ec2-user/anaconda3/envs/pytorch_p36/lib/python3.6/site-packages (from pytorch-pretrained-bert) (1.9.69)\n",
      "Requirement already satisfied: chardet<3.1.0,>=3.0.2 in /home/ec2-user/anaconda3/envs/pytorch_p36/lib/python3.6/site-packages (from requests->pytorch-pretrained-bert) (3.0.4)\n",
      "Requirement already satisfied: idna<2.8,>=2.5 in /home/ec2-user/anaconda3/envs/pytorch_p36/lib/python3.6/site-packages (from requests->pytorch-pretrained-bert) (2.6)\n",
      "Requirement already satisfied: urllib3<1.25,>=1.21.1 in /home/ec2-user/anaconda3/envs/pytorch_p36/lib/python3.6/site-packages (from requests->pytorch-pretrained-bert) (1.22)\n",
      "Requirement already satisfied: certifi>=2017.4.17 in /home/ec2-user/anaconda3/envs/pytorch_p36/lib/python3.6/site-packages (from requests->pytorch-pretrained-bert) (2018.8.24)\n",
      "Requirement already satisfied: botocore<1.13.0,>=1.12.69 in /home/ec2-user/anaconda3/envs/pytorch_p36/lib/python3.6/site-packages (from boto3->pytorch-pretrained-bert) (1.12.69)\n",
      "Requirement already satisfied: jmespath<1.0.0,>=0.7.1 in /home/ec2-user/anaconda3/envs/pytorch_p36/lib/python3.6/site-packages (from boto3->pytorch-pretrained-bert) (0.9.3)\n",
      "Requirement already satisfied: s3transfer<0.2.0,>=0.1.10 in /home/ec2-user/anaconda3/envs/pytorch_p36/lib/python3.6/site-packages (from boto3->pytorch-pretrained-bert) (0.1.13)\n",
      "Requirement already satisfied: docutils>=0.10 in /home/ec2-user/anaconda3/envs/pytorch_p36/lib/python3.6/site-packages (from botocore<1.13.0,>=1.12.69->boto3->pytorch-pretrained-bert) (0.14)\n",
      "Requirement already satisfied: python-dateutil<3.0.0,>=2.1; python_version >= \"2.7\" in /home/ec2-user/anaconda3/envs/pytorch_p36/lib/python3.6/site-packages (from botocore<1.13.0,>=1.12.69->boto3->pytorch-pretrained-bert) (2.7.3)\n",
      "Requirement already satisfied: six>=1.5 in /home/ec2-user/anaconda3/envs/pytorch_p36/lib/python3.6/site-packages (from python-dateutil<3.0.0,>=2.1; python_version >= \"2.7\"->botocore<1.13.0,>=1.12.69->boto3->pytorch-pretrained-bert) (1.11.0)\n",
      "Installing collected packages: tqdm, pytorch-pretrained-bert\n",
      "Successfully installed pytorch-pretrained-bert-0.4.0 tqdm-4.28.1\n",
      "\u001b[33mYou are using pip version 10.0.1, however version 18.1 is available.\n",
      "You should consider upgrading via the 'pip install --upgrade pip' command.\u001b[0m\n"
     ]
    }
   ],
   "source": [
    " !pip install pytorch-pretrained-bert"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Cloning into 'apex'...\n",
      "remote: Enumerating objects: 116, done.\u001b[K\n",
      "remote: Counting objects: 100% (116/116), done.\u001b[K\n",
      "remote: Compressing objects: 100% (81/81), done.\u001b[K\n",
      "remote: Total 2323 (delta 58), reused 73 (delta 34), pack-reused 2207\u001b[K\n",
      "Receiving objects: 100% (2323/2323), 7.87 MiB | 37.14 MiB/s, done.\n",
      "Resolving deltas: 100% (1415/1415), done.\n"
     ]
    }
   ],
   "source": [
    "! git clone https://github.com/NVIDIA/apex.git"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "g++ (GCC) 4.8.5 20150623 (Red Hat 4.8.5-28)\r\n",
      "Copyright (C) 2015 Free Software Foundation, Inc.\r\n",
      "This is free software; see the source for copying conditions.  There is NO\r\n",
      "warranty; not even for MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.\r\n",
      "\r\n"
     ]
    }
   ],
   "source": [
    "!g++ --version"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "torch.__version__  =  0.4.1\n",
      "running install\n",
      "running bdist_egg\n",
      "running egg_info\n",
      "writing apex.egg-info/PKG-INFO\n",
      "writing dependency_links to apex.egg-info/dependency_links.txt\n",
      "writing top-level names to apex.egg-info/top_level.txt\n",
      "reading manifest file 'apex.egg-info/SOURCES.txt'\n",
      "writing manifest file 'apex.egg-info/SOURCES.txt'\n",
      "installing library code to build/bdist.linux-x86_64/egg\n",
      "running install_lib\n",
      "running build_ext\n",
      "/home/ec2-user/anaconda3/envs/pytorch_p36/lib/python3.6/site-packages/torch/utils/cpp_extension.py:118: UserWarning: \n",
      "\n",
      "                               !! WARNING !!\n",
      "\n",
      "!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!\n",
      "Your compiler (g++ 4.8) may be ABI-incompatible with PyTorch!\n",
      "Please use a compiler that is ABI-compatible with GCC 4.9 and above.\n",
      "See https://gcc.gnu.org/onlinedocs/libstdc++/manual/abi.html.\n",
      "\n",
      "See https://gist.github.com/goldsborough/d466f43e8ffc948ff92de7486c5216d6\n",
      "for instructions on how to install GCC 4.9 or higher.\n",
      "!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!\n",
      "\n",
      "                              !! WARNING !!\n",
      "\n",
      "  warnings.warn(ABI_INCOMPATIBILITY_WARNING.format(compiler))\n",
      "building 'fused_adam_cuda' extension\n",
      "gcc -pthread -B /home/ec2-user/anaconda3/envs/pytorch_p36/compiler_compat -Wl,--sysroot=/ -Wsign-compare -DNDEBUG -g -fwrapv -O3 -Wall -Wstrict-prototypes -fPIC -I/home/ec2-user/anaconda3/envs/pytorch_p36/lib/python3.6/site-packages/torch/lib/include -I/home/ec2-user/anaconda3/envs/pytorch_p36/lib/python3.6/site-packages/torch/lib/include/TH -I/home/ec2-user/anaconda3/envs/pytorch_p36/lib/python3.6/site-packages/torch/lib/include/THC -I/usr/local/cuda-9.2/include -I/home/ec2-user/anaconda3/envs/pytorch_p36/include/python3.6m -c apex/optimizers/csrc/fused_adam_cuda.cpp -o build/temp.linux-x86_64-3.6/apex/optimizers/csrc/fused_adam_cuda.o -O3 -DTORCH_EXTENSION_NAME=fused_adam_cuda -D_GLIBCXX_USE_CXX11_ABI=0 -std=c++11\n",
      "\u001b[01m\u001b[Kgcc:\u001b[m\u001b[K \u001b[01;31m\u001b[Kerror: \u001b[m\u001b[Kapex/optimizers/csrc/fused_adam_cuda.cpp: No such file or directory\n",
      "error: command 'gcc' failed with exit status 1\n"
     ]
    }
   ],
   "source": [
    "#  !python search_data/apex/setup.py install --cuda_ext\n",
    "# import apex"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "import logging\n",
    "import os\n",
    "import argparse\n",
    "import random\n",
    "from tqdm import tqdm, trange\n",
    "import csv\n",
    "\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import torch\n",
    "from torch.utils.data import TensorDataset, DataLoader, RandomSampler, SequentialSampler\n",
    "from torch.utils.data.distributed import DistributedSampler\n",
    "\n",
    "from pytorch_pretrained_bert.tokenization import BertTokenizer\n",
    "from pytorch_pretrained_bert.modeling import BertForMultipleChoice\n",
    "from pytorch_pretrained_bert.optimization import BertAdam\n",
    "from pytorch_pretrained_bert.file_utils import PYTORCH_PRETRAINED_BERT_CACHE\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "header = ['query_id', 'query', 'passage_text', 'label', 'passage_id']\n",
    "test_header = ['query_id', 'query', 'passage_text', 'passage_id']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "# import boto3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# train = pd.read_csv('./search_data/data.tsv',delimiter='\\t',names=header)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "test = pd.read_csv('./search_data/eval2_unlabelled.tsv',delimiter='\\t',names=test_header)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "logging.basicConfig(format = '%(asctime)s - %(levelname)s - %(name)s -   %(message)s',\n",
    "                    datefmt = '%m/%d/%Y %H:%M:%S',\n",
    "                    level = logging.INFO)\n",
    "logger = logging.getLogger(__name__)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "class InputFeatures(object):\n",
    "    def __init__(self,\n",
    "                 example_id,\n",
    "                 choices_features,\n",
    "                 label\n",
    "\n",
    "    ):\n",
    "        self.example_id = example_id\n",
    "        self.choices_features = [\n",
    "            {\n",
    "                'input_ids': input_ids,\n",
    "                'input_mask': input_mask,\n",
    "                'segment_ids': segment_ids\n",
    "            }\n",
    "            for _, input_ids, input_mask, segment_ids in choices_features\n",
    "        ]\n",
    "        self.label = label\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "# sorted_train_data = train.sort_values(['query_id','passage_id'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "# train[train.query_id ==1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# sorted_train_data.to_csv('./data/sorted_train.csv',index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "# sorted_train_data = test[:5587]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "heading_collapsed": true
   },
   "source": [
    "## Test data analysis"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "hidden": true,
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>query_id</th>\n",
       "      <th>query</th>\n",
       "      <th>passage_text</th>\n",
       "      <th>passage_id</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1136966</td>\n",
       "      <td>#ffffff color code</td>\n",
       "      <td>Color hex is a easy to use tool to get the col...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1136966</td>\n",
       "      <td>#ffffff color code</td>\n",
       "      <td>#ffffff Color Conversion. The hexadecimal colo...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1136966</td>\n",
       "      <td>#ffffff color code</td>\n",
       "      <td>CSS Codes; Color Preview; Color Schemes; Color...</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1136966</td>\n",
       "      <td>#ffffff color code</td>\n",
       "      <td>Color Hex Color Codes Color-hex gives informat...</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1136966</td>\n",
       "      <td>#ffffff color code</td>\n",
       "      <td>Color Hex Color Codes. Color-hex gives informa...</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>1136966</td>\n",
       "      <td>#ffffff color code</td>\n",
       "      <td>Color information. #FFFFFF (or 0xFFFFFF) is kn...</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>1136966</td>\n",
       "      <td>#ffffff color code</td>\n",
       "      <td>#ffffff color RGB value is (255,255,255). This...</td>\n",
       "      <td>6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>1136966</td>\n",
       "      <td>#ffffff color code</td>\n",
       "      <td>Color Schemes with #ffffff. 1  #d9d9d9 #d9d9d9...</td>\n",
       "      <td>7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>1136966</td>\n",
       "      <td>#ffffff color code</td>\n",
       "      <td>Having a set of related colors can be useful i...</td>\n",
       "      <td>8</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>1136966</td>\n",
       "      <td>#ffffff color code</td>\n",
       "      <td>Hex color #FFFFFF is a web safe color. Inverse...</td>\n",
       "      <td>9</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>1136956</td>\n",
       "      <td>why did the monroe doctrine originate?</td>\n",
       "      <td>Although the European powers did make military...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>1136956</td>\n",
       "      <td>why did the monroe doctrine originate?</td>\n",
       "      <td>although the monroe doctrine declared unilater...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>1136956</td>\n",
       "      <td>why did the monroe doctrine originate?</td>\n",
       "      <td>What did the Monroe Doctrine state? A: The Mon...</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>1136956</td>\n",
       "      <td>why did the monroe doctrine originate?</td>\n",
       "      <td>Monroe Doctrine. On December 23, 1823, in his ...</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>1136956</td>\n",
       "      <td>why did the monroe doctrine originate?</td>\n",
       "      <td>Monroe Doctrine, principle of American foreign...</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>1136956</td>\n",
       "      <td>why did the monroe doctrine originate?</td>\n",
       "      <td>Quick Answer. The Monroe Doctrine was importan...</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>1136956</td>\n",
       "      <td>why did the monroe doctrine originate?</td>\n",
       "      <td>A: The Monroe Doctrine was important because i...</td>\n",
       "      <td>6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>1136956</td>\n",
       "      <td>why did the monroe doctrine originate?</td>\n",
       "      <td>The doctrine was not ratified by any congressi...</td>\n",
       "      <td>7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>1136956</td>\n",
       "      <td>why did the monroe doctrine originate?</td>\n",
       "      <td>The Monroe Doctrine, proposed by President jam...</td>\n",
       "      <td>8</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>1136956</td>\n",
       "      <td>why did the monroe doctrine originate?</td>\n",
       "      <td>Monroe doctrine a principle of US policy, orig...</td>\n",
       "      <td>9</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>1136841</td>\n",
       "      <td>why do trade winds move in this pattern</td>\n",
       "      <td>Global wind patterns. Global wind patterns: Wi...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21</th>\n",
       "      <td>1136841</td>\n",
       "      <td>why do trade winds move in this pattern</td>\n",
       "      <td>Wind Direction - Wind direction is described b...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22</th>\n",
       "      <td>1136841</td>\n",
       "      <td>why do trade winds move in this pattern</td>\n",
       "      <td>Cold heavy air flows south from the north pole...</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23</th>\n",
       "      <td>1136841</td>\n",
       "      <td>why do trade winds move in this pattern</td>\n",
       "      <td>The stronger the high pressure is and the clos...</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24</th>\n",
       "      <td>1136841</td>\n",
       "      <td>why do trade winds move in this pattern</td>\n",
       "      <td>Weather - Wind. 1  Trade winds - Trade winds o...</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25</th>\n",
       "      <td>1136841</td>\n",
       "      <td>why do trade winds move in this pattern</td>\n",
       "      <td>Interesting Facts about Wind. 1  The fastest w...</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26</th>\n",
       "      <td>1136841</td>\n",
       "      <td>why do trade winds move in this pattern</td>\n",
       "      <td>These winds are often grouped together as trad...</td>\n",
       "      <td>6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27</th>\n",
       "      <td>1136841</td>\n",
       "      <td>why do trade winds move in this pattern</td>\n",
       "      <td>Trade Winds. Trade winds were introduced in Ch...</td>\n",
       "      <td>7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>28</th>\n",
       "      <td>1136841</td>\n",
       "      <td>why do trade winds move in this pattern</td>\n",
       "      <td>Global Winds The Earth has consistent wind pat...</td>\n",
       "      <td>8</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29</th>\n",
       "      <td>1136841</td>\n",
       "      <td>why do trade winds move in this pattern</td>\n",
       "      <td>This is due to two factors: Air in the tropics...</td>\n",
       "      <td>9</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>415350</th>\n",
       "      <td>332649</td>\n",
       "      <td>how often to do lymphatic drainage massage</td>\n",
       "      <td>The lymphatic system has the task of picking u...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>415351</th>\n",
       "      <td>332649</td>\n",
       "      <td>how often to do lymphatic drainage massage</td>\n",
       "      <td>Lymphatic drainage is very gentle, is not pain...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>415352</th>\n",
       "      <td>332649</td>\n",
       "      <td>how often to do lymphatic drainage massage</td>\n",
       "      <td>Benefits of Lymphatic Massage. 1  Helps to inc...</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>415353</th>\n",
       "      <td>332649</td>\n",
       "      <td>how often to do lymphatic drainage massage</td>\n",
       "      <td>What is lymphatic massage? Lymphatic massage, ...</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>415354</th>\n",
       "      <td>332649</td>\n",
       "      <td>how often to do lymphatic drainage massage</td>\n",
       "      <td>Benefits of Lymphatic Massage. Helps to increa...</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>415355</th>\n",
       "      <td>332649</td>\n",
       "      <td>how often to do lymphatic drainage massage</td>\n",
       "      <td>Estheticians are trained in a very specific fo...</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>415356</th>\n",
       "      <td>332649</td>\n",
       "      <td>how often to do lymphatic drainage massage</td>\n",
       "      <td>Benefits of Lymphatic Massage. 1  Helps to inc...</td>\n",
       "      <td>6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>415357</th>\n",
       "      <td>332649</td>\n",
       "      <td>how often to do lymphatic drainage massage</td>\n",
       "      <td>Lymphatic Drainage is most productive in a ser...</td>\n",
       "      <td>7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>415358</th>\n",
       "      <td>332649</td>\n",
       "      <td>how often to do lymphatic drainage massage</td>\n",
       "      <td>Once in the lymphatic vessels, the system can ...</td>\n",
       "      <td>8</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>415359</th>\n",
       "      <td>332649</td>\n",
       "      <td>how often to do lymphatic drainage massage</td>\n",
       "      <td>Thank you for your question. Lymphatic massage...</td>\n",
       "      <td>9</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>415360</th>\n",
       "      <td>374435</td>\n",
       "      <td>how to put supervisor module standby hot cisco</td>\n",
       "      <td>This document is Cisco Public Information. Pag...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>415361</th>\n",
       "      <td>374435</td>\n",
       "      <td>how to put supervisor module standby hot cisco</td>\n",
       "      <td>Hybrid Mode. In this section, you are presente...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>415362</th>\n",
       "      <td>374435</td>\n",
       "      <td>how to put supervisor module standby hot cisco</td>\n",
       "      <td>SSO The show module command displays Hot. For ...</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>415363</th>\n",
       "      <td>374435</td>\n",
       "      <td>how to put supervisor module standby hot cisco</td>\n",
       "      <td>For any other states, the standby supervisor d...</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>415364</th>\n",
       "      <td>374435</td>\n",
       "      <td>how to put supervisor module standby hot cisco</td>\n",
       "      <td>This section explains the step by step procedu...</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>415365</th>\n",
       "      <td>374435</td>\n",
       "      <td>how to put supervisor module standby hot cisco</td>\n",
       "      <td>Configuration changes on the active unit are a...</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>415366</th>\n",
       "      <td>374435</td>\n",
       "      <td>how to put supervisor module standby hot cisco</td>\n",
       "      <td>Route Processor Redundancy (RPR) refers to the...</td>\n",
       "      <td>6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>415367</th>\n",
       "      <td>374435</td>\n",
       "      <td>how to put supervisor module standby hot cisco</td>\n",
       "      <td>Catalyst 6500 Series Switches allow a redundan...</td>\n",
       "      <td>7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>415368</th>\n",
       "      <td>374435</td>\n",
       "      <td>how to put supervisor module standby hot cisco</td>\n",
       "      <td>Conventions Refer to Cisco Technical Tips Conv...</td>\n",
       "      <td>8</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>415369</th>\n",
       "      <td>374435</td>\n",
       "      <td>how to put supervisor module standby hot cisco</td>\n",
       "      <td>Failover to Standby Supervisor and Verify. Now...</td>\n",
       "      <td>9</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>415370</th>\n",
       "      <td>123628</td>\n",
       "      <td>define male chauvinism</td>\n",
       "      <td>Chauvinism is an exaggerated patriotism and a ...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>415371</th>\n",
       "      <td>123628</td>\n",
       "      <td>define male chauvinism</td>\n",
       "      <td>male chauvinist noun [C] uk / ˌmeɪl ˈʃəʊ.vɪ.nɪ...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>415372</th>\n",
       "      <td>123628</td>\n",
       "      <td>define male chauvinism</td>\n",
       "      <td>male chauvinist meaning, definition, what is m...</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>415373</th>\n",
       "      <td>123628</td>\n",
       "      <td>define male chauvinism</td>\n",
       "      <td>She claims that many young women in the United...</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>415374</th>\n",
       "      <td>123628</td>\n",
       "      <td>define male chauvinism</td>\n",
       "      <td>A frequent contemporary use of the term in Eng...</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>415375</th>\n",
       "      <td>123628</td>\n",
       "      <td>define male chauvinism</td>\n",
       "      <td>a bastion of male chauvinism (Definition of “m...</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>415376</th>\n",
       "      <td>123628</td>\n",
       "      <td>define male chauvinism</td>\n",
       "      <td>Male chauvinism is a term used to describe the...</td>\n",
       "      <td>6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>415377</th>\n",
       "      <td>123628</td>\n",
       "      <td>define male chauvinism</td>\n",
       "      <td>Male chauvinism is the belief that men are sup...</td>\n",
       "      <td>7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>415378</th>\n",
       "      <td>123628</td>\n",
       "      <td>define male chauvinism</td>\n",
       "      <td>Chauvinism is a form of extreme patriotism and...</td>\n",
       "      <td>8</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>415379</th>\n",
       "      <td>123628</td>\n",
       "      <td>define male chauvinism</td>\n",
       "      <td>In modern English, the word has come to be use...</td>\n",
       "      <td>9</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>415380 rows × 4 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "        query_id                                           query  \\\n",
       "0        1136966                              #ffffff color code   \n",
       "1        1136966                              #ffffff color code   \n",
       "2        1136966                              #ffffff color code   \n",
       "3        1136966                              #ffffff color code   \n",
       "4        1136966                              #ffffff color code   \n",
       "5        1136966                              #ffffff color code   \n",
       "6        1136966                              #ffffff color code   \n",
       "7        1136966                              #ffffff color code   \n",
       "8        1136966                              #ffffff color code   \n",
       "9        1136966                              #ffffff color code   \n",
       "10       1136956          why did the monroe doctrine originate?   \n",
       "11       1136956          why did the monroe doctrine originate?   \n",
       "12       1136956          why did the monroe doctrine originate?   \n",
       "13       1136956          why did the monroe doctrine originate?   \n",
       "14       1136956          why did the monroe doctrine originate?   \n",
       "15       1136956          why did the monroe doctrine originate?   \n",
       "16       1136956          why did the monroe doctrine originate?   \n",
       "17       1136956          why did the monroe doctrine originate?   \n",
       "18       1136956          why did the monroe doctrine originate?   \n",
       "19       1136956          why did the monroe doctrine originate?   \n",
       "20       1136841         why do trade winds move in this pattern   \n",
       "21       1136841         why do trade winds move in this pattern   \n",
       "22       1136841         why do trade winds move in this pattern   \n",
       "23       1136841         why do trade winds move in this pattern   \n",
       "24       1136841         why do trade winds move in this pattern   \n",
       "25       1136841         why do trade winds move in this pattern   \n",
       "26       1136841         why do trade winds move in this pattern   \n",
       "27       1136841         why do trade winds move in this pattern   \n",
       "28       1136841         why do trade winds move in this pattern   \n",
       "29       1136841         why do trade winds move in this pattern   \n",
       "...          ...                                             ...   \n",
       "415350    332649      how often to do lymphatic drainage massage   \n",
       "415351    332649      how often to do lymphatic drainage massage   \n",
       "415352    332649      how often to do lymphatic drainage massage   \n",
       "415353    332649      how often to do lymphatic drainage massage   \n",
       "415354    332649      how often to do lymphatic drainage massage   \n",
       "415355    332649      how often to do lymphatic drainage massage   \n",
       "415356    332649      how often to do lymphatic drainage massage   \n",
       "415357    332649      how often to do lymphatic drainage massage   \n",
       "415358    332649      how often to do lymphatic drainage massage   \n",
       "415359    332649      how often to do lymphatic drainage massage   \n",
       "415360    374435  how to put supervisor module standby hot cisco   \n",
       "415361    374435  how to put supervisor module standby hot cisco   \n",
       "415362    374435  how to put supervisor module standby hot cisco   \n",
       "415363    374435  how to put supervisor module standby hot cisco   \n",
       "415364    374435  how to put supervisor module standby hot cisco   \n",
       "415365    374435  how to put supervisor module standby hot cisco   \n",
       "415366    374435  how to put supervisor module standby hot cisco   \n",
       "415367    374435  how to put supervisor module standby hot cisco   \n",
       "415368    374435  how to put supervisor module standby hot cisco   \n",
       "415369    374435  how to put supervisor module standby hot cisco   \n",
       "415370    123628                          define male chauvinism   \n",
       "415371    123628                          define male chauvinism   \n",
       "415372    123628                          define male chauvinism   \n",
       "415373    123628                          define male chauvinism   \n",
       "415374    123628                          define male chauvinism   \n",
       "415375    123628                          define male chauvinism   \n",
       "415376    123628                          define male chauvinism   \n",
       "415377    123628                          define male chauvinism   \n",
       "415378    123628                          define male chauvinism   \n",
       "415379    123628                          define male chauvinism   \n",
       "\n",
       "                                             passage_text  passage_id  \n",
       "0       Color hex is a easy to use tool to get the col...           0  \n",
       "1       #ffffff Color Conversion. The hexadecimal colo...           1  \n",
       "2       CSS Codes; Color Preview; Color Schemes; Color...           2  \n",
       "3       Color Hex Color Codes Color-hex gives informat...           3  \n",
       "4       Color Hex Color Codes. Color-hex gives informa...           4  \n",
       "5       Color information. #FFFFFF (or 0xFFFFFF) is kn...           5  \n",
       "6       #ffffff color RGB value is (255,255,255). This...           6  \n",
       "7       Color Schemes with #ffffff. 1  #d9d9d9 #d9d9d9...           7  \n",
       "8       Having a set of related colors can be useful i...           8  \n",
       "9       Hex color #FFFFFF is a web safe color. Inverse...           9  \n",
       "10      Although the European powers did make military...           0  \n",
       "11      although the monroe doctrine declared unilater...           1  \n",
       "12      What did the Monroe Doctrine state? A: The Mon...           2  \n",
       "13      Monroe Doctrine. On December 23, 1823, in his ...           3  \n",
       "14      Monroe Doctrine, principle of American foreign...           4  \n",
       "15      Quick Answer. The Monroe Doctrine was importan...           5  \n",
       "16      A: The Monroe Doctrine was important because i...           6  \n",
       "17      The doctrine was not ratified by any congressi...           7  \n",
       "18      The Monroe Doctrine, proposed by President jam...           8  \n",
       "19      Monroe doctrine a principle of US policy, orig...           9  \n",
       "20      Global wind patterns. Global wind patterns: Wi...           0  \n",
       "21      Wind Direction - Wind direction is described b...           1  \n",
       "22      Cold heavy air flows south from the north pole...           2  \n",
       "23      The stronger the high pressure is and the clos...           3  \n",
       "24      Weather - Wind. 1  Trade winds - Trade winds o...           4  \n",
       "25      Interesting Facts about Wind. 1  The fastest w...           5  \n",
       "26      These winds are often grouped together as trad...           6  \n",
       "27      Trade Winds. Trade winds were introduced in Ch...           7  \n",
       "28      Global Winds The Earth has consistent wind pat...           8  \n",
       "29      This is due to two factors: Air in the tropics...           9  \n",
       "...                                                   ...         ...  \n",
       "415350  The lymphatic system has the task of picking u...           0  \n",
       "415351  Lymphatic drainage is very gentle, is not pain...           1  \n",
       "415352  Benefits of Lymphatic Massage. 1  Helps to inc...           2  \n",
       "415353  What is lymphatic massage? Lymphatic massage, ...           3  \n",
       "415354  Benefits of Lymphatic Massage. Helps to increa...           4  \n",
       "415355  Estheticians are trained in a very specific fo...           5  \n",
       "415356  Benefits of Lymphatic Massage. 1  Helps to inc...           6  \n",
       "415357  Lymphatic Drainage is most productive in a ser...           7  \n",
       "415358  Once in the lymphatic vessels, the system can ...           8  \n",
       "415359  Thank you for your question. Lymphatic massage...           9  \n",
       "415360  This document is Cisco Public Information. Pag...           0  \n",
       "415361  Hybrid Mode. In this section, you are presente...           1  \n",
       "415362  SSO The show module command displays Hot. For ...           2  \n",
       "415363  For any other states, the standby supervisor d...           3  \n",
       "415364  This section explains the step by step procedu...           4  \n",
       "415365  Configuration changes on the active unit are a...           5  \n",
       "415366  Route Processor Redundancy (RPR) refers to the...           6  \n",
       "415367  Catalyst 6500 Series Switches allow a redundan...           7  \n",
       "415368  Conventions Refer to Cisco Technical Tips Conv...           8  \n",
       "415369  Failover to Standby Supervisor and Verify. Now...           9  \n",
       "415370  Chauvinism is an exaggerated patriotism and a ...           0  \n",
       "415371  male chauvinist noun [C] uk / ˌmeɪl ˈʃəʊ.vɪ.nɪ...           1  \n",
       "415372  male chauvinist meaning, definition, what is m...           2  \n",
       "415373  She claims that many young women in the United...           3  \n",
       "415374  A frequent contemporary use of the term in Eng...           4  \n",
       "415375  a bastion of male chauvinism (Definition of “m...           5  \n",
       "415376  Male chauvinism is a term used to describe the...           6  \n",
       "415377  Male chauvinism is the belief that men are sup...           7  \n",
       "415378  Chauvinism is a form of extreme patriotism and...           8  \n",
       "415379  In modern English, the word has come to be use...           9  \n",
       "\n",
       "[415380 rows x 4 columns]"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": [
    "test['combined'] = test['query'] + ' ' + test['passage_text']"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "hidden": true
   },
   "source": [
    "### Checking the sequence length distribution on test set"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": [
    "test['seq_length'] = test['combined'].apply(lambda x: len(x.split()))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "hidden": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>query_id</th>\n",
       "      <th>query</th>\n",
       "      <th>passage_text</th>\n",
       "      <th>passage_id</th>\n",
       "      <th>combined</th>\n",
       "      <th>seq_length</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1136966</td>\n",
       "      <td>#ffffff color code</td>\n",
       "      <td>Color hex is a easy to use tool to get the col...</td>\n",
       "      <td>0</td>\n",
       "      <td>#ffffff color code Color hex is a easy to use ...</td>\n",
       "      <td>28</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1136966</td>\n",
       "      <td>#ffffff color code</td>\n",
       "      <td>#ffffff Color Conversion. The hexadecimal colo...</td>\n",
       "      <td>1</td>\n",
       "      <td>#ffffff color code #ffffff Color Conversion. T...</td>\n",
       "      <td>30</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1136966</td>\n",
       "      <td>#ffffff color code</td>\n",
       "      <td>CSS Codes; Color Preview; Color Schemes; Color...</td>\n",
       "      <td>2</td>\n",
       "      <td>#ffffff color code CSS Codes; Color Preview; C...</td>\n",
       "      <td>99</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1136966</td>\n",
       "      <td>#ffffff color code</td>\n",
       "      <td>Color Hex Color Codes Color-hex gives informat...</td>\n",
       "      <td>3</td>\n",
       "      <td>#ffffff color code Color Hex Color Codes Color...</td>\n",
       "      <td>65</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1136966</td>\n",
       "      <td>#ffffff color code</td>\n",
       "      <td>Color Hex Color Codes. Color-hex gives informa...</td>\n",
       "      <td>4</td>\n",
       "      <td>#ffffff color code Color Hex Color Codes. Colo...</td>\n",
       "      <td>51</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>1136966</td>\n",
       "      <td>#ffffff color code</td>\n",
       "      <td>Color information. #FFFFFF (or 0xFFFFFF) is kn...</td>\n",
       "      <td>5</td>\n",
       "      <td>#ffffff color code Color information. #FFFFFF ...</td>\n",
       "      <td>34</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>1136966</td>\n",
       "      <td>#ffffff color code</td>\n",
       "      <td>#ffffff color RGB value is (255,255,255). This...</td>\n",
       "      <td>6</td>\n",
       "      <td>#ffffff color code #ffffff color RGB value is ...</td>\n",
       "      <td>87</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>1136966</td>\n",
       "      <td>#ffffff color code</td>\n",
       "      <td>Color Schemes with #ffffff. 1  #d9d9d9 #d9d9d9...</td>\n",
       "      <td>7</td>\n",
       "      <td>#ffffff color code Color Schemes with #ffffff....</td>\n",
       "      <td>23</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>1136966</td>\n",
       "      <td>#ffffff color code</td>\n",
       "      <td>Having a set of related colors can be useful i...</td>\n",
       "      <td>8</td>\n",
       "      <td>#ffffff color code Having a set of related col...</td>\n",
       "      <td>39</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>1136966</td>\n",
       "      <td>#ffffff color code</td>\n",
       "      <td>Hex color #FFFFFF is a web safe color. Inverse...</td>\n",
       "      <td>9</td>\n",
       "      <td>#ffffff color code Hex color #FFFFFF is a web ...</td>\n",
       "      <td>82</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>1136956</td>\n",
       "      <td>why did the monroe doctrine originate?</td>\n",
       "      <td>Although the European powers did make military...</td>\n",
       "      <td>0</td>\n",
       "      <td>why did the monroe doctrine originate? Althoug...</td>\n",
       "      <td>51</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>1136956</td>\n",
       "      <td>why did the monroe doctrine originate?</td>\n",
       "      <td>although the monroe doctrine declared unilater...</td>\n",
       "      <td>1</td>\n",
       "      <td>why did the monroe doctrine originate? althoug...</td>\n",
       "      <td>51</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>1136956</td>\n",
       "      <td>why did the monroe doctrine originate?</td>\n",
       "      <td>What did the Monroe Doctrine state? A: The Mon...</td>\n",
       "      <td>2</td>\n",
       "      <td>why did the monroe doctrine originate? What di...</td>\n",
       "      <td>42</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>1136956</td>\n",
       "      <td>why did the monroe doctrine originate?</td>\n",
       "      <td>Monroe Doctrine. On December 23, 1823, in his ...</td>\n",
       "      <td>3</td>\n",
       "      <td>why did the monroe doctrine originate? Monroe ...</td>\n",
       "      <td>36</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>1136956</td>\n",
       "      <td>why did the monroe doctrine originate?</td>\n",
       "      <td>Monroe Doctrine, principle of American foreign...</td>\n",
       "      <td>4</td>\n",
       "      <td>why did the monroe doctrine originate? Monroe ...</td>\n",
       "      <td>49</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>1136956</td>\n",
       "      <td>why did the monroe doctrine originate?</td>\n",
       "      <td>Quick Answer. The Monroe Doctrine was importan...</td>\n",
       "      <td>5</td>\n",
       "      <td>why did the monroe doctrine originate? Quick A...</td>\n",
       "      <td>50</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>1136956</td>\n",
       "      <td>why did the monroe doctrine originate?</td>\n",
       "      <td>A: The Monroe Doctrine was important because i...</td>\n",
       "      <td>6</td>\n",
       "      <td>why did the monroe doctrine originate? A: The ...</td>\n",
       "      <td>49</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>1136956</td>\n",
       "      <td>why did the monroe doctrine originate?</td>\n",
       "      <td>The doctrine was not ratified by any congressi...</td>\n",
       "      <td>7</td>\n",
       "      <td>why did the monroe doctrine originate? The doc...</td>\n",
       "      <td>38</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>1136956</td>\n",
       "      <td>why did the monroe doctrine originate?</td>\n",
       "      <td>The Monroe Doctrine, proposed by President jam...</td>\n",
       "      <td>8</td>\n",
       "      <td>why did the monroe doctrine originate? The Mon...</td>\n",
       "      <td>69</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>1136956</td>\n",
       "      <td>why did the monroe doctrine originate?</td>\n",
       "      <td>Monroe doctrine a principle of US policy, orig...</td>\n",
       "      <td>9</td>\n",
       "      <td>why did the monroe doctrine originate? Monroe ...</td>\n",
       "      <td>52</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>1136841</td>\n",
       "      <td>why do trade winds move in this pattern</td>\n",
       "      <td>Global wind patterns. Global wind patterns: Wi...</td>\n",
       "      <td>0</td>\n",
       "      <td>why do trade winds move in this pattern Global...</td>\n",
       "      <td>88</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21</th>\n",
       "      <td>1136841</td>\n",
       "      <td>why do trade winds move in this pattern</td>\n",
       "      <td>Wind Direction - Wind direction is described b...</td>\n",
       "      <td>1</td>\n",
       "      <td>why do trade winds move in this pattern Wind D...</td>\n",
       "      <td>59</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22</th>\n",
       "      <td>1136841</td>\n",
       "      <td>why do trade winds move in this pattern</td>\n",
       "      <td>Cold heavy air flows south from the north pole...</td>\n",
       "      <td>2</td>\n",
       "      <td>why do trade winds move in this pattern Cold h...</td>\n",
       "      <td>105</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23</th>\n",
       "      <td>1136841</td>\n",
       "      <td>why do trade winds move in this pattern</td>\n",
       "      <td>The stronger the high pressure is and the clos...</td>\n",
       "      <td>3</td>\n",
       "      <td>why do trade winds move in this pattern The st...</td>\n",
       "      <td>61</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24</th>\n",
       "      <td>1136841</td>\n",
       "      <td>why do trade winds move in this pattern</td>\n",
       "      <td>Weather - Wind. 1  Trade winds - Trade winds o...</td>\n",
       "      <td>4</td>\n",
       "      <td>why do trade winds move in this pattern Weathe...</td>\n",
       "      <td>66</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25</th>\n",
       "      <td>1136841</td>\n",
       "      <td>why do trade winds move in this pattern</td>\n",
       "      <td>Interesting Facts about Wind. 1  The fastest w...</td>\n",
       "      <td>5</td>\n",
       "      <td>why do trade winds move in this pattern Intere...</td>\n",
       "      <td>64</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26</th>\n",
       "      <td>1136841</td>\n",
       "      <td>why do trade winds move in this pattern</td>\n",
       "      <td>These winds are often grouped together as trad...</td>\n",
       "      <td>6</td>\n",
       "      <td>why do trade winds move in this pattern These ...</td>\n",
       "      <td>52</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27</th>\n",
       "      <td>1136841</td>\n",
       "      <td>why do trade winds move in this pattern</td>\n",
       "      <td>Trade Winds. Trade winds were introduced in Ch...</td>\n",
       "      <td>7</td>\n",
       "      <td>why do trade winds move in this pattern Trade ...</td>\n",
       "      <td>52</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>28</th>\n",
       "      <td>1136841</td>\n",
       "      <td>why do trade winds move in this pattern</td>\n",
       "      <td>Global Winds The Earth has consistent wind pat...</td>\n",
       "      <td>8</td>\n",
       "      <td>why do trade winds move in this pattern Global...</td>\n",
       "      <td>128</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29</th>\n",
       "      <td>1136841</td>\n",
       "      <td>why do trade winds move in this pattern</td>\n",
       "      <td>This is due to two factors: Air in the tropics...</td>\n",
       "      <td>9</td>\n",
       "      <td>why do trade winds move in this pattern This i...</td>\n",
       "      <td>62</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>415350</th>\n",
       "      <td>332649</td>\n",
       "      <td>how often to do lymphatic drainage massage</td>\n",
       "      <td>The lymphatic system has the task of picking u...</td>\n",
       "      <td>0</td>\n",
       "      <td>how often to do lymphatic drainage massage The...</td>\n",
       "      <td>63</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>415351</th>\n",
       "      <td>332649</td>\n",
       "      <td>how often to do lymphatic drainage massage</td>\n",
       "      <td>Lymphatic drainage is very gentle, is not pain...</td>\n",
       "      <td>1</td>\n",
       "      <td>how often to do lymphatic drainage massage Lym...</td>\n",
       "      <td>48</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>415352</th>\n",
       "      <td>332649</td>\n",
       "      <td>how often to do lymphatic drainage massage</td>\n",
       "      <td>Benefits of Lymphatic Massage. 1  Helps to inc...</td>\n",
       "      <td>2</td>\n",
       "      <td>how often to do lymphatic drainage massage Ben...</td>\n",
       "      <td>56</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>415353</th>\n",
       "      <td>332649</td>\n",
       "      <td>how often to do lymphatic drainage massage</td>\n",
       "      <td>What is lymphatic massage? Lymphatic massage, ...</td>\n",
       "      <td>3</td>\n",
       "      <td>how often to do lymphatic drainage massage Wha...</td>\n",
       "      <td>52</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>415354</th>\n",
       "      <td>332649</td>\n",
       "      <td>how often to do lymphatic drainage massage</td>\n",
       "      <td>Benefits of Lymphatic Massage. Helps to increa...</td>\n",
       "      <td>4</td>\n",
       "      <td>how often to do lymphatic drainage massage Ben...</td>\n",
       "      <td>55</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>415355</th>\n",
       "      <td>332649</td>\n",
       "      <td>how often to do lymphatic drainage massage</td>\n",
       "      <td>Estheticians are trained in a very specific fo...</td>\n",
       "      <td>5</td>\n",
       "      <td>how often to do lymphatic drainage massage Est...</td>\n",
       "      <td>55</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>415356</th>\n",
       "      <td>332649</td>\n",
       "      <td>how often to do lymphatic drainage massage</td>\n",
       "      <td>Benefits of Lymphatic Massage. 1  Helps to inc...</td>\n",
       "      <td>6</td>\n",
       "      <td>how often to do lymphatic drainage massage Ben...</td>\n",
       "      <td>65</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>415357</th>\n",
       "      <td>332649</td>\n",
       "      <td>how often to do lymphatic drainage massage</td>\n",
       "      <td>Lymphatic Drainage is most productive in a ser...</td>\n",
       "      <td>7</td>\n",
       "      <td>how often to do lymphatic drainage massage Lym...</td>\n",
       "      <td>47</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>415358</th>\n",
       "      <td>332649</td>\n",
       "      <td>how often to do lymphatic drainage massage</td>\n",
       "      <td>Once in the lymphatic vessels, the system can ...</td>\n",
       "      <td>8</td>\n",
       "      <td>how often to do lymphatic drainage massage Onc...</td>\n",
       "      <td>58</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>415359</th>\n",
       "      <td>332649</td>\n",
       "      <td>how often to do lymphatic drainage massage</td>\n",
       "      <td>Thank you for your question. Lymphatic massage...</td>\n",
       "      <td>9</td>\n",
       "      <td>how often to do lymphatic drainage massage Tha...</td>\n",
       "      <td>48</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>415360</th>\n",
       "      <td>374435</td>\n",
       "      <td>how to put supervisor module standby hot cisco</td>\n",
       "      <td>This document is Cisco Public Information. Pag...</td>\n",
       "      <td>0</td>\n",
       "      <td>how to put supervisor module standby hot cisco...</td>\n",
       "      <td>110</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>415361</th>\n",
       "      <td>374435</td>\n",
       "      <td>how to put supervisor module standby hot cisco</td>\n",
       "      <td>Hybrid Mode. In this section, you are presente...</td>\n",
       "      <td>1</td>\n",
       "      <td>how to put supervisor module standby hot cisco...</td>\n",
       "      <td>98</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>415362</th>\n",
       "      <td>374435</td>\n",
       "      <td>how to put supervisor module standby hot cisco</td>\n",
       "      <td>SSO The show module command displays Hot. For ...</td>\n",
       "      <td>2</td>\n",
       "      <td>how to put supervisor module standby hot cisco...</td>\n",
       "      <td>97</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>415363</th>\n",
       "      <td>374435</td>\n",
       "      <td>how to put supervisor module standby hot cisco</td>\n",
       "      <td>For any other states, the standby supervisor d...</td>\n",
       "      <td>3</td>\n",
       "      <td>how to put supervisor module standby hot cisco...</td>\n",
       "      <td>84</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>415364</th>\n",
       "      <td>374435</td>\n",
       "      <td>how to put supervisor module standby hot cisco</td>\n",
       "      <td>This section explains the step by step procedu...</td>\n",
       "      <td>4</td>\n",
       "      <td>how to put supervisor module standby hot cisco...</td>\n",
       "      <td>106</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>415365</th>\n",
       "      <td>374435</td>\n",
       "      <td>how to put supervisor module standby hot cisco</td>\n",
       "      <td>Configuration changes on the active unit are a...</td>\n",
       "      <td>5</td>\n",
       "      <td>how to put supervisor module standby hot cisco...</td>\n",
       "      <td>141</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>415366</th>\n",
       "      <td>374435</td>\n",
       "      <td>how to put supervisor module standby hot cisco</td>\n",
       "      <td>Route Processor Redundancy (RPR) refers to the...</td>\n",
       "      <td>6</td>\n",
       "      <td>how to put supervisor module standby hot cisco...</td>\n",
       "      <td>104</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>415367</th>\n",
       "      <td>374435</td>\n",
       "      <td>how to put supervisor module standby hot cisco</td>\n",
       "      <td>Catalyst 6500 Series Switches allow a redundan...</td>\n",
       "      <td>7</td>\n",
       "      <td>how to put supervisor module standby hot cisco...</td>\n",
       "      <td>114</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>415368</th>\n",
       "      <td>374435</td>\n",
       "      <td>how to put supervisor module standby hot cisco</td>\n",
       "      <td>Conventions Refer to Cisco Technical Tips Conv...</td>\n",
       "      <td>8</td>\n",
       "      <td>how to put supervisor module standby hot cisco...</td>\n",
       "      <td>95</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>415369</th>\n",
       "      <td>374435</td>\n",
       "      <td>how to put supervisor module standby hot cisco</td>\n",
       "      <td>Failover to Standby Supervisor and Verify. Now...</td>\n",
       "      <td>9</td>\n",
       "      <td>how to put supervisor module standby hot cisco...</td>\n",
       "      <td>98</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>415370</th>\n",
       "      <td>123628</td>\n",
       "      <td>define male chauvinism</td>\n",
       "      <td>Chauvinism is an exaggerated patriotism and a ...</td>\n",
       "      <td>0</td>\n",
       "      <td>define male chauvinism Chauvinism is an exagge...</td>\n",
       "      <td>47</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>415371</th>\n",
       "      <td>123628</td>\n",
       "      <td>define male chauvinism</td>\n",
       "      <td>male chauvinist noun [C] uk / ˌmeɪl ˈʃəʊ.vɪ.nɪ...</td>\n",
       "      <td>1</td>\n",
       "      <td>define male chauvinism male chauvinist noun [C...</td>\n",
       "      <td>43</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>415372</th>\n",
       "      <td>123628</td>\n",
       "      <td>define male chauvinism</td>\n",
       "      <td>male chauvinist meaning, definition, what is m...</td>\n",
       "      <td>2</td>\n",
       "      <td>define male chauvinism male chauvinist meaning...</td>\n",
       "      <td>27</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>415373</th>\n",
       "      <td>123628</td>\n",
       "      <td>define male chauvinism</td>\n",
       "      <td>She claims that many young women in the United...</td>\n",
       "      <td>3</td>\n",
       "      <td>define male chauvinism She claims that many yo...</td>\n",
       "      <td>42</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>415374</th>\n",
       "      <td>123628</td>\n",
       "      <td>define male chauvinism</td>\n",
       "      <td>A frequent contemporary use of the term in Eng...</td>\n",
       "      <td>4</td>\n",
       "      <td>define male chauvinism A frequent contemporary...</td>\n",
       "      <td>55</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>415375</th>\n",
       "      <td>123628</td>\n",
       "      <td>define male chauvinism</td>\n",
       "      <td>a bastion of male chauvinism (Definition of “m...</td>\n",
       "      <td>5</td>\n",
       "      <td>define male chauvinism a bastion of male chauv...</td>\n",
       "      <td>30</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>415376</th>\n",
       "      <td>123628</td>\n",
       "      <td>define male chauvinism</td>\n",
       "      <td>Male chauvinism is a term used to describe the...</td>\n",
       "      <td>6</td>\n",
       "      <td>define male chauvinism Male chauvinism is a te...</td>\n",
       "      <td>59</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>415377</th>\n",
       "      <td>123628</td>\n",
       "      <td>define male chauvinism</td>\n",
       "      <td>Male chauvinism is the belief that men are sup...</td>\n",
       "      <td>7</td>\n",
       "      <td>define male chauvinism Male chauvinism is the ...</td>\n",
       "      <td>38</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>415378</th>\n",
       "      <td>123628</td>\n",
       "      <td>define male chauvinism</td>\n",
       "      <td>Chauvinism is a form of extreme patriotism and...</td>\n",
       "      <td>8</td>\n",
       "      <td>define male chauvinism Chauvinism is a form of...</td>\n",
       "      <td>49</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>415379</th>\n",
       "      <td>123628</td>\n",
       "      <td>define male chauvinism</td>\n",
       "      <td>In modern English, the word has come to be use...</td>\n",
       "      <td>9</td>\n",
       "      <td>define male chauvinism In modern English, the ...</td>\n",
       "      <td>57</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>415380 rows × 6 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "        query_id                                           query  \\\n",
       "0        1136966                              #ffffff color code   \n",
       "1        1136966                              #ffffff color code   \n",
       "2        1136966                              #ffffff color code   \n",
       "3        1136966                              #ffffff color code   \n",
       "4        1136966                              #ffffff color code   \n",
       "5        1136966                              #ffffff color code   \n",
       "6        1136966                              #ffffff color code   \n",
       "7        1136966                              #ffffff color code   \n",
       "8        1136966                              #ffffff color code   \n",
       "9        1136966                              #ffffff color code   \n",
       "10       1136956          why did the monroe doctrine originate?   \n",
       "11       1136956          why did the monroe doctrine originate?   \n",
       "12       1136956          why did the monroe doctrine originate?   \n",
       "13       1136956          why did the monroe doctrine originate?   \n",
       "14       1136956          why did the monroe doctrine originate?   \n",
       "15       1136956          why did the monroe doctrine originate?   \n",
       "16       1136956          why did the monroe doctrine originate?   \n",
       "17       1136956          why did the monroe doctrine originate?   \n",
       "18       1136956          why did the monroe doctrine originate?   \n",
       "19       1136956          why did the monroe doctrine originate?   \n",
       "20       1136841         why do trade winds move in this pattern   \n",
       "21       1136841         why do trade winds move in this pattern   \n",
       "22       1136841         why do trade winds move in this pattern   \n",
       "23       1136841         why do trade winds move in this pattern   \n",
       "24       1136841         why do trade winds move in this pattern   \n",
       "25       1136841         why do trade winds move in this pattern   \n",
       "26       1136841         why do trade winds move in this pattern   \n",
       "27       1136841         why do trade winds move in this pattern   \n",
       "28       1136841         why do trade winds move in this pattern   \n",
       "29       1136841         why do trade winds move in this pattern   \n",
       "...          ...                                             ...   \n",
       "415350    332649      how often to do lymphatic drainage massage   \n",
       "415351    332649      how often to do lymphatic drainage massage   \n",
       "415352    332649      how often to do lymphatic drainage massage   \n",
       "415353    332649      how often to do lymphatic drainage massage   \n",
       "415354    332649      how often to do lymphatic drainage massage   \n",
       "415355    332649      how often to do lymphatic drainage massage   \n",
       "415356    332649      how often to do lymphatic drainage massage   \n",
       "415357    332649      how often to do lymphatic drainage massage   \n",
       "415358    332649      how often to do lymphatic drainage massage   \n",
       "415359    332649      how often to do lymphatic drainage massage   \n",
       "415360    374435  how to put supervisor module standby hot cisco   \n",
       "415361    374435  how to put supervisor module standby hot cisco   \n",
       "415362    374435  how to put supervisor module standby hot cisco   \n",
       "415363    374435  how to put supervisor module standby hot cisco   \n",
       "415364    374435  how to put supervisor module standby hot cisco   \n",
       "415365    374435  how to put supervisor module standby hot cisco   \n",
       "415366    374435  how to put supervisor module standby hot cisco   \n",
       "415367    374435  how to put supervisor module standby hot cisco   \n",
       "415368    374435  how to put supervisor module standby hot cisco   \n",
       "415369    374435  how to put supervisor module standby hot cisco   \n",
       "415370    123628                          define male chauvinism   \n",
       "415371    123628                          define male chauvinism   \n",
       "415372    123628                          define male chauvinism   \n",
       "415373    123628                          define male chauvinism   \n",
       "415374    123628                          define male chauvinism   \n",
       "415375    123628                          define male chauvinism   \n",
       "415376    123628                          define male chauvinism   \n",
       "415377    123628                          define male chauvinism   \n",
       "415378    123628                          define male chauvinism   \n",
       "415379    123628                          define male chauvinism   \n",
       "\n",
       "                                             passage_text  passage_id  \\\n",
       "0       Color hex is a easy to use tool to get the col...           0   \n",
       "1       #ffffff Color Conversion. The hexadecimal colo...           1   \n",
       "2       CSS Codes; Color Preview; Color Schemes; Color...           2   \n",
       "3       Color Hex Color Codes Color-hex gives informat...           3   \n",
       "4       Color Hex Color Codes. Color-hex gives informa...           4   \n",
       "5       Color information. #FFFFFF (or 0xFFFFFF) is kn...           5   \n",
       "6       #ffffff color RGB value is (255,255,255). This...           6   \n",
       "7       Color Schemes with #ffffff. 1  #d9d9d9 #d9d9d9...           7   \n",
       "8       Having a set of related colors can be useful i...           8   \n",
       "9       Hex color #FFFFFF is a web safe color. Inverse...           9   \n",
       "10      Although the European powers did make military...           0   \n",
       "11      although the monroe doctrine declared unilater...           1   \n",
       "12      What did the Monroe Doctrine state? A: The Mon...           2   \n",
       "13      Monroe Doctrine. On December 23, 1823, in his ...           3   \n",
       "14      Monroe Doctrine, principle of American foreign...           4   \n",
       "15      Quick Answer. The Monroe Doctrine was importan...           5   \n",
       "16      A: The Monroe Doctrine was important because i...           6   \n",
       "17      The doctrine was not ratified by any congressi...           7   \n",
       "18      The Monroe Doctrine, proposed by President jam...           8   \n",
       "19      Monroe doctrine a principle of US policy, orig...           9   \n",
       "20      Global wind patterns. Global wind patterns: Wi...           0   \n",
       "21      Wind Direction - Wind direction is described b...           1   \n",
       "22      Cold heavy air flows south from the north pole...           2   \n",
       "23      The stronger the high pressure is and the clos...           3   \n",
       "24      Weather - Wind. 1  Trade winds - Trade winds o...           4   \n",
       "25      Interesting Facts about Wind. 1  The fastest w...           5   \n",
       "26      These winds are often grouped together as trad...           6   \n",
       "27      Trade Winds. Trade winds were introduced in Ch...           7   \n",
       "28      Global Winds The Earth has consistent wind pat...           8   \n",
       "29      This is due to two factors: Air in the tropics...           9   \n",
       "...                                                   ...         ...   \n",
       "415350  The lymphatic system has the task of picking u...           0   \n",
       "415351  Lymphatic drainage is very gentle, is not pain...           1   \n",
       "415352  Benefits of Lymphatic Massage. 1  Helps to inc...           2   \n",
       "415353  What is lymphatic massage? Lymphatic massage, ...           3   \n",
       "415354  Benefits of Lymphatic Massage. Helps to increa...           4   \n",
       "415355  Estheticians are trained in a very specific fo...           5   \n",
       "415356  Benefits of Lymphatic Massage. 1  Helps to inc...           6   \n",
       "415357  Lymphatic Drainage is most productive in a ser...           7   \n",
       "415358  Once in the lymphatic vessels, the system can ...           8   \n",
       "415359  Thank you for your question. Lymphatic massage...           9   \n",
       "415360  This document is Cisco Public Information. Pag...           0   \n",
       "415361  Hybrid Mode. In this section, you are presente...           1   \n",
       "415362  SSO The show module command displays Hot. For ...           2   \n",
       "415363  For any other states, the standby supervisor d...           3   \n",
       "415364  This section explains the step by step procedu...           4   \n",
       "415365  Configuration changes on the active unit are a...           5   \n",
       "415366  Route Processor Redundancy (RPR) refers to the...           6   \n",
       "415367  Catalyst 6500 Series Switches allow a redundan...           7   \n",
       "415368  Conventions Refer to Cisco Technical Tips Conv...           8   \n",
       "415369  Failover to Standby Supervisor and Verify. Now...           9   \n",
       "415370  Chauvinism is an exaggerated patriotism and a ...           0   \n",
       "415371  male chauvinist noun [C] uk / ˌmeɪl ˈʃəʊ.vɪ.nɪ...           1   \n",
       "415372  male chauvinist meaning, definition, what is m...           2   \n",
       "415373  She claims that many young women in the United...           3   \n",
       "415374  A frequent contemporary use of the term in Eng...           4   \n",
       "415375  a bastion of male chauvinism (Definition of “m...           5   \n",
       "415376  Male chauvinism is a term used to describe the...           6   \n",
       "415377  Male chauvinism is the belief that men are sup...           7   \n",
       "415378  Chauvinism is a form of extreme patriotism and...           8   \n",
       "415379  In modern English, the word has come to be use...           9   \n",
       "\n",
       "                                                 combined  seq_length  \n",
       "0       #ffffff color code Color hex is a easy to use ...          28  \n",
       "1       #ffffff color code #ffffff Color Conversion. T...          30  \n",
       "2       #ffffff color code CSS Codes; Color Preview; C...          99  \n",
       "3       #ffffff color code Color Hex Color Codes Color...          65  \n",
       "4       #ffffff color code Color Hex Color Codes. Colo...          51  \n",
       "5       #ffffff color code Color information. #FFFFFF ...          34  \n",
       "6       #ffffff color code #ffffff color RGB value is ...          87  \n",
       "7       #ffffff color code Color Schemes with #ffffff....          23  \n",
       "8       #ffffff color code Having a set of related col...          39  \n",
       "9       #ffffff color code Hex color #FFFFFF is a web ...          82  \n",
       "10      why did the monroe doctrine originate? Althoug...          51  \n",
       "11      why did the monroe doctrine originate? althoug...          51  \n",
       "12      why did the monroe doctrine originate? What di...          42  \n",
       "13      why did the monroe doctrine originate? Monroe ...          36  \n",
       "14      why did the monroe doctrine originate? Monroe ...          49  \n",
       "15      why did the monroe doctrine originate? Quick A...          50  \n",
       "16      why did the monroe doctrine originate? A: The ...          49  \n",
       "17      why did the monroe doctrine originate? The doc...          38  \n",
       "18      why did the monroe doctrine originate? The Mon...          69  \n",
       "19      why did the monroe doctrine originate? Monroe ...          52  \n",
       "20      why do trade winds move in this pattern Global...          88  \n",
       "21      why do trade winds move in this pattern Wind D...          59  \n",
       "22      why do trade winds move in this pattern Cold h...         105  \n",
       "23      why do trade winds move in this pattern The st...          61  \n",
       "24      why do trade winds move in this pattern Weathe...          66  \n",
       "25      why do trade winds move in this pattern Intere...          64  \n",
       "26      why do trade winds move in this pattern These ...          52  \n",
       "27      why do trade winds move in this pattern Trade ...          52  \n",
       "28      why do trade winds move in this pattern Global...         128  \n",
       "29      why do trade winds move in this pattern This i...          62  \n",
       "...                                                   ...         ...  \n",
       "415350  how often to do lymphatic drainage massage The...          63  \n",
       "415351  how often to do lymphatic drainage massage Lym...          48  \n",
       "415352  how often to do lymphatic drainage massage Ben...          56  \n",
       "415353  how often to do lymphatic drainage massage Wha...          52  \n",
       "415354  how often to do lymphatic drainage massage Ben...          55  \n",
       "415355  how often to do lymphatic drainage massage Est...          55  \n",
       "415356  how often to do lymphatic drainage massage Ben...          65  \n",
       "415357  how often to do lymphatic drainage massage Lym...          47  \n",
       "415358  how often to do lymphatic drainage massage Onc...          58  \n",
       "415359  how often to do lymphatic drainage massage Tha...          48  \n",
       "415360  how to put supervisor module standby hot cisco...         110  \n",
       "415361  how to put supervisor module standby hot cisco...          98  \n",
       "415362  how to put supervisor module standby hot cisco...          97  \n",
       "415363  how to put supervisor module standby hot cisco...          84  \n",
       "415364  how to put supervisor module standby hot cisco...         106  \n",
       "415365  how to put supervisor module standby hot cisco...         141  \n",
       "415366  how to put supervisor module standby hot cisco...         104  \n",
       "415367  how to put supervisor module standby hot cisco...         114  \n",
       "415368  how to put supervisor module standby hot cisco...          95  \n",
       "415369  how to put supervisor module standby hot cisco...          98  \n",
       "415370  define male chauvinism Chauvinism is an exagge...          47  \n",
       "415371  define male chauvinism male chauvinist noun [C...          43  \n",
       "415372  define male chauvinism male chauvinist meaning...          27  \n",
       "415373  define male chauvinism She claims that many yo...          42  \n",
       "415374  define male chauvinism A frequent contemporary...          55  \n",
       "415375  define male chauvinism a bastion of male chauv...          30  \n",
       "415376  define male chauvinism Male chauvinism is a te...          59  \n",
       "415377  define male chauvinism Male chauvinism is the ...          38  \n",
       "415378  define male chauvinism Chauvinism is a form of...          49  \n",
       "415379  define male chauvinism In modern English, the ...          57  \n",
       "\n",
       "[415380 rows x 6 columns]"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "hidden": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(286, 3)"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "max(test['seq_length']), min(test['seq_length'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "hidden": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>query_id</th>\n",
       "      <th>passage_id</th>\n",
       "      <th>seq_length</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>count</th>\n",
       "      <td>4.153800e+05</td>\n",
       "      <td>415380.000000</td>\n",
       "      <td>415380.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>7.641694e+05</td>\n",
       "      <td>4.500000</td>\n",
       "      <td>57.130247</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std</th>\n",
       "      <td>3.691768e+05</td>\n",
       "      <td>2.872285</td>\n",
       "      <td>19.467680</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>min</th>\n",
       "      <td>5.700000e+01</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>3.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25%</th>\n",
       "      <td>4.566740e+05</td>\n",
       "      <td>2.000000</td>\n",
       "      <td>45.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50%</th>\n",
       "      <td>8.695660e+05</td>\n",
       "      <td>4.500000</td>\n",
       "      <td>54.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75%</th>\n",
       "      <td>1.113454e+06</td>\n",
       "      <td>7.000000</td>\n",
       "      <td>64.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>max</th>\n",
       "      <td>1.136966e+06</td>\n",
       "      <td>9.000000</td>\n",
       "      <td>286.000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "           query_id     passage_id     seq_length\n",
       "count  4.153800e+05  415380.000000  415380.000000\n",
       "mean   7.641694e+05       4.500000      57.130247\n",
       "std    3.691768e+05       2.872285      19.467680\n",
       "min    5.700000e+01       0.000000       3.000000\n",
       "25%    4.566740e+05       2.000000      45.000000\n",
       "50%    8.695660e+05       4.500000      54.000000\n",
       "75%    1.113454e+06       7.000000      64.000000\n",
       "max    1.136966e+06       9.000000     286.000000"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test.describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": [
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {
    "hidden": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[<matplotlib.axes._subplots.AxesSubplot object at 0x7f57f43d0748>]],\n",
       "      dtype=object)"
      ]
     },
     "execution_count": 55,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYwAAAEICAYAAABMGMOEAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAHb1JREFUeJzt3X+QXWWd5/H3x4SEDCjhl12ZJGXHNeVUNGvEXsis1GwLK3SiTrAKnbAsBCeamTXMam2cJWg5ID+2wpbIDruIE4eMwVFDBqXIQpiYQW5Z7haQoIEQEGkhVtIbyUoS8ILCNn73j/N0PPbc7n5u307fH3xeVafuud/zPOc8T053f/Oc89xzFRGYmZmN5Q3NboCZmbUHJwwzM8vihGFmZlmcMMzMLIsThpmZZXHCMDOzLE4YZg2QdLWkv2/SsSuSPt6MY9vrkxOGWRtoZmIyG+KEYWZmWZwwrCNJukLSgKRfSnpK0rmS3iBpraSfSnpe0mZJp5TqXCLpZ2nb5yTtlfRv6zzuYkn/W9IRSY9K6i1tq0i6VtL/Su36rqTTStsvLR3/80PHl9QHfBb4E0lVSY+WDvmWkfZnNtGcMKzjSHo7cDnwryLijcD5wF7gL4ALgH8D/D5wGLgl1VkA3ApckradCsyp87izgXuB64BTgM8A35Z0eqnYvwM+BrwZmJbKDB3/y8DFwCzgJGA2QET8I/BfgDsi4sSIeNdY+zM7FpwwrBO9BkwHFkg6LiL2RsRPgT8HPhcR+yPiFeBq4EJJU4ELgXsi4vtp2+eB39R53H8PbI2IrRHxm4jYDuwElpbK/F1E/CQifgVsBhal+IXA/4yIH0TEq8BfATkPehtpf2YTzgnDOk5E9AOfpkgIByVtkvT7wFuAu9LloiPAkxTJpYtiVLGvtI+XgOfrPPRbgI8M7T8d42yKEcOQn5fWXwZOTOvDj/9y5vFH2p/ZhHPCsI4UEd+MiLMp/ogHcAPFH+QlETGztBwfEQPAAWDuUH1Jv0dxWaoe+4CvD9v/CRGxLqPuAUqXwCTNGHZ8P1bams4JwzqOpLdLOkfSdODXwK8oLi99Bbhe0ltSudMlLUvV7gQ+KOlsSdOAa6j/9+PvgQ9JOl/SFEnHS+qVlHMv5M5U91+n418NqLT9OaBbkn9nrWn8w2edaDqwDvgFxSWbNwNXAn8NbAG+K+mXwIPAWQARsQdYDXyT4n/7h4H99Rw0IvYByyhmNP1fihHHX5Lxe5aO/xfApnT8KnAQeCUV+Yf0+rykH9bTLrOJIn+BklltkvYCH4+If2rCsU8EjgDzI+LZyT6+WS0eYZi1CEkfkvR7kk4AvgjsppgObNYSnDDMxiDpvvSBueHLZyf4UMuA/5OW+cDy8CUAayHZl6QkTaGYUz4QER+UNI/ieuupwCPAJRHxarrReDvwHoppgX8SEXvTPq4EVlJMZfyPEbEtxfsori9PAf42c1aJmZlNonpGGJ+imLc+5Abgpoh4G8UNwpUpvhI4nOI3pXJDn2RdDrwD6AO+nGaSTKH4tO0SYAFwUSprZmYtZGpOoTQt8APA9cB/kiTgHIrHEgBspJgGeCvFsPrqFL8T+B+p/DJgU/oU7bOS+oEzU7n+iHgmHWtTKvvEaG067bTToru7O6f5ALz00kuccMIJ2eXbQSf2CdyvduN+tY+XXnqJH//4x7+IiNPHLv3PZSUM4L8B/xl4Y3p/KnAkIgbT+/2k596k130AETEo6YVUfjbFNEZq1Nk3LH7WWA3q7u5m586dmc2HSqVCb29vdvl20Il9Aver3bhf7aNSqfC+973vZ+OtP2bCkPRB4GBEPFJ+8mYzSFoFrALo6uqiUqlk161Wq3WVbwed2Cdwv9qN+9U+qtVqQ/VzRhjvBf5Y0lLgeOBNFDeoZ0qamkYZc4CBVH6A4hEL+9ND3U6iuPk9FB9SrjNS/HdExHpgPUBPT0/Uk/079X8LndYncL/ajfvVPhpNgDmfQL0yIuZERDfFTevvRcTFwAMUT9gEWAHcnda3pPek7d9LUwO3AMslTU8zrOYDDwM7gPmS5qVHIixPZc3MrIXk3sOo5Qpgk6TrgB8Bt6X4bcDX003tQxQJgIjYI2kzxc3sQWB1RLwGIOlyYBvFtNoN6TEJZmbWQupKGBFRASpp/Rl+O8upXObXwEdGqH89xUyr4fGtwNZ62mJmZpPLn/Q2M7MsThhmZpbFCcPMzLI4YZiZWZZGZklZC+pee+/R9b3rPtDElphZp/EIw8zMsjhhmJlZFicMMzPL4oRhZmZZnDDMzCyLE4aZmWVxwjAzsyxOGGZmlsUJw8zMsjhhtKnutfeye+CF3/lkt5nZseSEYWZmWZwwzMwsixOGmZllGTNhSDpe0sOSHpW0R9IXUvxrkp6VtCsti1Jckm6W1C/pMUlnlPa1QtLTaVlRir9H0u5U52ZJOhadNTOz8ct5vPkrwDkRUZV0HPADSfelbX8ZEXcOK78EmJ+Ws4BbgbMknQJcBfQAATwiaUtEHE5lPgE8RPHd3n3AfZiZWcsYc4QRhWp6e1xaYpQqy4DbU70HgZmSZgHnA9sj4lBKEtuBvrTtTRHxYEQEcDtwQQN9MjOzYyDrHoakKZJ2AQcp/ug/lDZdny473SRpeorNBvaVqu9PsdHi+2vEzcyshWR9415EvAYskjQTuEvSO4ErgZ8D04D1wBXANceqoQCSVgGrALq6uqhUKtl1q9VqXeVb3ZqFg3TNKF7L/VqzcPDo+n//xt1H1xfOPmkym9eQTjtXQ9yv9tKJ/apWq2MXGkVdX9EaEUckPQD0RcQXU/gVSX8HfCa9HwDmlqrNSbEBoHdYvJLic2qUr3X89RTJiZ6enujt7a1VrKZKpUI95VvdZWvvZc3CQW7cPZW9F/f+TryWcplW12nnaoj71V46sV+NJsCcWVKnp5EFkmYA7wd+nO49kGY0XQA8nqpsAS5Ns6UWAy9ExAFgG3CepJMlnQycB2xL216UtDjt61LgbszMrKXkjDBmARslTaFIMJsj4h5J35N0OiBgF/DnqfxWYCnQD7wMfAwgIg5JuhbYkcpdExGH0vonga8BMyhmR3mGlJlZixkzYUTEY8C7a8TPGaF8AKtH2LYB2FAjvhN451htMTOz5vEnvc3MLIsThpmZZXHCMDOzLE4YZmaWxQnDzMyyOGGYmVkWJwwzM8vihGFmZlmcMMzMLIsThpmZZXHCMDOzLHU93tyaq3uER5ebmU0GjzDMzCyLE4aZmWVxwjAzsyxOGGZmlsUJw8zMsjhhmJlZljEThqTjJT0s6VFJeyR9IcXnSXpIUr+kOyRNS/Hp6X1/2t5d2teVKf6UpPNL8b4U65e0duK7aWZmjcoZYbwCnBMR7wIWAX2SFgM3ADdFxNuAw8DKVH4lcDjFb0rlkLQAWA68A+gDvixpiqQpwC3AEmABcFEqa2ZmLWTMhBGFanp7XFoCOAe4M8U3Ahek9WXpPWn7uZKU4psi4pWIeBboB85MS39EPBMRrwKbUlkzM2shWfcw0khgF3AQ2A78FDgSEYOpyH5gdlqfDewDSNtfAE4tx4fVGSluZmYtJOvRIBHxGrBI0kzgLuAPjmmrRiBpFbAKoKuri0qlkl23Wq3WVb4VrVk4+Dvvu2YUsXK/hpcZ0k5974RzVYv71V46sV/VanXsQqOo61lSEXFE0gPAHwIzJU1No4g5wEAqNgDMBfZLmgqcBDxfig8p1xkpPvz464H1AD09PdHb25vd9kqlQj3lW9Flw54ltWbhIDfunsrei3tHLDOkXKbVdcK5qsX9ai+d2K9GE2DOLKnT08gCSTOA9wNPAg8AF6ZiK4C70/qW9J60/XsRESm+PM2imgfMBx4GdgDz06yraRQ3xrc01CszM5twOSOMWcDGNJvpDcDmiLhH0hPAJknXAT8CbkvlbwO+LqkfOESRAIiIPZI2A08Ag8DqdKkLSZcD24ApwIaI2DNhPTQzswkxZsKIiMeAd9eIP0Mxw2l4/NfAR0bY1/XA9TXiW4GtGe01M7Mm8Se9zcwsi79A6XWi/OVLe9d9oIktMbN25RGGmZllccIwM7MsThhmZpbFCcPMzLI4YZiZWRYnDDMzy+KEYWZmWZwwzMwsixOGmZll8Se9W1z3CI8rNzObbB5hmJlZFicMMzPL4oRhZmZZnDDMzCyLE4aZmWVxwjAzsyxjJgxJcyU9IOkJSXskfSrFr5Y0IGlXWpaW6lwpqV/SU5LOL8X7Uqxf0tpSfJ6kh1L8DknTJrqjnax77b1HFzOzYyVnhDEIrImIBcBiYLWkBWnbTRGxKC1bAdK25cA7gD7gy5KmSJoC3AIsARYAF5X2c0Pa19uAw8DKCeqfmZlNkDETRkQciIgfpvVfAk8Cs0epsgzYFBGvRMSzQD9wZlr6I+KZiHgV2AQskyTgHODOVH8jcMF4O2RmZsdGXfcwJHUD7wYeSqHLJT0maYOkk1NsNrCvVG1/io0UPxU4EhGDw+JmZtZCsh8NIulE4NvApyPiRUm3AtcCkV5vBP70mLTyt21YBawC6OrqolKpZNetVqt1lW8VaxYOjrita8bo20fS6v8O7XquxuJ+tZdO7Fe1Wm2oflbCkHQcRbL4RkR8ByAinitt/ypwT3o7AMwtVZ+TYowQfx6YKWlqGmWUy/+OiFgPrAfo6emJ3t7enOYDxR/Jesq3istGuZG9ZuEgN+6u/3Fgey/ubaBFx167nquxuF/tpRP71WgCzJklJeA24MmI+FIpPqtU7MPA42l9C7Bc0nRJ84D5wMPADmB+mhE1jeLG+JaICOAB4MJUfwVwd0O9MjOzCZfz39P3ApcAuyXtSrHPUsxyWkRxSWov8GcAEbFH0mbgCYoZVqsj4jUASZcD24ApwIaI2JP2dwWwSdJ1wI8oEpSZmbWQMRNGRPwAUI1NW0epcz1wfY341lr1IuIZillUZmbWovxJbzMzy+KEYWZmWZwwzMwsixOGmZllccIwM7MsThhmZpbFCcPMzLI4YZiZWRYnDDMzy+KEYWZmWZwwzMwsixOGmZllccIwM7Ms9X/7jrW97tKXMu1d94EmtsTM2okTRovpHuUb9szMmsmXpMzMLIsThpmZZXHCMDOzLE4YZmaWZcyEIWmupAckPSFpj6RPpfgpkrZLejq9npziknSzpH5Jj0k6o7SvFan805JWlOLvkbQ71blZUq3vEDczsybKGWEMAmsiYgGwGFgtaQGwFrg/IuYD96f3AEuA+WlZBdwKRYIBrgLOAs4ErhpKMqnMJ0r1+hrvmpmZTaQxE0ZEHIiIH6b1XwJPArOBZcDGVGwjcEFaXwbcHoUHgZmSZgHnA9sj4lBEHAa2A31p25si4sGICOD20r7MzKxF1PU5DEndwLuBh4CuiDiQNv0c6Errs4F9pWr7U2y0+P4a8VrHX0UxaqGrq4tKpZLd9mq1Wlf5ZlmzcDC7bNeM+srX0or/Ju1yrurlfrWXTuxXtVptqH52wpB0IvBt4NMR8WL5NkNEhKRoqCUZImI9sB6gp6cnent7s+tWKhXqKd8sl9Xxwb01Cwe5cXdjn73ce3FvQ/WPhXY5V/Vyv9pLJ/ar0QSYNUtK0nEUyeIbEfGdFH4uXU4ivR5M8QFgbqn6nBQbLT6nRtzMzFpIziwpAbcBT0bEl0qbtgBDM51WAHeX4pem2VKLgRfSpattwHmSTk43u88DtqVtL0panI51aWlfZmbWInKuZ7wXuATYLWlXin0WWAdslrQS+Bnw0bRtK7AU6AdeBj4GEBGHJF0L7EjlromIQ2n9k8DXgBnAfWkxM7MWMmbCiIgfACN9LuLcGuUDWD3CvjYAG2rEdwLvHKstZmbWPP6kt5mZZXHCMDOzLE4YZmaWxQnDzMyyOGGYmVkWJwwzM8vihGFmZlmcMMzMLIsThpmZZXHCMDOzLE4YZmaWxQnDzMyyNPbtOzYhuuv40iQzs2bxCMPMzLI4YZiZWRYnDDMzy+KEYWZmWZwwzMwsy5gJQ9IGSQclPV6KXS1pQNKutCwtbbtSUr+kpySdX4r3pVi/pLWl+DxJD6X4HZKmTWQHbXTda+89upiZjSZnhPE1oK9G/KaIWJSWrQCSFgDLgXekOl+WNEXSFOAWYAmwALgolQW4Ie3rbcBhYGUjHTIzs2NjzIQREd8HDmXubxmwKSJeiYhngX7gzLT0R8QzEfEqsAlYJknAOcCdqf5G4II6+2BmZpOgkQ/uXS7pUmAnsCYiDgOzgQdLZfanGMC+YfGzgFOBIxExWKP8PyNpFbAKoKuri0qlkt3YarVaV/nJtGbh4NiFauiaMf66tbTKv08rn6tGuF/tpRP7Va1WG6o/3oRxK3AtEOn1RuBPG2pJhohYD6wH6Onpid7e3uy6lUqFespPpsvGef9gzcJBbtw9cR/W33tx74TtqxGtfK4a4X61l07sV6MJcFx/bSLiuaF1SV8F7klvB4C5paJzUowR4s8DMyVNTaOMcnkzM2sh45pWK2lW6e2HgaEZVFuA5ZKmS5oHzAceBnYA89OMqGkUN8a3REQADwAXpvorgLvH0yYzMzu2xhxhSPoW0AucJmk/cBXQK2kRxSWpvcCfAUTEHkmbgSeAQWB1RLyW9nM5sA2YAmyIiD3pEFcAmyRdB/wIuG3CemdmZhNmzIQRERfVCI/4Rz0irgeurxHfCmytEX+GYhaVmZm1MH/S28zMsjhhmJlZFicMMzPL4oRhZmZZnDDMzCyLE4aZmWVxwjAzsyxOGGZmlsUJw8zMsjhhmJlZFicMMzPL4oRhZmZZnDDMzCyLE4aZmWVxwjAzsyxOGGZmlsUJw8zMsjhhmJlZljEThqQNkg5KerwUO0XSdklPp9eTU1ySbpbUL+kxSWeU6qxI5Z+WtKIUf4+k3anOzZI00Z00M7PG5Ywwvgb0DYutBe6PiPnA/ek9wBJgflpWAbdCkWCAq4CzKL6/+6qhJJPKfKJUb/ixzMysBYyZMCLi+8ChYeFlwMa0vhG4oBS/PQoPAjMlzQLOB7ZHxKGIOAxsB/rStjdFxIMREcDtpX2ZmVkLmTrOel0RcSCt/xzoSuuzgX2lcvtTbLT4/hrxmiStohi50NXVRaVSyW5wtVqtq/xkWrNwcFz1umaMv24trfLv08rnqhHuV3vpxH5Vq9WG6o83YRwVESEpGt1P5rHWA+sBenp6ore3N7tupVKhnvKT6bK1946r3pqFg9y4u+FTeNTei3snbF+NaOVz1Qj3q710Yr8aTYDj/WvznKRZEXEgXVY6mOIDwNxSuTkpNgD0DotXUnxOjfIdqbuUGPau+0ATW2JmVr/xTqvdAgzNdFoB3F2KX5pmSy0GXkiXrrYB50k6Od3sPg/Ylra9KGlxmh11aWlf1kTda+89upiZQcYIQ9K3KEYHp0naTzHbaR2wWdJK4GfAR1PxrcBSoB94GfgYQEQcknQtsCOVuyYihm6kf5JiJtYM4L60mJlZixkzYUTERSNsOrdG2QBWj7CfDcCGGvGdwDvHaoeZmTWXP+ltZmZZnDDMzCyLE4aZmWVxwjAzsywT96kvq4unq5pZu/EIw8zMsniEYUd51GNmo/EIw8zMsniEYWPyM7DMDDzCMDOzTE4YZmaWxZekjjHfSDazTuGEYXXx/Qyz1y9fkjIzsyxOGGZmlsUJw8zMsvgeho2b72eYvb54hGFmZlkaShiS9kraLWmXpJ0pdoqk7ZKeTq8np7gk3SypX9Jjks4o7WdFKv+0pBWNdcnMzI6Fibgk9b6I+EXp/Vrg/ohYJ2lten8FsASYn5azgFuBsySdAlwF9AABPCJpS0QcnoC22STx5SmzzncsLkktAzam9Y3ABaX47VF4EJgpaRZwPrA9Ig6lJLEd6DsG7TIzswYoIsZfWXoWOEwxMvibiFgv6UhEzEzbBRyOiJmS7gHWRcQP0rb7KUYevcDxEXFdin8e+FVEfLHG8VYBqwC6urres2nTpuy2VqtVTjzxxHH3tR67B16YlON0zYDnfjUph6rLwtknNVR/Ms/VZHK/2ksn9qtarfKhD33okYjoGU/9Ri9JnR0RA5LeDGyX9OPyxogISePPSMNExHpgPUBPT0/09vZm161UKtRTvhGXTdLjQNYsHOTG3a030W3vxb0N1Z/MczWZ3K/20on9qlQqDdVv6K9NRAyk14OS7gLOBJ6TNCsiDqRLTgdT8QFgbqn6nBQboBhllOOVRtplrcP3Nsw6x7jvYUg6QdIbh9aB84DHgS3A0EynFcDdaX0LcGmaLbUYeCEiDgDbgPMknZxmVJ2XYmZm1kIaGWF0AXcVtymYCnwzIv5R0g5gs6SVwM+Aj6byW4GlQD/wMvAxgIg4JOlaYEcqd01EHGqgXWZmdgyMO2FExDPAu2rEnwfOrREPYPUI+9oAbBhvW6w9+PKUWXvzJ73NzCxL602xaVP+oqT6eLRh1n48wjAzsyweYdiE82jLrDM5YVjT+fKUWXvwJSkzM8viEUYDfOll4nWvvZc1Cwe5bO29Hm2YtRiPMMzMLIsThpmZZfElKWsLwy//+XKV2eTzCMPMzLJ4hGFtyVNxzSafE4a1rNxZaE4eZpPDCaNOnkprZq9XThjWUTzaMDt2nDAyeFTRnpw8zCaWZ0mZmVkWjzDsdWekEaNHIWaja5mEIakP+GtgCvC3EbGuyU2yDpJzWdGJxGx0LZEwJE0BbgHeD+wHdkjaEhFPNKtNvm9hQ0a6F+J7JPZ60xIJAzgT6I+IZwAkbQKWAcc8YfiX3uox0n8kcv6DMfQU3lz+ebRWo4hodhuQdCHQFxEfT+8vAc6KiMuHlVsFrEpv3w48VcdhTgN+MQHNbSWd2Cdwv9qN+9U+TgNOiIjTx1O5VUYYWSJiPbB+PHUl7YyIngluUlN1Yp/A/Wo37lf7SH3qHm/9VplWOwDMLb2fk2JmZtYiWiVh7ADmS5onaRqwHNjS5DaZmVlJS1ySiohBSZcD2yim1W6IiD0TfJhxXcpqcZ3YJ3C/2o371T4a6lNL3PQ2M7PW1yqXpMzMrMU5YZiZWZaOTxiS+iQ9Jalf0tpmt6cRkvZK2i1pl6SdKXaKpO2Snk6vJze7nWORtEHSQUmPl2I1+6HCzen8PSbpjOa1fHQj9OtqSQPpnO2StLS07crUr6cknd+cVo9O0lxJD0h6QtIeSZ9K8bY+X6P0q93P1/GSHpb0aOrXF1J8nqSHUvvvSJOLkDQ9ve9P27tHPUBEdOxCcQP9p8BbgWnAo8CCZrergf7sBU4bFvuvwNq0vha4odntzOjHHwFnAI+P1Q9gKXAfIGAx8FCz219nv64GPlOj7IL08zgdmJd+Tqc0uw812jkLOCOtvxH4SWp7W5+vUfrV7udLwIlp/TjgoXQeNgPLU/wrwH9I658EvpLWlwN3jLb/Th9hHH3kSES8Cgw9cqSTLAM2pvWNwAVNbEuWiPg+cGhYeKR+LANuj8KDwExJsyanpfUZoV8jWQZsiohXIuJZoJ/i57WlRMSBiPhhWv8l8CQwmzY/X6P0ayTtcr4iIqrp7XFpCeAc4M4UH36+hs7jncC5kjTS/js9YcwG9pXe72f0H4pWF8B3JT2SHpMC0BURB9L6z4Gu5jStYSP1oxPO4eXp8syG0iXDtutXulzxbor/tXbM+RrWL2jz8yVpiqRdwEFgO8Vo6EhEDKYi5bYf7Vfa/gJw6kj77vSE0WnOjogzgCXAakl/VN4Yxbiy7edJd0o/kluBfwEsAg4ANza3OeMj6UTg28CnI+LF8rZ2Pl81+tX25ysiXouIRRRPzDgT+IOJ2nenJ4yOeuRIRAyk14PAXRQ/DM8NDfnT68HmtbAhI/Wjrc9hRDyXfoF/A3yV317GaJt+STqO4o/qNyLiOync9uerVr864XwNiYgjwAPAH1JcGhz6oHa57Uf7lbafBDw/0j47PWF0zCNHJJ0g6Y1D68B5wOMU/VmRiq0A7m5OCxs2Uj+2AJem2TeLgRdKl0Ja3rDr9x+mOGdQ9Gt5mqUyD5gPPDzZ7RtLup59G/BkRHyptKmtz9dI/eqA83W6pJlpfQbFdww9SZE4LkzFhp+vofN4IfC9NGKsrdl39Y/1QjFr4ycU1/E+1+z2NNCPt1LM0ngU2DPUF4rrjfcDTwP/BJzS7LZm9OVbFMP9/0dxPXXlSP2gmPVxSzp/u4GeZre/zn59PbX7sfTLOatU/nOpX08BS5rd/hH6dDbF5abHgF1pWdru52uUfrX7+fqXwI9S+x8H/irF30qR4PqBfwCmp/jx6X1/2v7W0fbvR4OYmVmWTr8kZWZmE8QJw8zMsjhhmJlZFicMMzPL4oRhZmZZnDDMzCyLE4aZmWX5//gVXvZu6iokAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "test.hist(column='seq_length',bins = 100)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": [
    "import pickle"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": [
    "rr_file = open('./search_data/new_bhokal_df','rb')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "hidden": true,
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "sorted_train_data = pickle.load(rr_file)\n",
    "# pd.read_csv('./search_data/new/sorted_train.csv')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Main methods to process input data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "code_folding": [
     0
    ]
   },
   "outputs": [],
   "source": [
    "def convert_examples_to_features_test(examples, tokenizer, max_seq_length):\n",
    "    \"\"\"Loads a data file into a list of `InputBatch`s.\"\"\"\n",
    "\n",
    "    features = []\n",
    "#     for example_index, example in examples.iterrows():\n",
    "    for example_index in range(0,examples.shape[0],10):\n",
    "        \n",
    "        example = examples.iloc[example_index]\n",
    "        context_tokens = tokenizer.tokenize(example['query'])\n",
    "        start_ending_tokens = tokenizer.tokenize('')\n",
    "        \n",
    "        choices_features = []\n",
    "        one_ques = examples[example_index:example_index+10]\n",
    "        \n",
    "        #since this is test data\n",
    "        label = None\n",
    "        \n",
    "        endings = one_ques['passage_text']\n",
    "        for ending_index, ending in enumerate(endings):\n",
    "            # We create a copy of the context tokens in order to be\n",
    "            # able to shrink it according to ending_tokens\n",
    "            context_tokens_choice = context_tokens[:]\n",
    "            ending_tokens = start_ending_tokens + tokenizer.tokenize(ending)\n",
    "            # Modifies `context_tokens_choice` and `ending_tokens` in\n",
    "            # place so that the total length is less than the\n",
    "            # specified length.  Account for [CLS], [SEP], [SEP] with\n",
    "            # \"- 3\"\n",
    "            _truncate_seq_pair(context_tokens_choice, ending_tokens, max_seq_length-3)\n",
    "\n",
    "            tokens = [\"[CLS]\"] + context_tokens_choice + [\"[SEP]\"] + ending_tokens + [\"[SEP]\"]\n",
    "            segment_ids = [0] * (len(context_tokens_choice) + 2) + [1] * (len(ending_tokens) + 1)\n",
    "\n",
    "            input_ids = tokenizer.convert_tokens_to_ids(tokens)\n",
    "            input_mask = [1] * len(input_ids)\n",
    "\n",
    "            # Zero-pad up to the sequence length.\n",
    "            padding = [0] * (max_seq_length - len(input_ids))\n",
    "            input_ids += padding\n",
    "            input_mask += padding\n",
    "            segment_ids += padding\n",
    "\n",
    "            assert len(input_ids) == max_seq_length\n",
    "            assert len(input_mask) == max_seq_length\n",
    "            assert len(segment_ids) == max_seq_length\n",
    "\n",
    "            choices_features.append((tokens, input_ids, input_mask, segment_ids))\n",
    "#         if(example.query_id%5000==0):\n",
    "#             print(example.query_id)\n",
    "#         if example_index < 5:\n",
    "#             logger.info(\"*** Example ***\")\n",
    "#             logger.info(f\"query_id: {example.query_id}\")\n",
    "#             for choice_idx, (tokens, input_ids, input_mask, segment_ids) in enumerate(choices_features):\n",
    "#                 logger.info(f\"choice: {choice_idx}\")\n",
    "#                 logger.info(f\"tokens: {' '.join(tokens)}\")\n",
    "#                 logger.info(f\"input_ids: {' '.join(map(str, input_ids))}\")\n",
    "#                 logger.info(f\"input_mask: {' '.join(map(str, input_mask))}\")\n",
    "#                 logger.info(f\"segment_ids: {' '.join(map(str, segment_ids))}\")\n",
    "        if example_index % 1000 == 0:\n",
    "            print (example_index)\n",
    "            \n",
    "        features.append(\n",
    "            InputFeatures(\n",
    "                example_id = example.query_id,\n",
    "                choices_features = choices_features,\n",
    "                label = label\n",
    "            )\n",
    "        )\n",
    "\n",
    "    return features\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "code_folding": [
     0
    ]
   },
   "outputs": [],
   "source": [
    "def convert_examples_to_features(examples, tokenizer, max_seq_length,\n",
    "                                 is_training):\n",
    "    \"\"\"Loads a data file into a list of `InputBatch`s.\"\"\"\n",
    "\n",
    "    # Swag is a multiple choice task. To perform this task using Bert,\n",
    "    # we will use the formatting proposed in \"Improving Languagea\n",
    "    # Understanding by Generative Pre-Training\" and suggested by\n",
    "    # @jacobdevlin-google in this issue\n",
    "    # https://github.com/google-research/bert/issues/38.\n",
    "    #\n",
    "    # Each choice will correspond to a sample on which we run the\n",
    "    # inference. For a given Swag example, we will create the 4\n",
    "    # following inputs:\n",
    "    # - [CLS] context [SEP] choice_1 [SEP]\n",
    "    # - [CLS] context [SEP] choice_2 [SEP]\n",
    "    # - [CLS] context [SEP] choice_3 [SEP]\n",
    "    # - [CLS] context [SEP] choice_4 [SEP]\n",
    "    # The model will output a single value for each input. To get the\n",
    "    # final decision of the model, we will run a softmax over these 4\n",
    "    # outputs.\n",
    "    features = []\n",
    "#     for example_index, example in examples.iterrows():\n",
    "    for example_index in range(0,examples.shape[0],10):\n",
    "        \n",
    "        example = examples.iloc[example_index]\n",
    "        context_tokens = tokenizer.tokenize(example['query'])\n",
    "        start_ending_tokens = tokenizer.tokenize('')\n",
    "        \n",
    "        choices_features = []\n",
    "        one_ques = examples[example_index:example_index+10]\n",
    "        \n",
    "        label = None\n",
    "        \n",
    "        if is_training:\n",
    "            \n",
    "            try:\n",
    "                label = one_ques[one_ques.label==1].iloc[0]['passage_id']\n",
    "            except Exception as e:\n",
    "                print(e)\n",
    "                print(one_ques)\n",
    "        \n",
    "        endings = one_ques['passage_text']\n",
    "        for ending_index, ending in enumerate(endings):\n",
    "            # We create a copy of the context tokens in order to be\n",
    "            # able to shrink it according to ending_tokens\n",
    "            context_tokens_choice = context_tokens[:]\n",
    "            ending_tokens = start_ending_tokens + tokenizer.tokenize(ending)\n",
    "            # Modifies `context_tokens_choice` and `ending_tokens` in\n",
    "            # place so that the total length is less than the\n",
    "            # specified length.  Account for [CLS], [SEP], [SEP] with\n",
    "            # \"- 3\"\n",
    "            _truncate_seq_pair(context_tokens_choice, ending_tokens, max_seq_length-3)\n",
    "\n",
    "            tokens = [\"[CLS]\"] + context_tokens_choice + [\"[SEP]\"] + ending_tokens + [\"[SEP]\"]\n",
    "            segment_ids = [0] * (len(context_tokens_choice) + 2) + [1] * (len(ending_tokens) + 1)\n",
    "\n",
    "            input_ids = tokenizer.convert_tokens_to_ids(tokens)\n",
    "            input_mask = [1] * len(input_ids)\n",
    "\n",
    "            # Zero-pad up to the sequence length.\n",
    "            padding = [0] * (max_seq_length - len(input_ids))\n",
    "            input_ids += padding\n",
    "            input_mask += padding\n",
    "            segment_ids += padding\n",
    "\n",
    "            assert len(input_ids) == max_seq_length\n",
    "            assert len(input_mask) == max_seq_length\n",
    "            assert len(segment_ids) == max_seq_length\n",
    "\n",
    "            choices_features.append((tokens, input_ids, input_mask, segment_ids))\n",
    "#         if(example.query_id%5000==0):\n",
    "#             print(example.query_id)\n",
    "        if example_index < 5:\n",
    "            logger.info(\"*** Example ***\")\n",
    "            logger.info(f\"query_id: {example.query_id}\")\n",
    "            for choice_idx, (tokens, input_ids, input_mask, segment_ids) in enumerate(choices_features):\n",
    "                logger.info(f\"choice: {choice_idx}\")\n",
    "                logger.info(f\"tokens: {' '.join(tokens)}\")\n",
    "                logger.info(f\"input_ids: {' '.join(map(str, input_ids))}\")\n",
    "                logger.info(f\"input_mask: {' '.join(map(str, input_mask))}\")\n",
    "                logger.info(f\"segment_ids: {' '.join(map(str, segment_ids))}\")\n",
    "            if is_training:\n",
    "                logger.info(f\"label: {label}\")\n",
    "\n",
    "        features.append(\n",
    "            InputFeatures(\n",
    "                example_id = example.query_id,\n",
    "                choices_features = choices_features,\n",
    "                label = label\n",
    "            )\n",
    "        )\n",
    "\n",
    "    return features\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "code_folding": [
     0,
     16
    ]
   },
   "outputs": [],
   "source": [
    "def _truncate_seq_pair(tokens_a, tokens_b, max_length):\n",
    "    \"\"\"Truncates a sequence pair in place to the maximum length.\"\"\"\n",
    "\n",
    "    # This is a simple heuristic which will always truncate the longer sequence\n",
    "    # one token at a time. This makes more sense than truncating an equal percent\n",
    "    # of tokens from each, since if one sequence is very short then each token\n",
    "    # that's truncated likely contains more information than a longer sequence.\n",
    "    while True:\n",
    "        total_length = len(tokens_a) + len(tokens_b)\n",
    "        if total_length <= max_length:\n",
    "            break\n",
    "        if len(tokens_a) > len(tokens_b):\n",
    "            tokens_a.pop()\n",
    "        else:\n",
    "            tokens_b.pop()\n",
    "\n",
    "def select_field(features, field):\n",
    "    return [\n",
    "        [\n",
    "            choice[field]\n",
    "            for choice in feature.choices_features\n",
    "        ]\n",
    "        for feature in features\n",
    "    ]\n",
    "\n",
    "def warmup_linear(x, warmup=0.002):\n",
    "    if x < warmup:\n",
    "        return x/warmup\n",
    "    return 1.0 - x"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Evaluation functions - Accuracy and Mean Reciprocal Rank"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def accuracy(out, labels):\n",
    "    outputs = np.argmax(out, axis=1)\n",
    "    return np.sum(outputs == labels)\n",
    "\n",
    "\n",
    "def get_mrr2(logits,labels):\n",
    "    \n",
    "#     ranks = np.argsort(logits,axis=1)[::-1]\n",
    "    sum1 = 0.0\n",
    "    count = 0\n",
    "    for i in range(len(labels)):\n",
    "        rank = np.argsort(logits[i])[::-1]\n",
    "        l = labels[i]\n",
    "#     for l in labels:\n",
    "        sum1 +=1/(np.where(rank==l)[0][0]+1)\n",
    "        count+=1\n",
    "        \n",
    "        \n",
    "    return sum1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "class args:\n",
    "    \n",
    "    bert_model = \"bert-base-uncased\"\n",
    "    do_train = True\n",
    "    do_lower_case = True\n",
    "    do_eval = True\n",
    "    #data_dir = \"./search_data/swagaf/data/\"\n",
    "    train_batch_size = 240\n",
    "    learning_rate = 2e-5\n",
    "    num_train_epochs = 1.0\n",
    "    max_seq_length = 290\n",
    "    output_dir = \"./search_data/swagaf/output_data_all/\"\n",
    "    gradient_accumulation_steps = 10\n",
    "    \n",
    "    ### default attribs\n",
    "    local_rank = -1\n",
    "    no_cuda = False\n",
    "    seed = 42\n",
    "    fp16  = False\n",
    "    warmup_proportion = 0.1\n",
    "    loss_scale = 0\n",
    "    eval_batch_size = 30\n",
    "    \n",
    "    \n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(240, 10, 1.0)"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "args.train_batch_size , args.gradient_accumulation_steps ,args.num_train_epochs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "24.0"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "args.train_batch_size / args.gradient_accumulation_steps"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "if args.local_rank == -1 or args.no_cuda:\n",
    "    device = torch.device(\"cuda\" if torch.cuda.is_available() and not args.no_cuda else \"cpu\")\n",
    "    n_gpu = torch.cuda.device_count()\n",
    "else:\n",
    "    torch.cuda.set_device(args.local_rank)\n",
    "    device = torch.device(\"cuda\", args.local_rank)\n",
    "    n_gpu = 1\n",
    "    # Initializes the distributed backend which will take care of sychronizing nodes/GPUs\n",
    "    torch.distributed.init_process_group(backend='nccl')\n",
    "logger.info(\"device: {} n_gpu: {}, distributed training: {}, 16-bits training: {}\".format(\n",
    "    device, n_gpu, bool(args.local_rank != -1), args.fp16))\n",
    "\n",
    "if args.gradient_accumulation_steps < 1:\n",
    "    raise ValueError(\"Invalid gradient_accumulation_steps parameter: {}, should be >= 1\".format(\n",
    "                        args.gradient_accumulation_steps))\n",
    "\n",
    "args.train_batch_size = int(args.train_batch_size / args.gradient_accumulation_steps)\n",
    "\n",
    "random.seed(args.seed)\n",
    "np.random.seed(args.seed)\n",
    "torch.manual_seed(args.seed)\n",
    "if n_gpu > 0:\n",
    "    torch.cuda.manual_seed_all(args.seed)\n",
    "\n",
    "if not args.do_train and not args.do_eval:\n",
    "    raise ValueError(\"At least one of `do_train` or `do_eval` must be True.\")\n",
    "\n",
    "if os.path.exists(args.output_dir) and os.listdir(args.output_dir):\n",
    "    raise ValueError(\"Output directory ({}) already exists and is not empty.\".format(args.output_dir))\n",
    "os.makedirs(args.output_dir, exist_ok=True)\n",
    "\n",
    "tokenizer = BertTokenizer.from_pretrained(args.bert_model, do_lower_case=args.do_lower_case)\n",
    "\n",
    "train_examples = None\n",
    "num_train_steps = None"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Run when in training mode"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "12/29/2018 09:41:34 - INFO - __main__ -   device: cuda n_gpu: 8, distributed training: False, 16-bits training: False\n",
      "12/29/2018 09:41:34 - INFO - pytorch_pretrained_bert.file_utils -   https://s3.amazonaws.com/models.huggingface.co/bert/bert-base-uncased-vocab.txt not found in cache, downloading to /tmp/tmpwlpencui\n",
      "100%|██████████| 231508/231508 [00:00<00:00, 27068881.87B/s]\n",
      "12/29/2018 09:41:34 - INFO - pytorch_pretrained_bert.file_utils -   copying /tmp/tmpwlpencui to cache at /home/ec2-user/.pytorch_pretrained_bert/26bc1ad6c0ac742e9b52263248f6d0f00068293b33709fae12320c0e35ccfbbb.542ce4285a40d23a559526243235df47c5f75c197f04f37d1a0c124c32c9a084\n",
      "12/29/2018 09:41:34 - INFO - pytorch_pretrained_bert.file_utils -   creating metadata file for /home/ec2-user/.pytorch_pretrained_bert/26bc1ad6c0ac742e9b52263248f6d0f00068293b33709fae12320c0e35ccfbbb.542ce4285a40d23a559526243235df47c5f75c197f04f37d1a0c124c32c9a084\n",
      "12/29/2018 09:41:34 - INFO - pytorch_pretrained_bert.file_utils -   removing temp file /tmp/tmpwlpencui\n",
      "12/29/2018 09:41:34 - INFO - pytorch_pretrained_bert.tokenization -   loading vocabulary file https://s3.amazonaws.com/models.huggingface.co/bert/bert-base-uncased-vocab.txt from cache at /home/ec2-user/.pytorch_pretrained_bert/26bc1ad6c0ac742e9b52263248f6d0f00068293b33709fae12320c0e35ccfbbb.542ce4285a40d23a559526243235df47c5f75c197f04f37d1a0c124c32c9a084\n"
     ]
    }
   ],
   "source": [
    "# if args.do_train:\n",
    "# #     train_examples_orig = read_swag_examples(os.path.join(args.data_dir, 'train.csv'), is_training = True)\n",
    "#     train_examples_orig = sorted_train_data\n",
    "#     print (\"Total Training Examples length \" + str(len(train_examples_orig)))\n",
    "    \n",
    "#     # taking only 100 examples for now\n",
    "#     train_examples = train_examples_orig\n",
    "#     num_train_steps = int(\n",
    "#         len(train_examples) / args.train_batch_size / args.gradient_accumulation_steps * args.num_train_epochs)\n",
    "#     print (\"Taking Training Examples length \" + str(len(train_examples)))\n",
    "#     print (\"Num Train Steps \" + str(num_train_steps))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1197.8333333333335"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "num_train_steps = 287480 / args.train_batch_size / args.gradient_accumulation_steps * args.num_train_epochs\n",
    "num_train_steps"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "# train_examples = sorted_train_data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "# from apex.parallel import DistributedDataParallel as DDP"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Load BERT Model , use one of the two options"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "12/29/2018 10:14:43 - INFO - pytorch_pretrained_bert.modeling -   loading archive file https://s3.amazonaws.com/models.huggingface.co/bert/bert-base-uncased.tar.gz from cache at /home/ec2-user/.pytorch_pretrained_bert/distributed_-1/9c41111e2de84547a463fd39217199738d1e3deb72d4fec4399e6e241983c6f0.ae3cef932725ca7a30cdcb93fc6e09150a55e2a130ec7af63975a16c153ae2ba\n",
      "12/29/2018 10:14:43 - INFO - pytorch_pretrained_bert.modeling -   extracting archive file /home/ec2-user/.pytorch_pretrained_bert/distributed_-1/9c41111e2de84547a463fd39217199738d1e3deb72d4fec4399e6e241983c6f0.ae3cef932725ca7a30cdcb93fc6e09150a55e2a130ec7af63975a16c153ae2ba to temp dir /tmp/tmp2pwlzyok\n",
      "12/29/2018 10:14:47 - INFO - pytorch_pretrained_bert.modeling -   Model config {\n",
      "  \"attention_probs_dropout_prob\": 0.1,\n",
      "  \"hidden_act\": \"gelu\",\n",
      "  \"hidden_dropout_prob\": 0.1,\n",
      "  \"hidden_size\": 768,\n",
      "  \"initializer_range\": 0.02,\n",
      "  \"intermediate_size\": 3072,\n",
      "  \"max_position_embeddings\": 512,\n",
      "  \"num_attention_heads\": 12,\n",
      "  \"num_hidden_layers\": 12,\n",
      "  \"type_vocab_size\": 2,\n",
      "  \"vocab_size\": 30522\n",
      "}\n",
      "\n",
      "12/29/2018 10:14:51 - INFO - pytorch_pretrained_bert.modeling -   Weights of BertForMultipleChoice not initialized from pretrained model: ['classifier.weight', 'classifier.bias']\n",
      "12/29/2018 10:14:51 - INFO - pytorch_pretrained_bert.modeling -   Weights from pretrained model not used in BertForMultipleChoice: ['cls.predictions.bias', 'cls.predictions.transform.dense.weight', 'cls.predictions.transform.dense.bias', 'cls.predictions.decoder.weight', 'cls.seq_relationship.weight', 'cls.seq_relationship.bias', 'cls.predictions.transform.LayerNorm.weight', 'cls.predictions.transform.LayerNorm.bias']\n"
     ]
    }
   ],
   "source": [
    "# use google provided models \n",
    "model = BertForMultipleChoice.from_pretrained(args.bert_model,\n",
    "    cache_dir=PYTORCH_PRETRAINED_BERT_CACHE / 'distributed_{}'.format(args.local_rank),\n",
    "    num_choices=10)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 93,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "12/29/2018 10:45:11 - INFO - pytorch_pretrained_bert.modeling -   loading archive file https://s3.amazonaws.com/models.huggingface.co/bert/bert-base-uncased.tar.gz from cache at /home/ec2-user/.pytorch_pretrained_bert/9c41111e2de84547a463fd39217199738d1e3deb72d4fec4399e6e241983c6f0.ae3cef932725ca7a30cdcb93fc6e09150a55e2a130ec7af63975a16c153ae2ba\n",
      "12/29/2018 10:45:11 - INFO - pytorch_pretrained_bert.modeling -   extracting archive file /home/ec2-user/.pytorch_pretrained_bert/9c41111e2de84547a463fd39217199738d1e3deb72d4fec4399e6e241983c6f0.ae3cef932725ca7a30cdcb93fc6e09150a55e2a130ec7af63975a16c153ae2ba to temp dir /tmp/tmp5vev9031\n",
      "12/29/2018 10:45:15 - INFO - pytorch_pretrained_bert.modeling -   Model config {\n",
      "  \"attention_probs_dropout_prob\": 0.1,\n",
      "  \"hidden_act\": \"gelu\",\n",
      "  \"hidden_dropout_prob\": 0.1,\n",
      "  \"hidden_size\": 768,\n",
      "  \"initializer_range\": 0.02,\n",
      "  \"intermediate_size\": 3072,\n",
      "  \"max_position_embeddings\": 512,\n",
      "  \"num_attention_heads\": 12,\n",
      "  \"num_hidden_layers\": 12,\n",
      "  \"type_vocab_size\": 2,\n",
      "  \"vocab_size\": 30522\n",
      "}\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# OR Load a trained model that you have fine-tuned\n",
    "model_state_dict = torch.load('./search_data/swagaf/output_data_all_160_epoch_2/pytorch_model.bin')\n",
    "model = BertForMultipleChoice.from_pretrained(args.bert_model,\n",
    "    state_dict=model_state_dict,\n",
    "    num_choices=10)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "if args.fp16:\n",
    "    model.half()\n",
    "\n",
    "model.to(device)\n",
    "if args.local_rank != -1:\n",
    "    try:\n",
    "        from apex.parallel import DistributedDataParallel as DDP\n",
    "    except ImportError:\n",
    "        raise ImportError(\"Please install apex from https://www.github.com/nvidia/apex to use distributed and fp16 training.\")\n",
    "\n",
    "    model = DDP(model)\n",
    "elif n_gpu > 1:\n",
    "    model = torch.nn.DataParallel(model)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Prepare optimizer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 94,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "param_optimizer = list(model.named_parameters())\n",
    "\n",
    "# hack to remove pooler, which is not used\n",
    "# thus it produce None grad that break apex\n",
    "param_optimizer = [n for n in param_optimizer if 'pooler' not in n[0]]\n",
    "\n",
    "no_decay = ['bias', 'LayerNorm.bias', 'LayerNorm.weight']\n",
    "optimizer_grouped_parameters = [\n",
    "    {'params': [p for n, p in param_optimizer if not any(nd in n for nd in no_decay)], 'weight_decay': 0.01},\n",
    "    {'params': [p for n, p in param_optimizer if any(nd in n for nd in no_decay)], 'weight_decay': 0.0}\n",
    "    ]\n",
    "t_total = num_train_steps\n",
    "if args.local_rank != -1:\n",
    "    t_total = t_total // torch.distributed.get_world_size()\n",
    "if args.fp16:\n",
    "    try:\n",
    "        from apex.optimizers import FP16_Optimizer\n",
    "        from apex.optimizers import FusedAdam\n",
    "    except ImportError:\n",
    "        raise ImportError(\"Please install apex from https://www.github.com/nvidia/apex to use distributed and fp16 training.\")\n",
    "\n",
    "    optimizer = FusedAdam(optimizer_grouped_parameters,\n",
    "                          lr=args.learning_rate,\n",
    "                          bias_correction=False,\n",
    "                          max_grad_norm=1.0)\n",
    "    if args.loss_scale == 0:\n",
    "        optimizer = FP16_Optimizer(optimizer, dynamic_loss_scale=True)\n",
    "    else:\n",
    "        optimizer = FP16_Optimizer(optimizer, static_loss_scale=args.loss_scale)\n",
    "else:\n",
    "    optimizer = BertAdam(optimizer_grouped_parameters,\n",
    "                         lr=args.learning_rate,\n",
    "                         warmup=args.warmup_proportion,\n",
    "                         t_total=t_total)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Trying Multi-threading for Input processing\n",
    "\n",
    "Since the training data was huge (5.2 Million rows) we tried multi-thread processing for converting raw-text to our specified input format (specifically to  InputFeatures class). But this was taking extravagant memory and made the notebook unreponsive even on 32 Cores, 200 GB Memory machine.  \n",
    "\n",
    "### Eventually we used df.apply() which processed 5.2 Million rows in ~1.5 hours on single thread"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#testing\n",
    "from multiprocessing import Pool\n",
    "\n",
    "def f(x):\n",
    "    return [x*x]\n",
    "\n",
    "with Pool(5) as p:\n",
    "        nums = p.map(f, [1, 2, 3])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "features_list = []"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "code_folding": []
   },
   "outputs": [],
   "source": [
    "def convert_examples_to_features_parellel_test(train_examples):\n",
    "    train_features = convert_examples_to_features(\n",
    "        train_examples, tokenizer, args.max_seq_length, False)\n",
    "    return train_features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def convert_examples_to_features_parellel(train_examples,is_training):\n",
    "    train_features = convert_examples_to_features(\n",
    "        train_examples, tokenizer, args.max_seq_length, is_training)\n",
    "    return train_features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [],
   "source": [
    "import time"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1546059811.0388947"
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "time.time()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0\n",
      "1000\n",
      "2000\n",
      "3000\n",
      "4000\n",
      "5000\n",
      "6000\n",
      "7000\n",
      "8000\n",
      "9000\n",
      "10000\n",
      "11000\n",
      "12000\n",
      "13000\n",
      "14000\n",
      "15000\n",
      "16000\n",
      "17000\n",
      "18000\n",
      "19000\n",
      "20000\n",
      "21000\n",
      "22000\n",
      "23000\n",
      "24000\n",
      "25000\n",
      "26000\n",
      "27000\n",
      "28000\n",
      "29000\n",
      "30000\n",
      "31000\n",
      "32000\n",
      "33000\n",
      "34000\n",
      "35000\n",
      "36000\n",
      "37000\n",
      "38000\n",
      "39000\n",
      "40000\n",
      "41000\n",
      "42000\n",
      "43000\n",
      "44000\n",
      "45000\n",
      "46000\n",
      "47000\n",
      "48000\n",
      "49000\n",
      "50000\n",
      "51000\n",
      "52000\n",
      "53000\n",
      "54000\n",
      "55000\n",
      "56000\n",
      "57000\n",
      "58000\n",
      "59000\n",
      "60000\n",
      "61000\n",
      "62000\n",
      "63000\n",
      "64000\n",
      "65000\n",
      "66000\n",
      "67000\n",
      "68000\n",
      "69000\n",
      "70000\n",
      "71000\n",
      "72000\n",
      "73000\n",
      "74000\n",
      "75000\n",
      "76000\n",
      "77000\n",
      "78000\n",
      "79000\n",
      "80000\n",
      "81000\n",
      "82000\n",
      "83000\n",
      "84000\n",
      "85000\n",
      "86000\n",
      "87000\n",
      "88000\n",
      "89000\n",
      "90000\n",
      "91000\n",
      "92000\n",
      "93000\n",
      "94000\n",
      "95000\n",
      "96000\n",
      "97000\n",
      "98000\n",
      "99000\n",
      "100000\n",
      "101000\n",
      "102000\n",
      "103000\n",
      "104000\n",
      "105000\n",
      "106000\n",
      "107000\n",
      "108000\n",
      "109000\n",
      "110000\n",
      "111000\n",
      "112000\n",
      "113000\n",
      "114000\n",
      "115000\n",
      "116000\n",
      "117000\n",
      "118000\n",
      "119000\n",
      "120000\n",
      "121000\n",
      "122000\n",
      "123000\n",
      "124000\n",
      "125000\n",
      "126000\n",
      "127000\n",
      "128000\n",
      "129000\n",
      "130000\n",
      "131000\n",
      "132000\n",
      "133000\n",
      "134000\n",
      "135000\n",
      "136000\n",
      "137000\n",
      "138000\n",
      "139000\n",
      "140000\n",
      "141000\n",
      "142000\n",
      "143000\n",
      "144000\n",
      "145000\n",
      "146000\n",
      "147000\n",
      "148000\n",
      "149000\n",
      "150000\n",
      "151000\n",
      "152000\n",
      "153000\n",
      "154000\n",
      "155000\n",
      "156000\n",
      "157000\n",
      "158000\n",
      "159000\n",
      "160000\n",
      "161000\n",
      "162000\n",
      "163000\n",
      "164000\n",
      "165000\n",
      "166000\n",
      "167000\n",
      "168000\n",
      "169000\n",
      "170000\n",
      "171000\n",
      "172000\n",
      "173000\n",
      "174000\n",
      "175000\n",
      "176000\n",
      "177000\n",
      "178000\n",
      "179000\n",
      "180000\n",
      "181000\n",
      "182000\n",
      "183000\n",
      "184000\n",
      "185000\n",
      "186000\n",
      "187000\n",
      "188000\n",
      "189000\n",
      "190000\n",
      "191000\n",
      "192000\n",
      "193000\n",
      "194000\n",
      "195000\n",
      "196000\n",
      "197000\n",
      "198000\n",
      "199000\n",
      "200000\n",
      "201000\n",
      "202000\n",
      "203000\n",
      "204000\n",
      "205000\n",
      "206000\n",
      "207000\n",
      "208000\n",
      "209000\n",
      "210000\n",
      "211000\n",
      "212000\n",
      "213000\n",
      "214000\n",
      "215000\n",
      "216000\n",
      "217000\n",
      "218000\n",
      "219000\n",
      "220000\n",
      "221000\n",
      "222000\n",
      "223000\n",
      "224000\n",
      "225000\n",
      "226000\n",
      "227000\n",
      "228000\n",
      "229000\n",
      "230000\n",
      "231000\n",
      "232000\n",
      "233000\n",
      "234000\n",
      "235000\n",
      "236000\n",
      "237000\n",
      "238000\n",
      "239000\n",
      "240000\n",
      "241000\n",
      "242000\n",
      "243000\n",
      "244000\n",
      "245000\n",
      "246000\n",
      "247000\n",
      "248000\n",
      "249000\n",
      "250000\n",
      "251000\n",
      "252000\n",
      "253000\n",
      "254000\n",
      "255000\n",
      "256000\n",
      "257000\n",
      "258000\n",
      "259000\n",
      "260000\n",
      "261000\n",
      "262000\n",
      "263000\n",
      "264000\n",
      "265000\n",
      "266000\n",
      "267000\n",
      "268000\n",
      "269000\n",
      "270000\n",
      "271000\n",
      "272000\n",
      "273000\n",
      "274000\n",
      "275000\n",
      "276000\n",
      "277000\n",
      "278000\n",
      "279000\n",
      "280000\n",
      "281000\n",
      "282000\n",
      "283000\n",
      "284000\n",
      "285000\n",
      "286000\n",
      "287000\n",
      "288000\n",
      "289000\n",
      "290000\n",
      "291000\n",
      "292000\n",
      "293000\n",
      "294000\n",
      "295000\n",
      "296000\n",
      "297000\n",
      "298000\n",
      "299000\n",
      "300000\n",
      "301000\n",
      "302000\n",
      "303000\n",
      "304000\n",
      "305000\n",
      "306000\n",
      "307000\n",
      "308000\n",
      "309000\n",
      "310000\n",
      "311000\n",
      "312000\n",
      "313000\n",
      "314000\n",
      "315000\n",
      "316000\n",
      "317000\n",
      "318000\n",
      "319000\n",
      "320000\n",
      "321000\n",
      "322000\n",
      "323000\n",
      "324000\n",
      "325000\n",
      "326000\n",
      "327000\n",
      "328000\n",
      "329000\n",
      "330000\n",
      "331000\n",
      "332000\n",
      "333000\n",
      "334000\n",
      "335000\n",
      "336000\n",
      "337000\n",
      "338000\n",
      "339000\n",
      "340000\n",
      "341000\n",
      "342000\n",
      "343000\n",
      "344000\n",
      "345000\n",
      "346000\n",
      "347000\n",
      "348000\n",
      "349000\n",
      "350000\n",
      "351000\n",
      "352000\n",
      "353000\n",
      "354000\n",
      "355000\n",
      "356000\n",
      "357000\n",
      "358000\n",
      "359000\n",
      "360000\n",
      "361000\n",
      "362000\n",
      "363000\n",
      "364000\n",
      "365000\n",
      "366000\n",
      "367000\n",
      "368000\n",
      "369000\n",
      "370000\n",
      "371000\n",
      "372000\n",
      "373000\n",
      "374000\n",
      "375000\n",
      "376000\n",
      "377000\n",
      "378000\n",
      "379000\n",
      "380000\n",
      "381000\n",
      "382000\n",
      "383000\n",
      "384000\n",
      "385000\n",
      "386000\n",
      "387000\n",
      "388000\n",
      "389000\n",
      "390000\n",
      "391000\n",
      "392000\n",
      "393000\n",
      "394000\n",
      "395000\n",
      "396000\n",
      "397000\n",
      "398000\n",
      "399000\n",
      "400000\n",
      "401000\n",
      "402000\n",
      "403000\n",
      "404000\n",
      "405000\n",
      "406000\n",
      "407000\n",
      "408000\n",
      "409000\n",
      "410000\n",
      "411000\n",
      "412000\n",
      "413000\n",
      "414000\n",
      "415000\n",
      "7.136777623494466\n"
     ]
    }
   ],
   "source": [
    "start = time.time()\n",
    "test_features = convert_examples_to_features_test(test, tokenizer, args.max_seq_length)\n",
    "print((time.time()-start)/60)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [],
   "source": [
    "a = test_features[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "a.choices_features,a.label"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# dfs = [sorted_train_data[i:i+500] for i in range(0,sorted_train_data.shape[0],500)]\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "dfs = [train_examples_orig[i:i+500] for i in range(0,train_examples_orig.shape[0],500)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>query_id</th>\n",
       "      <th>query</th>\n",
       "      <th>passage_text</th>\n",
       "      <th>label</th>\n",
       "      <th>passage_id</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>Androgen receptor define</td>\n",
       "      <td>The androgen receptor (AR), also known as NR3C...</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>Androgen receptor define</td>\n",
       "      <td>The AR gene provides instructions for making a...</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1</td>\n",
       "      <td>Androgen receptor define</td>\n",
       "      <td>The androgen receptor gene is more than 90 kb ...</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1</td>\n",
       "      <td>Androgen receptor define</td>\n",
       "      <td>Mutations in the AR gene cause androgen insens...</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1</td>\n",
       "      <td>Androgen receptor define</td>\n",
       "      <td>Normal function of the androgen receptor. Test...</td>\n",
       "      <td>0</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   query_id                      query  \\\n",
       "0         1   Androgen receptor define   \n",
       "1         1   Androgen receptor define   \n",
       "2         1   Androgen receptor define   \n",
       "3         1   Androgen receptor define   \n",
       "4         1   Androgen receptor define   \n",
       "\n",
       "                                        passage_text  label  passage_id  \n",
       "0  The androgen receptor (AR), also known as NR3C...      1           0  \n",
       "1  The AR gene provides instructions for making a...      0           1  \n",
       "2  The androgen receptor gene is more than 90 kb ...      0           2  \n",
       "3  Mutations in the AR gene cause androgen insens...      0           3  \n",
       "4  Normal function of the androgen receptor. Test...      0           4  "
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dfs[0].head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def convert_examples_with_pool(dfs,threads=4,is_training=True):\n",
    "    with Pool(threads) as p:\n",
    "         f_list = p.map(convert_examples_to_features_parellel,dfs)\n",
    "    return f_list\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [],
   "source": [
    "from multiprocessing import Pool"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from multiprocessing import Pool\n",
    "\n",
    "\n",
    "with Pool(16) as p:\n",
    "     f_list = p.map(convert_examples_to_features_parellel,dfs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "400"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(f_list)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Pickling"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pickle"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Saving test data processed.\n",
    "# file = open('./search_data/test_2_list_all_290', 'wb')\n",
    "# pickle.dump(test_features,file)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [],
   "source": [
    "r_file = open('./search_data/f_list_all', 'rb')\n",
    "f_list_all = pickle.load(r_file)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "524188"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Since our dataset is 10-class classification\n",
    "len(f_list_all)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [],
   "source": [
    "# del model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [],
   "source": [
    "## To clear out unused GPU and CPU memory\n",
    "import gc\n",
    "gc.collect()\n",
    "torch.cuda.empty_cache()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [],
   "source": [
    "# to utilize a variable across all jupyter notebooks\n",
    "# %store f_list"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "290"
      ]
     },
     "execution_count": 52,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "args.max_seq_length"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "heading_collapsed": true
   },
   "source": [
    "## Training\n",
    "\n",
    "Running on a small dataset here, in Competition we ran 2 models on complete training data- \n",
    "1. 2 Epochs, 1st with 120 max_seq_length, 2nd with 160 max_seq_length\n",
    "2. 3 Epochs, 1st with 120 max_seq_length, 2nd with 160 max_seq_length, 3rd with 198 max_seq_length"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "hidden": true
   },
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "hidden": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "args.do_train"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {
    "hidden": true,
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "12/26/2018 18:11:14 - INFO - __main__ -     Batch size = 24\n",
      "12/26/2018 18:11:14 - INFO - __main__ -     Num steps = 1197\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "***** Running training *****\n",
      "  Num examples = %d 28261\n"
     ]
    }
   ],
   "source": [
    "if args.do_train:\n",
    "#     train_features = convert_examples_to_features(\n",
    "#         train_examples, tokenizer, args.max_seq_length, True)\n",
    "    train_features = f_list_all\n",
    "    print(\"***** Running training *****\")\n",
    "    print(\"  Num examples = %d\", len(train_features))\n",
    "    logger.info(\"  Batch size = %d\", args.train_batch_size)\n",
    "    logger.info(\"  Num steps = %d\", num_train_steps)\n",
    "    all_input_ids = torch.tensor(select_field(train_features, 'input_ids'), dtype=torch.long)\n",
    "    all_input_mask = torch.tensor(select_field(train_features, 'input_mask'), dtype=torch.long)\n",
    "    all_segment_ids = torch.tensor(select_field(train_features, 'segment_ids'), dtype=torch.long)\n",
    "    all_label = torch.tensor([f.label for f in train_features], dtype=torch.long)\n",
    "    train_data = TensorDataset(all_input_ids, all_input_mask, all_segment_ids, all_label)\n",
    "    if args.local_rank == -1:\n",
    "        train_sampler = RandomSampler(train_data)\n",
    "    else:\n",
    "        train_sampler = DistributedSampler(train_data)\n",
    "    train_dataloader = DataLoader(train_data, sampler=train_sampler, batch_size=args.train_batch_size)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 82,
   "metadata": {
    "hidden": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<torch.utils.data.dataloader.DataLoader at 0x7f3b7f406d30>"
      ]
     },
     "execution_count": 82,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_dataloader"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": [
    "import gc\n",
    "gc.collect()\n",
    "torch.cuda.empty_cache()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 81,
   "metadata": {
    "hidden": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1.0"
      ]
     },
     "execution_count": 81,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "args.num_train_epochs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {
    "hidden": true,
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch:   0%|          | 0/1 [00:00<?, ?it/s]\n",
      "Iteration:   0%|          | 0/1178 [00:00<?, ?it/s]\u001b[A/home/ec2-user/anaconda3/envs/pytorch_p36/lib/python3.6/site-packages/torch/nn/parallel/_functions.py:58: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "\n",
      "Iteration:   0%|          | 1/1178 [00:35<11:34:09, 35.39s/it]\u001b[A\n",
      "Iteration:   0%|          | 2/1178 [00:36<8:10:22, 25.02s/it] \u001b[A\n",
      "Iteration:   0%|          | 3/1178 [00:36<5:45:34, 17.65s/it]\u001b[A\n",
      "Iteration:   0%|          | 4/1178 [00:37<4:05:31, 12.55s/it]\u001b[A\n",
      "Iteration:   0%|          | 5/1178 [00:37<2:54:13,  8.91s/it]\u001b[A\n",
      "Iteration:   1%|          | 6/1178 [00:38<2:04:19,  6.36s/it]\u001b[A\n",
      "Iteration:   1%|          | 7/1178 [00:39<1:37:24,  4.99s/it]\u001b[A\n",
      "Iteration:   1%|          | 8/1178 [00:40<1:10:32,  3.62s/it]\u001b[A\n",
      "Iteration:   1%|          | 9/1178 [00:40<51:46,  2.66s/it]  \u001b[A\n",
      "Iteration:   1%|          | 10/1178 [00:41<38:58,  2.00s/it]\u001b[A\n",
      "Iteration:   1%|          | 11/1178 [00:41<29:52,  1.54s/it]\u001b[A\n",
      "Iteration:   1%|          | 12/1178 [00:42<23:19,  1.20s/it]\u001b[A\n",
      "Iteration:   1%|          | 13/1178 [00:42<20:22,  1.05s/it]\u001b[A\n",
      "Iteration:   1%|          | 14/1178 [00:43<16:39,  1.16it/s]\u001b[A\n",
      "Iteration:   1%|▏         | 15/1178 [00:43<14:03,  1.38it/s]\u001b[A\n",
      "Iteration:   1%|▏         | 16/1178 [00:44<12:14,  1.58it/s]\u001b[A\n",
      "Iteration:   1%|▏         | 17/1178 [00:44<10:58,  1.76it/s]\u001b[A\n",
      "Iteration:   2%|▏         | 18/1178 [00:44<10:05,  1.92it/s]\u001b[A\n",
      "Iteration:   2%|▏         | 19/1178 [00:45<09:27,  2.04it/s]\u001b[A\n",
      "Iteration:   2%|▏         | 20/1178 [00:45<09:17,  2.08it/s]\u001b[A\n",
      "Iteration:   2%|▏         | 21/1178 [00:46<08:54,  2.16it/s]\u001b[A\n",
      "Iteration:   2%|▏         | 22/1178 [00:46<08:38,  2.23it/s]\u001b[A\n",
      "Iteration:   2%|▏         | 23/1178 [00:47<08:27,  2.28it/s]\u001b[A\n",
      "Iteration:   2%|▏         | 24/1178 [00:47<08:19,  2.31it/s]\u001b[A\n",
      "Iteration:   2%|▏         | 25/1178 [00:47<08:13,  2.34it/s]\u001b[A\n",
      "Iteration:   2%|▏         | 26/1178 [00:48<08:07,  2.36it/s]\u001b[A\n",
      "Iteration:   2%|▏         | 27/1178 [00:48<08:06,  2.37it/s]\u001b[A\n",
      "Iteration:   2%|▏         | 28/1178 [00:49<08:05,  2.37it/s]\u001b[A\n",
      "Iteration:   2%|▏         | 29/1178 [00:49<08:05,  2.37it/s]\u001b[A\n",
      "Iteration:   3%|▎         | 30/1178 [00:49<08:22,  2.28it/s]\u001b[A\n",
      "Iteration:   3%|▎         | 31/1178 [00:50<08:17,  2.30it/s]\u001b[A\n",
      "Iteration:   3%|▎         | 32/1178 [00:50<08:12,  2.33it/s]\u001b[A\n",
      "Iteration:   3%|▎         | 33/1178 [00:51<08:10,  2.34it/s]\u001b[A\n",
      "Iteration:   3%|▎         | 34/1178 [00:51<08:07,  2.35it/s]\u001b[A\n",
      "Iteration:   3%|▎         | 35/1178 [00:52<08:05,  2.35it/s]\u001b[A\n",
      "Iteration:   3%|▎         | 36/1178 [00:52<08:02,  2.37it/s]\u001b[A\n",
      "Iteration:   3%|▎         | 37/1178 [00:52<08:00,  2.38it/s]\u001b[A\n",
      "Iteration:   3%|▎         | 38/1178 [00:53<07:59,  2.38it/s]\u001b[A\n",
      "Iteration:   3%|▎         | 39/1178 [00:53<07:57,  2.38it/s]\u001b[A\n",
      "Iteration:   3%|▎         | 40/1178 [00:54<08:12,  2.31it/s]\u001b[A\n",
      "Iteration:   3%|▎         | 41/1178 [00:54<08:07,  2.33it/s]\u001b[A\n",
      "Iteration:   4%|▎         | 42/1178 [00:55<08:01,  2.36it/s]\u001b[A\n",
      "Iteration:   4%|▎         | 43/1178 [00:56<15:18,  1.24it/s]\u001b[A\n",
      "Iteration:   4%|▎         | 44/1178 [00:57<13:07,  1.44it/s]\u001b[A\n",
      "Iteration:   4%|▍         | 45/1178 [00:57<11:33,  1.63it/s]\u001b[A\n",
      "Iteration:   4%|▍         | 46/1178 [00:58<10:27,  1.80it/s]\u001b[A\n",
      "Iteration:   4%|▍         | 47/1178 [00:58<09:41,  1.95it/s]\u001b[A\n",
      "Iteration:   4%|▍         | 48/1178 [00:58<09:07,  2.06it/s]\u001b[A\n",
      "Iteration:   4%|▍         | 49/1178 [00:59<08:44,  2.15it/s]\u001b[A\n",
      "Iteration:   4%|▍         | 50/1178 [00:59<08:44,  2.15it/s]\u001b[A\n",
      "Iteration:   4%|▍         | 51/1178 [01:00<08:29,  2.21it/s]\u001b[A\n",
      "Iteration:   4%|▍         | 52/1178 [01:00<08:17,  2.26it/s]\u001b[A\n",
      "Iteration:   4%|▍         | 53/1178 [01:01<08:09,  2.30it/s]\u001b[A\n",
      "Iteration:   5%|▍         | 54/1178 [01:01<08:02,  2.33it/s]\u001b[A\n",
      "Iteration:   5%|▍         | 55/1178 [01:01<07:57,  2.35it/s]\u001b[A\n",
      "Iteration:   5%|▍         | 56/1178 [01:02<07:54,  2.36it/s]\u001b[A\n",
      "Iteration:   5%|▍         | 57/1178 [01:02<07:51,  2.38it/s]\u001b[A\n",
      "Iteration:   5%|▍         | 58/1178 [01:03<07:49,  2.39it/s]\u001b[A\n",
      "Iteration:   5%|▌         | 59/1178 [01:03<07:48,  2.39it/s]\u001b[A\n",
      "Iteration:   5%|▌         | 60/1178 [01:03<08:02,  2.32it/s]\u001b[A\n",
      "Iteration:   5%|▌         | 61/1178 [01:04<07:57,  2.34it/s]\u001b[A\n",
      "Iteration:   5%|▌         | 62/1178 [01:04<07:54,  2.35it/s]\u001b[A\n",
      "Iteration:   5%|▌         | 63/1178 [01:05<07:51,  2.36it/s]\u001b[A\n",
      "Iteration:   5%|▌         | 64/1178 [01:05<07:50,  2.37it/s]\u001b[A\n",
      "Iteration:   6%|▌         | 65/1178 [01:06<07:48,  2.38it/s]\u001b[A\n",
      "Iteration:   6%|▌         | 66/1178 [01:06<07:47,  2.38it/s]\u001b[A\n",
      "Iteration:   6%|▌         | 67/1178 [01:06<07:45,  2.38it/s]\u001b[A\n",
      "Iteration:   6%|▌         | 68/1178 [01:07<07:43,  2.39it/s]\u001b[A\n",
      "Iteration:   6%|▌         | 69/1178 [01:07<07:43,  2.39it/s]\u001b[A\n",
      "Iteration:   6%|▌         | 70/1178 [01:08<08:01,  2.30it/s]\u001b[A\n",
      "Iteration:   6%|▌         | 71/1178 [01:08<07:58,  2.31it/s]\u001b[A\n",
      "Iteration:   6%|▌         | 72/1178 [01:09<07:53,  2.33it/s]\u001b[A\n",
      "Iteration:   6%|▌         | 73/1178 [01:09<07:51,  2.34it/s]\u001b[A\n",
      "Iteration:   6%|▋         | 74/1178 [01:09<07:49,  2.35it/s]\u001b[A\n",
      "Iteration:   6%|▋         | 75/1178 [01:10<07:47,  2.36it/s]\u001b[A\n",
      "Iteration:   6%|▋         | 76/1178 [01:10<07:46,  2.36it/s]\u001b[A\n",
      "Iteration:   7%|▋         | 77/1178 [01:11<07:46,  2.36it/s]\u001b[A\n",
      "Iteration:   7%|▋         | 78/1178 [01:13<17:27,  1.05it/s]\u001b[A\n",
      "Iteration:   7%|▋         | 79/1178 [01:13<14:31,  1.26it/s]\u001b[A\n",
      "Iteration:   7%|▋         | 80/1178 [01:14<12:43,  1.44it/s]\u001b[A\n",
      "Iteration:   7%|▋         | 81/1178 [01:14<11:12,  1.63it/s]\u001b[A\n",
      "Iteration:   7%|▋         | 82/1178 [01:15<10:06,  1.81it/s]\u001b[A\n",
      "Iteration:   7%|▋         | 83/1178 [01:15<09:22,  1.95it/s]\u001b[A\n",
      "Iteration:   7%|▋         | 84/1178 [01:15<08:53,  2.05it/s]\u001b[A\n",
      "Iteration:   7%|▋         | 85/1178 [01:16<08:33,  2.13it/s]\u001b[A\n",
      "Iteration:   7%|▋         | 86/1178 [01:16<08:16,  2.20it/s]\u001b[A\n",
      "Iteration:   7%|▋         | 87/1178 [01:17<08:05,  2.25it/s]\u001b[A\n",
      "Iteration:   7%|▋         | 88/1178 [01:17<07:56,  2.29it/s]\u001b[A\n",
      "Iteration:   8%|▊         | 89/1178 [01:18<07:51,  2.31it/s]\u001b[A\n",
      "Iteration:   8%|▊         | 90/1178 [01:18<08:03,  2.25it/s]\u001b[A\n",
      "Iteration:   8%|▊         | 91/1178 [01:18<07:55,  2.29it/s]\u001b[A\n",
      "Iteration:   8%|▊         | 92/1178 [01:19<07:48,  2.32it/s]\u001b[A\n",
      "Iteration:   8%|▊         | 93/1178 [01:19<07:45,  2.33it/s]\u001b[A\n",
      "Iteration:   8%|▊         | 94/1178 [01:20<07:41,  2.35it/s]\u001b[A\n",
      "Iteration:   8%|▊         | 95/1178 [01:20<07:39,  2.36it/s]\u001b[A\n",
      "Iteration:   8%|▊         | 96/1178 [01:21<07:36,  2.37it/s]\u001b[A\n",
      "Iteration:   8%|▊         | 97/1178 [01:21<07:35,  2.37it/s]\u001b[A\n",
      "Iteration:   8%|▊         | 98/1178 [01:21<07:34,  2.38it/s]\u001b[A\n",
      "Iteration:   8%|▊         | 99/1178 [01:22<07:34,  2.38it/s]\u001b[A\n",
      "Iteration:   8%|▊         | 100/1178 [01:22<07:49,  2.30it/s]\u001b[A\n",
      "Iteration:   9%|▊         | 101/1178 [01:23<07:46,  2.31it/s]\u001b[A\n",
      "Iteration:   9%|▊         | 102/1178 [01:23<07:43,  2.32it/s]\u001b[A\n",
      "Iteration:   9%|▊         | 103/1178 [01:24<07:38,  2.35it/s]\u001b[A\n",
      "Iteration:   9%|▉         | 104/1178 [01:24<07:34,  2.36it/s]\u001b[A\n",
      "Iteration:   9%|▉         | 105/1178 [01:24<07:32,  2.37it/s]\u001b[A\n",
      "Iteration:   9%|▉         | 106/1178 [01:25<07:29,  2.38it/s]\u001b[A\n",
      "Iteration:   9%|▉         | 107/1178 [01:25<07:27,  2.39it/s]\u001b[A\n",
      "Iteration:   9%|▉         | 108/1178 [01:26<07:27,  2.39it/s]\u001b[A\n",
      "Iteration:   9%|▉         | 109/1178 [01:26<07:27,  2.39it/s]\u001b[A\n",
      "Iteration:   9%|▉         | 110/1178 [01:27<07:43,  2.31it/s]\u001b[A\n",
      "Iteration:   9%|▉         | 111/1178 [01:27<07:38,  2.33it/s]\u001b[A\n",
      "Iteration:  10%|▉         | 112/1178 [01:29<14:53,  1.19it/s]\u001b[A\n",
      "Iteration:  10%|▉         | 113/1178 [01:29<12:38,  1.40it/s]\u001b[A\n",
      "Iteration:  10%|▉         | 114/1178 [01:30<11:02,  1.61it/s]\u001b[A\n",
      "Iteration:  10%|▉         | 115/1178 [01:30<09:57,  1.78it/s]\u001b[A\n",
      "Iteration:  10%|▉         | 116/1178 [01:30<09:10,  1.93it/s]\u001b[A\n",
      "Iteration:  10%|▉         | 117/1178 [01:31<08:37,  2.05it/s]\u001b[A\n",
      "Iteration:  10%|█         | 118/1178 [01:31<08:14,  2.14it/s]\u001b[A\n",
      "Iteration:  10%|█         | 119/1178 [01:32<07:58,  2.21it/s]\u001b[A\n",
      "Iteration:  10%|█         | 120/1178 [01:32<08:04,  2.19it/s]\u001b[A\n",
      "Iteration:  10%|█         | 121/1178 [01:33<07:51,  2.24it/s]\u001b[A\n",
      "Iteration:  10%|█         | 122/1178 [01:33<07:41,  2.29it/s]\u001b[A\n",
      "Iteration:  10%|█         | 123/1178 [01:33<07:35,  2.32it/s]\u001b[A\n",
      "Iteration:  11%|█         | 124/1178 [01:34<07:31,  2.34it/s]\u001b[A\n",
      "Iteration:  11%|█         | 125/1178 [01:34<07:29,  2.34it/s]\u001b[A\n",
      "Iteration:  11%|█         | 126/1178 [01:35<07:25,  2.36it/s]\u001b[A\n",
      "Iteration:  11%|█         | 127/1178 [01:35<07:23,  2.37it/s]\u001b[A\n",
      "Iteration:  11%|█         | 128/1178 [01:35<07:22,  2.38it/s]\u001b[A\n",
      "Iteration:  11%|█         | 129/1178 [01:36<07:20,  2.38it/s]\u001b[A\n",
      "Iteration:  11%|█         | 130/1178 [01:36<07:34,  2.31it/s]\u001b[A\n",
      "Iteration:  11%|█         | 131/1178 [01:37<07:30,  2.33it/s]\u001b[A\n",
      "Iteration:  11%|█         | 132/1178 [01:37<07:26,  2.34it/s]\u001b[A\n",
      "Iteration:  11%|█▏        | 133/1178 [01:38<07:24,  2.35it/s]\u001b[A\n",
      "Iteration:  11%|█▏        | 134/1178 [01:38<07:22,  2.36it/s]\u001b[A\n",
      "Iteration:  11%|█▏        | 135/1178 [01:38<07:21,  2.36it/s]\u001b[A\n",
      "Iteration:  12%|█▏        | 136/1178 [01:39<07:19,  2.37it/s]\u001b[A\n",
      "Iteration:  12%|█▏        | 137/1178 [01:39<07:16,  2.38it/s]\u001b[A\n",
      "Iteration:  12%|█▏        | 138/1178 [01:40<07:15,  2.39it/s]\u001b[A\n",
      "Iteration:  12%|█▏        | 139/1178 [01:40<07:16,  2.38it/s]\u001b[A\n",
      "Iteration:  12%|█▏        | 140/1178 [01:41<07:30,  2.31it/s]\u001b[A\n",
      "Iteration:  12%|█▏        | 141/1178 [01:41<07:25,  2.33it/s]\u001b[A\n",
      "Iteration:  12%|█▏        | 142/1178 [01:41<07:22,  2.34it/s]\u001b[A\n",
      "Iteration:  12%|█▏        | 143/1178 [01:42<07:18,  2.36it/s]\u001b[A\n",
      "Iteration:  12%|█▏        | 144/1178 [01:42<07:15,  2.38it/s]\u001b[A\n",
      "Iteration:  12%|█▏        | 145/1178 [01:43<07:12,  2.39it/s]\u001b[A\n",
      "Iteration:  12%|█▏        | 146/1178 [01:44<13:42,  1.25it/s]\u001b[A\n",
      "Iteration:  12%|█▏        | 147/1178 [01:45<11:43,  1.47it/s]\u001b[A\n",
      "Iteration:  13%|█▎        | 148/1178 [01:45<10:20,  1.66it/s]\u001b[A\n",
      "Iteration:  13%|█▎        | 149/1178 [01:46<09:23,  1.83it/s]\u001b[A\n",
      "Iteration:  13%|█▎        | 150/1178 [01:46<08:58,  1.91it/s]\u001b[A\n",
      "Iteration:  13%|█▎        | 151/1178 [01:47<08:27,  2.03it/s]\u001b[A\n",
      "Iteration:  13%|█▎        | 152/1178 [01:47<08:04,  2.12it/s]\u001b[A\n",
      "Iteration:  13%|█▎        | 153/1178 [01:47<07:47,  2.19it/s]\u001b[A\n",
      "Iteration:  13%|█▎        | 154/1178 [01:48<07:34,  2.25it/s]\u001b[A\n",
      "Iteration:  13%|█▎        | 155/1178 [01:48<07:26,  2.29it/s]\u001b[A\n",
      "Iteration:  13%|█▎        | 156/1178 [01:49<07:19,  2.32it/s]\u001b[A\n",
      "Iteration:  13%|█▎        | 157/1178 [01:49<07:14,  2.35it/s]\u001b[A\n",
      "Iteration:  13%|█▎        | 158/1178 [01:49<07:13,  2.35it/s]\u001b[A\n",
      "Iteration:  13%|█▎        | 159/1178 [01:50<07:11,  2.36it/s]\u001b[A\n",
      "Iteration:  14%|█▎        | 160/1178 [01:50<07:26,  2.28it/s]\u001b[A\n",
      "Iteration:  14%|█▎        | 161/1178 [01:51<07:20,  2.31it/s]\u001b[A\n",
      "Iteration:  14%|█▍        | 162/1178 [01:51<07:16,  2.33it/s]\u001b[A\n",
      "Iteration:  14%|█▍        | 163/1178 [01:52<07:13,  2.34it/s]\u001b[A\n",
      "Iteration:  14%|█▍        | 164/1178 [01:52<07:09,  2.36it/s]\u001b[A\n",
      "Iteration:  14%|█▍        | 165/1178 [01:52<07:08,  2.37it/s]\u001b[A\n",
      "Iteration:  14%|█▍        | 166/1178 [01:53<07:06,  2.37it/s]\u001b[A\n",
      "Iteration:  14%|█▍        | 167/1178 [01:53<07:05,  2.38it/s]\u001b[A\n",
      "Iteration:  14%|█▍        | 168/1178 [01:54<07:05,  2.38it/s]\u001b[A\n",
      "Iteration:  14%|█▍        | 169/1178 [01:54<07:04,  2.38it/s]\u001b[A\n",
      "Iteration:  14%|█▍        | 170/1178 [01:55<07:18,  2.30it/s]\u001b[A\n",
      "Iteration:  15%|█▍        | 171/1178 [01:55<07:13,  2.32it/s]\u001b[A\n",
      "Iteration:  15%|█▍        | 172/1178 [01:55<07:09,  2.34it/s]\u001b[A\n",
      "Iteration:  15%|█▍        | 173/1178 [01:56<07:07,  2.35it/s]\u001b[A\n",
      "Iteration:  15%|█▍        | 174/1178 [01:56<07:04,  2.36it/s]\u001b[A\n",
      "Iteration:  15%|█▍        | 175/1178 [01:57<07:02,  2.37it/s]\u001b[A\n",
      "Iteration:  15%|█▍        | 176/1178 [01:57<07:01,  2.38it/s]\u001b[A\n",
      "Iteration:  15%|█▌        | 177/1178 [01:58<07:00,  2.38it/s]\u001b[A\n",
      "Iteration:  15%|█▌        | 178/1178 [01:58<06:59,  2.38it/s]\u001b[A\n",
      "Iteration:  15%|█▌        | 179/1178 [01:58<06:59,  2.38it/s]\u001b[A\n",
      "Iteration:  15%|█▌        | 180/1178 [02:00<13:26,  1.24it/s]\u001b[A\n",
      "Iteration:  15%|█▌        | 181/1178 [02:00<11:29,  1.45it/s]\u001b[A\n",
      "Iteration:  15%|█▌        | 182/1178 [02:01<10:07,  1.64it/s]\u001b[A\n",
      "Iteration:  16%|█▌        | 183/1178 [02:01<09:10,  1.81it/s]\u001b[A\n",
      "Iteration:  16%|█▌        | 184/1178 [02:02<08:30,  1.95it/s]\u001b[A\n",
      "Iteration:  16%|█▌        | 185/1178 [02:02<08:01,  2.06it/s]\u001b[A\n",
      "Iteration:  16%|█▌        | 186/1178 [02:03<07:43,  2.14it/s]\u001b[A\n",
      "Iteration:  16%|█▌        | 187/1178 [02:03<07:27,  2.21it/s]\u001b[A\n",
      "Iteration:  16%|█▌        | 188/1178 [02:03<07:16,  2.27it/s]\u001b[A\n",
      "Iteration:  16%|█▌        | 189/1178 [02:04<07:09,  2.30it/s]\u001b[A\n",
      "Iteration:  16%|█▌        | 190/1178 [02:04<07:18,  2.25it/s]\u001b[A\n",
      "Iteration:  16%|█▌        | 191/1178 [02:05<07:10,  2.29it/s]\u001b[A\n",
      "Iteration:  16%|█▋        | 192/1178 [02:05<07:05,  2.32it/s]\u001b[A\n",
      "Iteration:  16%|█▋        | 193/1178 [02:06<07:00,  2.34it/s]\u001b[A\n",
      "Iteration:  16%|█▋        | 194/1178 [02:06<06:59,  2.34it/s]\u001b[A\n",
      "Iteration:  17%|█▋        | 195/1178 [02:06<06:58,  2.35it/s]\u001b[A\n",
      "Iteration:  17%|█▋        | 196/1178 [02:07<06:55,  2.36it/s]\u001b[A\n",
      "Iteration:  17%|█▋        | 197/1178 [02:07<06:53,  2.37it/s]\u001b[A\n",
      "Iteration:  17%|█▋        | 198/1178 [02:08<06:51,  2.38it/s]\u001b[A\n",
      "Iteration:  17%|█▋        | 199/1178 [02:08<06:50,  2.38it/s]\u001b[A\n",
      "Iteration:  17%|█▋        | 200/1178 [02:09<07:04,  2.30it/s]\u001b[A\n",
      "Iteration:  17%|█▋        | 201/1178 [02:09<06:59,  2.33it/s]\u001b[A\n",
      "Iteration:  17%|█▋        | 202/1178 [02:09<06:56,  2.34it/s]\u001b[A\n",
      "Iteration:  17%|█▋        | 203/1178 [02:10<06:53,  2.36it/s]\u001b[A\n",
      "Iteration:  17%|█▋        | 204/1178 [02:10<06:51,  2.37it/s]\u001b[A\n",
      "Iteration:  17%|█▋        | 205/1178 [02:11<06:50,  2.37it/s]\u001b[A\n",
      "Iteration:  17%|█▋        | 206/1178 [02:11<06:48,  2.38it/s]\u001b[A\n",
      "Iteration:  18%|█▊        | 207/1178 [02:11<06:46,  2.39it/s]\u001b[A\n",
      "Iteration:  18%|█▊        | 208/1178 [02:12<06:45,  2.39it/s]\u001b[A\n",
      "Iteration:  18%|█▊        | 209/1178 [02:12<06:45,  2.39it/s]\u001b[A\n",
      "Iteration:  18%|█▊        | 210/1178 [02:13<06:58,  2.31it/s]\u001b[A\n",
      "Iteration:  18%|█▊        | 211/1178 [02:13<06:54,  2.33it/s]\u001b[A\n",
      "Iteration:  18%|█▊        | 212/1178 [02:14<06:51,  2.35it/s]\u001b[A\n",
      "Iteration:  18%|█▊        | 213/1178 [02:14<06:49,  2.36it/s]\u001b[A\n",
      "Iteration:  18%|█▊        | 214/1178 [02:16<12:47,  1.26it/s]\u001b[A\n",
      "Iteration:  18%|█▊        | 215/1178 [02:16<10:58,  1.46it/s]\u001b[A\n",
      "Iteration:  18%|█▊        | 216/1178 [02:17<09:40,  1.66it/s]\u001b[A\n",
      "Iteration:  18%|█▊        | 217/1178 [02:17<08:47,  1.82it/s]\u001b[A\n",
      "Iteration:  19%|█▊        | 218/1178 [02:17<08:10,  1.96it/s]\u001b[A\n",
      "Iteration:  19%|█▊        | 219/1178 [02:18<07:43,  2.07it/s]\u001b[A\n",
      "Iteration:  19%|█▊        | 220/1178 [02:18<07:39,  2.09it/s]\u001b[A\n",
      "Iteration:  19%|█▉        | 221/1178 [02:19<07:22,  2.16it/s]\u001b[A\n",
      "Iteration:  19%|█▉        | 222/1178 [02:19<07:09,  2.22it/s]\u001b[A\n",
      "Iteration:  19%|█▉        | 223/1178 [02:20<07:01,  2.26it/s]\u001b[A\n",
      "Iteration:  19%|█▉        | 224/1178 [02:20<06:55,  2.30it/s]\u001b[A\n",
      "Iteration:  19%|█▉        | 225/1178 [02:20<06:50,  2.32it/s]\u001b[A\n",
      "Iteration:  19%|█▉        | 226/1178 [02:21<06:46,  2.34it/s]\u001b[A\n",
      "Iteration:  19%|█▉        | 227/1178 [02:21<06:43,  2.35it/s]\u001b[A\n",
      "Iteration:  19%|█▉        | 228/1178 [02:22<06:41,  2.37it/s]\u001b[A\n",
      "Iteration:  19%|█▉        | 229/1178 [02:22<06:38,  2.38it/s]\u001b[A\n",
      "Iteration:  20%|█▉        | 230/1178 [02:23<06:51,  2.31it/s]\u001b[A\n",
      "Iteration:  20%|█▉        | 231/1178 [02:23<06:46,  2.33it/s]\u001b[A\n",
      "Iteration:  20%|█▉        | 232/1178 [02:23<06:42,  2.35it/s]\u001b[A\n",
      "Iteration:  20%|█▉        | 233/1178 [02:24<06:39,  2.37it/s]\u001b[A\n",
      "Iteration:  20%|█▉        | 234/1178 [02:24<06:37,  2.38it/s]\u001b[A\n",
      "Iteration:  20%|█▉        | 235/1178 [02:25<06:35,  2.38it/s]\u001b[A\n",
      "Iteration:  20%|██        | 236/1178 [02:25<06:34,  2.39it/s]\u001b[A\n",
      "Iteration:  20%|██        | 237/1178 [02:25<06:33,  2.39it/s]\u001b[A\n",
      "Iteration:  20%|██        | 238/1178 [02:26<06:33,  2.39it/s]\u001b[A\n",
      "Iteration:  20%|██        | 239/1178 [02:26<06:33,  2.39it/s]\u001b[A\n",
      "Iteration:  20%|██        | 240/1178 [02:27<06:47,  2.30it/s]\u001b[A\n",
      "Iteration:  20%|██        | 241/1178 [02:27<06:42,  2.33it/s]\u001b[A\n",
      "Iteration:  21%|██        | 242/1178 [02:28<06:38,  2.35it/s]\u001b[A\n",
      "Iteration:  21%|██        | 243/1178 [02:28<06:36,  2.36it/s]\u001b[A\n",
      "Iteration:  21%|██        | 244/1178 [02:28<06:34,  2.37it/s]\u001b[A\n",
      "Iteration:  21%|██        | 245/1178 [02:29<06:32,  2.37it/s]\u001b[A\n",
      "Iteration:  21%|██        | 246/1178 [02:29<06:30,  2.39it/s]\u001b[A\n",
      "Iteration:  21%|██        | 247/1178 [02:30<06:30,  2.39it/s]\u001b[A\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Iteration:  21%|██        | 248/1178 [02:31<12:16,  1.26it/s]\u001b[A\n",
      "Iteration:  21%|██        | 249/1178 [02:32<10:31,  1.47it/s]\u001b[A\n",
      "Iteration:  21%|██        | 250/1178 [02:32<09:32,  1.62it/s]\u001b[A\n",
      "Iteration:  21%|██▏       | 251/1178 [02:33<08:35,  1.80it/s]\u001b[A\n",
      "Iteration:  21%|██▏       | 252/1178 [02:33<07:56,  1.94it/s]\u001b[A\n",
      "Iteration:  21%|██▏       | 253/1178 [02:33<07:29,  2.06it/s]\u001b[A\n",
      "Iteration:  22%|██▏       | 254/1178 [02:34<07:10,  2.15it/s]\u001b[A\n",
      "Iteration:  22%|██▏       | 255/1178 [02:34<06:57,  2.21it/s]\u001b[A\n",
      "Iteration:  22%|██▏       | 256/1178 [02:35<06:47,  2.26it/s]\u001b[A\n",
      "Iteration:  22%|██▏       | 257/1178 [02:35<06:40,  2.30it/s]\u001b[A\n",
      "Iteration:  22%|██▏       | 258/1178 [02:36<06:35,  2.32it/s]\u001b[A\n",
      "Iteration:  22%|██▏       | 259/1178 [02:36<06:32,  2.34it/s]\u001b[A\n",
      "Iteration:  22%|██▏       | 260/1178 [02:36<06:43,  2.27it/s]\u001b[A\n",
      "Iteration:  22%|██▏       | 261/1178 [02:37<06:39,  2.29it/s]\u001b[A\n",
      "Iteration:  22%|██▏       | 262/1178 [02:37<06:35,  2.32it/s]\u001b[A\n",
      "Iteration:  22%|██▏       | 263/1178 [02:38<06:31,  2.34it/s]\u001b[A\n",
      "Iteration:  22%|██▏       | 264/1178 [02:38<06:28,  2.35it/s]\u001b[A\n",
      "Iteration:  22%|██▏       | 265/1178 [02:39<06:26,  2.36it/s]\u001b[A\n",
      "Iteration:  23%|██▎       | 266/1178 [02:39<06:24,  2.37it/s]\u001b[A\n",
      "Iteration:  23%|██▎       | 267/1178 [02:39<06:22,  2.38it/s]\u001b[A\n",
      "Iteration:  23%|██▎       | 268/1178 [02:40<06:20,  2.39it/s]\u001b[A\n",
      "Iteration:  23%|██▎       | 269/1178 [02:40<06:20,  2.39it/s]\u001b[A\n",
      "Iteration:  23%|██▎       | 270/1178 [02:41<06:33,  2.31it/s]\u001b[A\n",
      "Iteration:  23%|██▎       | 271/1178 [02:41<06:29,  2.33it/s]\u001b[A\n",
      "Iteration:  23%|██▎       | 272/1178 [02:42<06:25,  2.35it/s]\u001b[A\n",
      "Iteration:  23%|██▎       | 273/1178 [02:42<06:22,  2.36it/s]\u001b[A\n",
      "Iteration:  23%|██▎       | 274/1178 [02:42<06:21,  2.37it/s]\u001b[A\n",
      "Iteration:  23%|██▎       | 275/1178 [02:43<06:20,  2.37it/s]\u001b[A\n",
      "Iteration:  23%|██▎       | 276/1178 [02:43<06:19,  2.38it/s]\u001b[A\n",
      "Iteration:  24%|██▎       | 277/1178 [02:44<06:18,  2.38it/s]\u001b[A\n",
      "Iteration:  24%|██▎       | 278/1178 [02:44<06:17,  2.38it/s]\u001b[A\n",
      "Iteration:  24%|██▎       | 279/1178 [02:44<06:17,  2.38it/s]\u001b[A\n",
      "Iteration:  24%|██▍       | 280/1178 [02:45<06:30,  2.30it/s]\u001b[A\n",
      "Iteration:  24%|██▍       | 281/1178 [02:45<06:26,  2.32it/s]\u001b[A\n",
      "Iteration:  24%|██▍       | 282/1178 [02:47<12:00,  1.24it/s]\u001b[A\n",
      "Iteration:  24%|██▍       | 283/1178 [02:47<10:16,  1.45it/s]\u001b[A\n",
      "Iteration:  24%|██▍       | 284/1178 [02:48<09:03,  1.65it/s]\u001b[A\n",
      "Iteration:  24%|██▍       | 285/1178 [02:48<08:13,  1.81it/s]\u001b[A\n",
      "Iteration:  24%|██▍       | 286/1178 [02:49<07:36,  1.95it/s]\u001b[A\n",
      "Iteration:  24%|██▍       | 287/1178 [02:49<07:11,  2.07it/s]\u001b[A\n",
      "Iteration:  24%|██▍       | 288/1178 [02:50<06:53,  2.15it/s]\u001b[A\n",
      "Iteration:  25%|██▍       | 289/1178 [02:50<06:40,  2.22it/s]\u001b[A\n",
      "Iteration:  25%|██▍       | 290/1178 [02:50<06:44,  2.19it/s]\u001b[A\n",
      "Iteration:  25%|██▍       | 291/1178 [02:51<06:34,  2.25it/s]\u001b[A\n",
      "Iteration:  25%|██▍       | 292/1178 [02:51<06:27,  2.29it/s]\u001b[A\n",
      "Iteration:  25%|██▍       | 293/1178 [02:52<06:22,  2.31it/s]\u001b[A\n",
      "Iteration:  25%|██▍       | 294/1178 [02:52<06:18,  2.34it/s]\u001b[A\n",
      "Iteration:  25%|██▌       | 295/1178 [02:53<06:15,  2.35it/s]\u001b[A\n",
      "Iteration:  25%|██▌       | 296/1178 [02:53<06:12,  2.36it/s]\u001b[A\n",
      "Iteration:  25%|██▌       | 297/1178 [02:53<06:11,  2.37it/s]\u001b[A\n",
      "Iteration:  25%|██▌       | 298/1178 [02:54<06:10,  2.37it/s]\u001b[A\n",
      "Iteration:  25%|██▌       | 299/1178 [02:54<06:10,  2.38it/s]\u001b[A\n",
      "Iteration:  25%|██▌       | 300/1178 [02:55<06:22,  2.30it/s]\u001b[A\n",
      "Iteration:  26%|██▌       | 301/1178 [02:55<06:17,  2.32it/s]\u001b[A\n",
      "Iteration:  26%|██▌       | 302/1178 [02:56<06:13,  2.34it/s]\u001b[A\n",
      "Iteration:  26%|██▌       | 303/1178 [02:56<06:11,  2.36it/s]\u001b[A\n",
      "Iteration:  26%|██▌       | 304/1178 [02:56<06:10,  2.36it/s]\u001b[A\n",
      "Iteration:  26%|██▌       | 305/1178 [02:57<06:09,  2.37it/s]\u001b[A\n",
      "Iteration:  26%|██▌       | 306/1178 [02:57<06:07,  2.37it/s]\u001b[A\n",
      "Iteration:  26%|██▌       | 307/1178 [02:58<06:06,  2.37it/s]\u001b[A\n",
      "Iteration:  26%|██▌       | 308/1178 [02:58<06:06,  2.38it/s]\u001b[A\n",
      "Iteration:  26%|██▌       | 309/1178 [02:58<06:05,  2.38it/s]\u001b[A\n",
      "Iteration:  26%|██▋       | 310/1178 [02:59<06:17,  2.30it/s]\u001b[A\n",
      "Iteration:  26%|██▋       | 311/1178 [02:59<06:13,  2.32it/s]\u001b[A\n",
      "Iteration:  26%|██▋       | 312/1178 [03:00<06:10,  2.34it/s]\u001b[A\n",
      "Iteration:  27%|██▋       | 313/1178 [03:00<06:08,  2.35it/s]\u001b[A\n",
      "Iteration:  27%|██▋       | 314/1178 [03:01<06:06,  2.35it/s]\u001b[A\n",
      "Iteration:  27%|██▋       | 315/1178 [03:01<06:05,  2.36it/s]\u001b[A\n",
      "Iteration:  27%|██▋       | 316/1178 [03:03<11:28,  1.25it/s]\u001b[A\n",
      "Iteration:  27%|██▋       | 317/1178 [03:03<09:50,  1.46it/s]\u001b[A\n",
      "Iteration:  27%|██▋       | 318/1178 [03:04<08:41,  1.65it/s]\u001b[A\n",
      "Iteration:  27%|██▋       | 319/1178 [03:04<07:52,  1.82it/s]\u001b[A\n",
      "Iteration:  27%|██▋       | 320/1178 [03:04<07:31,  1.90it/s]\u001b[A\n",
      "Iteration:  27%|██▋       | 321/1178 [03:05<07:04,  2.02it/s]\u001b[A\n",
      "Iteration:  27%|██▋       | 322/1178 [03:05<06:44,  2.12it/s]\u001b[A\n",
      "Iteration:  27%|██▋       | 323/1178 [03:06<06:30,  2.19it/s]\u001b[A\n",
      "Iteration:  28%|██▊       | 324/1178 [03:06<06:20,  2.25it/s]\u001b[A\n",
      "Iteration:  28%|██▊       | 325/1178 [03:07<06:13,  2.28it/s]\u001b[A\n",
      "Iteration:  28%|██▊       | 326/1178 [03:07<06:08,  2.31it/s]\u001b[A\n",
      "Iteration:  28%|██▊       | 327/1178 [03:07<06:05,  2.33it/s]\u001b[A\n",
      "Iteration:  28%|██▊       | 328/1178 [03:08<06:03,  2.34it/s]\u001b[A\n",
      "Iteration:  28%|██▊       | 329/1178 [03:08<06:00,  2.36it/s]\u001b[A\n",
      "Iteration:  28%|██▊       | 330/1178 [03:09<06:11,  2.28it/s]\u001b[A\n",
      "Iteration:  28%|██▊       | 331/1178 [03:09<06:06,  2.31it/s]\u001b[A\n",
      "Iteration:  28%|██▊       | 332/1178 [03:10<06:03,  2.33it/s]\u001b[A\n",
      "Iteration:  28%|██▊       | 333/1178 [03:10<06:01,  2.34it/s]\u001b[A\n",
      "Iteration:  28%|██▊       | 334/1178 [03:10<05:58,  2.35it/s]\u001b[A\n",
      "Iteration:  28%|██▊       | 335/1178 [03:11<05:56,  2.36it/s]\u001b[A\n",
      "Iteration:  29%|██▊       | 336/1178 [03:11<05:55,  2.37it/s]\u001b[A\n",
      "Iteration:  29%|██▊       | 337/1178 [03:12<05:54,  2.37it/s]\u001b[A\n",
      "Iteration:  29%|██▊       | 338/1178 [03:12<05:53,  2.38it/s]\u001b[A\n",
      "Iteration:  29%|██▉       | 339/1178 [03:12<05:52,  2.38it/s]\u001b[A\n",
      "Iteration:  29%|██▉       | 340/1178 [03:13<06:04,  2.30it/s]\u001b[A\n",
      "Iteration:  29%|██▉       | 341/1178 [03:13<06:01,  2.32it/s]\u001b[A\n",
      "Iteration:  29%|██▉       | 342/1178 [03:14<05:57,  2.34it/s]\u001b[A\n",
      "Iteration:  29%|██▉       | 343/1178 [03:14<05:55,  2.35it/s]\u001b[A\n",
      "Iteration:  29%|██▉       | 344/1178 [03:15<05:52,  2.36it/s]\u001b[A\n",
      "Iteration:  29%|██▉       | 345/1178 [03:15<05:51,  2.37it/s]\u001b[A\n",
      "Iteration:  29%|██▉       | 346/1178 [03:15<05:50,  2.37it/s]\u001b[A\n",
      "Iteration:  29%|██▉       | 347/1178 [03:16<05:49,  2.38it/s]\u001b[A\n",
      "Iteration:  30%|██▉       | 348/1178 [03:16<05:49,  2.38it/s]\u001b[A\n",
      "Iteration:  30%|██▉       | 349/1178 [03:17<05:48,  2.38it/s]\u001b[A\n",
      "Iteration:  30%|██▉       | 350/1178 [03:18<11:11,  1.23it/s]\u001b[A\n",
      "Iteration:  30%|██▉       | 351/1178 [03:19<09:33,  1.44it/s]\u001b[A\n",
      "Iteration:  30%|██▉       | 352/1178 [03:19<08:25,  1.64it/s]\u001b[A\n",
      "Iteration:  30%|██▉       | 353/1178 [03:20<07:37,  1.80it/s]\u001b[A\n",
      "Iteration:  30%|███       | 354/1178 [03:20<07:03,  1.94it/s]\u001b[A\n",
      "Iteration:  30%|███       | 355/1178 [03:21<06:40,  2.06it/s]\u001b[A\n",
      "Iteration:  30%|███       | 356/1178 [03:21<06:22,  2.15it/s]\u001b[A\n",
      "Iteration:  30%|███       | 357/1178 [03:21<06:11,  2.21it/s]\u001b[A\n",
      "Iteration:  30%|███       | 358/1178 [03:22<06:02,  2.26it/s]\u001b[A\n",
      "Iteration:  30%|███       | 359/1178 [03:22<05:58,  2.28it/s]\u001b[A\n",
      "Iteration:  31%|███       | 360/1178 [03:23<06:06,  2.23it/s]\u001b[A\n",
      "Iteration:  31%|███       | 361/1178 [03:23<05:59,  2.28it/s]\u001b[A\n",
      "Iteration:  31%|███       | 362/1178 [03:24<05:54,  2.30it/s]\u001b[A\n",
      "Iteration:  31%|███       | 363/1178 [03:24<05:50,  2.32it/s]\u001b[A\n",
      "Iteration:  31%|███       | 364/1178 [03:24<05:47,  2.34it/s]\u001b[A\n",
      "Iteration:  31%|███       | 365/1178 [03:25<05:45,  2.35it/s]\u001b[A\n",
      "Iteration:  31%|███       | 366/1178 [03:25<05:44,  2.36it/s]\u001b[A\n",
      "Iteration:  31%|███       | 367/1178 [03:26<05:42,  2.37it/s]\u001b[A\n",
      "Iteration:  31%|███       | 368/1178 [03:26<05:42,  2.37it/s]\u001b[A\n",
      "Iteration:  31%|███▏      | 369/1178 [03:26<05:41,  2.37it/s]\u001b[A\n",
      "Iteration:  31%|███▏      | 370/1178 [03:27<05:52,  2.29it/s]\u001b[A\n",
      "Iteration:  31%|███▏      | 371/1178 [03:27<05:48,  2.32it/s]\u001b[A\n",
      "Iteration:  32%|███▏      | 372/1178 [03:28<05:44,  2.34it/s]\u001b[A\n",
      "Iteration:  32%|███▏      | 373/1178 [03:28<05:43,  2.34it/s]\u001b[A\n",
      "Iteration:  32%|███▏      | 374/1178 [03:29<05:41,  2.36it/s]\u001b[A\n",
      "Iteration:  32%|███▏      | 375/1178 [03:29<05:39,  2.36it/s]\u001b[A\n",
      "Iteration:  32%|███▏      | 376/1178 [03:29<05:38,  2.37it/s]\u001b[A\n",
      "Iteration:  32%|███▏      | 377/1178 [03:30<05:37,  2.37it/s]\u001b[A\n",
      "Iteration:  32%|███▏      | 378/1178 [03:30<05:36,  2.38it/s]\u001b[A\n",
      "Iteration:  32%|███▏      | 379/1178 [03:31<05:35,  2.38it/s]\u001b[A\n",
      "Iteration:  32%|███▏      | 380/1178 [03:31<05:46,  2.30it/s]\u001b[A\n",
      "Iteration:  32%|███▏      | 381/1178 [03:32<05:42,  2.33it/s]\u001b[A\n",
      "Iteration:  32%|███▏      | 382/1178 [03:32<05:39,  2.34it/s]\u001b[A\n",
      "Iteration:  33%|███▎      | 383/1178 [03:32<05:37,  2.36it/s]\u001b[A\n",
      "Iteration:  33%|███▎      | 384/1178 [03:34<10:37,  1.25it/s]\u001b[A\n",
      "Iteration:  33%|███▎      | 385/1178 [03:35<09:06,  1.45it/s]\u001b[A\n",
      "Iteration:  33%|███▎      | 386/1178 [03:35<08:01,  1.64it/s]\u001b[A\n",
      "Iteration:  33%|███▎      | 387/1178 [03:35<07:16,  1.81it/s]\u001b[A\n",
      "Iteration:  33%|███▎      | 388/1178 [03:36<06:45,  1.95it/s]\u001b[A\n",
      "Iteration:  33%|███▎      | 389/1178 [03:36<06:23,  2.06it/s]\u001b[A\n",
      "Iteration:  33%|███▎      | 390/1178 [03:37<06:19,  2.08it/s]\u001b[A\n",
      "Iteration:  33%|███▎      | 391/1178 [03:37<06:07,  2.14it/s]\u001b[A\n",
      "Iteration:  33%|███▎      | 392/1178 [03:38<05:57,  2.20it/s]\u001b[A\n",
      "Iteration:  33%|███▎      | 393/1178 [03:38<05:49,  2.25it/s]\u001b[A\n",
      "Iteration:  33%|███▎      | 394/1178 [03:38<05:43,  2.28it/s]\u001b[A\n",
      "Iteration:  34%|███▎      | 395/1178 [03:39<05:39,  2.31it/s]\u001b[A\n",
      "Iteration:  34%|███▎      | 396/1178 [03:39<05:36,  2.32it/s]\u001b[A\n",
      "Iteration:  34%|███▎      | 397/1178 [03:40<05:34,  2.33it/s]\u001b[A\n",
      "Iteration:  34%|███▍      | 398/1178 [03:40<05:32,  2.34it/s]\u001b[A\n",
      "Iteration:  34%|███▍      | 399/1178 [03:41<05:33,  2.34it/s]\u001b[A\n",
      "Iteration:  34%|███▍      | 400/1178 [03:41<05:43,  2.27it/s]\u001b[A\n",
      "Iteration:  34%|███▍      | 401/1178 [03:41<05:38,  2.30it/s]\u001b[A\n",
      "Iteration:  34%|███▍      | 402/1178 [03:42<05:34,  2.32it/s]\u001b[A\n",
      "Iteration:  34%|███▍      | 403/1178 [03:42<05:32,  2.33it/s]\u001b[A\n",
      "Iteration:  34%|███▍      | 404/1178 [03:43<05:31,  2.33it/s]\u001b[A\n",
      "Iteration:  34%|███▍      | 405/1178 [03:43<05:30,  2.34it/s]\u001b[A\n",
      "Iteration:  34%|███▍      | 406/1178 [03:44<05:28,  2.35it/s]\u001b[A\n",
      "Iteration:  35%|███▍      | 407/1178 [03:44<05:27,  2.36it/s]\u001b[A\n",
      "Iteration:  35%|███▍      | 408/1178 [03:44<05:25,  2.37it/s]\u001b[A\n",
      "Iteration:  35%|███▍      | 409/1178 [03:45<05:24,  2.37it/s]\u001b[A\n",
      "Iteration:  35%|███▍      | 410/1178 [03:45<05:36,  2.28it/s]\u001b[A\n",
      "Iteration:  35%|███▍      | 411/1178 [03:46<05:32,  2.31it/s]\u001b[A\n",
      "Iteration:  35%|███▍      | 412/1178 [03:46<05:29,  2.33it/s]\u001b[A\n",
      "Iteration:  35%|███▌      | 413/1178 [03:47<05:28,  2.33it/s]\u001b[A\n",
      "Iteration:  35%|███▌      | 414/1178 [03:47<05:26,  2.34it/s]\u001b[A\n",
      "Iteration:  35%|███▌      | 415/1178 [03:47<05:24,  2.35it/s]\u001b[A\n",
      "Iteration:  35%|███▌      | 416/1178 [03:48<05:23,  2.36it/s]\u001b[A\n",
      "Iteration:  35%|███▌      | 417/1178 [03:48<05:21,  2.36it/s]\u001b[A\n",
      "Iteration:  35%|███▌      | 418/1178 [03:50<10:13,  1.24it/s]\u001b[A\n",
      "Iteration:  36%|███▌      | 419/1178 [03:50<08:45,  1.44it/s]\u001b[A\n",
      "Iteration:  36%|███▌      | 420/1178 [03:51<07:55,  1.59it/s]\u001b[A\n",
      "Iteration:  36%|███▌      | 421/1178 [03:51<07:09,  1.76it/s]\u001b[A\n",
      "Iteration:  36%|███▌      | 422/1178 [03:52<06:36,  1.91it/s]\u001b[A\n",
      "Iteration:  36%|███▌      | 423/1178 [03:52<06:13,  2.02it/s]\u001b[A\n",
      "Iteration:  36%|███▌      | 424/1178 [03:53<05:56,  2.11it/s]\u001b[A\n",
      "Iteration:  36%|███▌      | 425/1178 [03:53<05:44,  2.19it/s]\u001b[A\n",
      "Iteration:  36%|███▌      | 426/1178 [03:53<05:35,  2.24it/s]\u001b[A\n",
      "Iteration:  36%|███▌      | 427/1178 [03:54<05:29,  2.28it/s]\u001b[A\n",
      "Iteration:  36%|███▋      | 428/1178 [03:54<05:24,  2.31it/s]\u001b[A\n",
      "Iteration:  36%|███▋      | 429/1178 [03:55<05:21,  2.33it/s]\u001b[A\n",
      "Iteration:  37%|███▋      | 430/1178 [03:55<05:30,  2.26it/s]\u001b[A\n",
      "Iteration:  37%|███▋      | 431/1178 [03:56<05:26,  2.29it/s]\u001b[A\n",
      "Iteration:  37%|███▋      | 432/1178 [03:56<05:22,  2.31it/s]\u001b[A\n",
      "Iteration:  37%|███▋      | 433/1178 [03:56<05:19,  2.33it/s]\u001b[A\n",
      "Iteration:  37%|███▋      | 434/1178 [03:57<05:17,  2.35it/s]\u001b[A\n",
      "Iteration:  37%|███▋      | 435/1178 [03:57<05:17,  2.34it/s]\u001b[A\n",
      "Iteration:  37%|███▋      | 436/1178 [03:58<05:15,  2.35it/s]\u001b[A\n",
      "Iteration:  37%|███▋      | 437/1178 [03:58<05:14,  2.35it/s]\u001b[A\n",
      "Iteration:  37%|███▋      | 438/1178 [03:59<05:13,  2.36it/s]\u001b[A\n",
      "Iteration:  37%|███▋      | 439/1178 [03:59<05:13,  2.36it/s]\u001b[A\n",
      "Iteration:  37%|███▋      | 440/1178 [03:59<05:23,  2.28it/s]\u001b[A\n",
      "Iteration:  37%|███▋      | 441/1178 [04:00<05:20,  2.30it/s]\u001b[A\n",
      "Iteration:  38%|███▊      | 442/1178 [04:00<05:17,  2.32it/s]\u001b[A\n",
      "Iteration:  38%|███▊      | 443/1178 [04:01<05:14,  2.34it/s]\u001b[A\n",
      "Iteration:  38%|███▊      | 444/1178 [04:01<05:11,  2.36it/s]\u001b[A\n",
      "Iteration:  38%|███▊      | 445/1178 [04:02<05:09,  2.37it/s]\u001b[A\n",
      "Iteration:  38%|███▊      | 446/1178 [04:02<05:08,  2.37it/s]\u001b[A\n",
      "Iteration:  38%|███▊      | 447/1178 [04:02<05:07,  2.38it/s]\u001b[A\n",
      "Iteration:  38%|███▊      | 448/1178 [04:03<05:06,  2.38it/s]\u001b[A\n",
      "Iteration:  38%|███▊      | 449/1178 [04:03<05:05,  2.38it/s]\u001b[A\n",
      "Iteration:  38%|███▊      | 450/1178 [04:04<05:17,  2.30it/s]\u001b[A\n",
      "Iteration:  38%|███▊      | 451/1178 [04:04<05:14,  2.32it/s]\u001b[A\n",
      "Iteration:  38%|███▊      | 452/1178 [04:06<09:47,  1.24it/s]\u001b[A\n",
      "Iteration:  38%|███▊      | 453/1178 [04:06<08:23,  1.44it/s]\u001b[A\n",
      "Iteration:  39%|███▊      | 454/1178 [04:07<07:24,  1.63it/s]\u001b[A\n",
      "Iteration:  39%|███▊      | 455/1178 [04:07<06:42,  1.80it/s]\u001b[A\n",
      "Iteration:  39%|███▊      | 456/1178 [04:07<06:12,  1.94it/s]\u001b[A\n",
      "Iteration:  39%|███▉      | 457/1178 [04:08<05:52,  2.05it/s]\u001b[A\n",
      "Iteration:  39%|███▉      | 458/1178 [04:08<05:37,  2.13it/s]\u001b[A\n",
      "Iteration:  39%|███▉      | 459/1178 [04:09<05:27,  2.20it/s]\u001b[A\n",
      "Iteration:  39%|███▉      | 460/1178 [04:09<05:31,  2.17it/s]\u001b[A\n",
      "Iteration:  39%|███▉      | 461/1178 [04:10<05:22,  2.22it/s]\u001b[A\n",
      "Iteration:  39%|███▉      | 462/1178 [04:10<05:16,  2.26it/s]\u001b[A\n",
      "Iteration:  39%|███▉      | 463/1178 [04:10<05:13,  2.28it/s]\u001b[A\n",
      "Iteration:  39%|███▉      | 464/1178 [04:11<05:09,  2.30it/s]\u001b[A\n",
      "Iteration:  39%|███▉      | 465/1178 [04:11<05:07,  2.32it/s]\u001b[A\n",
      "Iteration:  40%|███▉      | 466/1178 [04:12<05:05,  2.33it/s]\u001b[A\n",
      "Iteration:  40%|███▉      | 467/1178 [04:12<05:03,  2.34it/s]\u001b[A\n",
      "Iteration:  40%|███▉      | 468/1178 [04:13<05:02,  2.35it/s]\u001b[A\n",
      "Iteration:  40%|███▉      | 469/1178 [04:13<05:01,  2.35it/s]\u001b[A\n",
      "Iteration:  40%|███▉      | 470/1178 [04:14<05:11,  2.27it/s]\u001b[A\n",
      "Iteration:  40%|███▉      | 471/1178 [04:14<05:07,  2.30it/s]\u001b[A\n",
      "Iteration:  40%|████      | 472/1178 [04:14<05:04,  2.32it/s]\u001b[A\n",
      "Iteration:  40%|████      | 473/1178 [04:15<05:03,  2.32it/s]\u001b[A\n",
      "Iteration:  40%|████      | 474/1178 [04:15<05:00,  2.34it/s]\u001b[A\n",
      "Iteration:  40%|████      | 475/1178 [04:16<05:00,  2.34it/s]\u001b[A\n",
      "Iteration:  40%|████      | 476/1178 [04:16<04:59,  2.35it/s]\u001b[A\n",
      "Iteration:  40%|████      | 477/1178 [04:16<04:59,  2.34it/s]\u001b[A\n",
      "Iteration:  41%|████      | 478/1178 [04:17<04:58,  2.34it/s]\u001b[A\n",
      "Iteration:  41%|████      | 479/1178 [04:17<04:57,  2.35it/s]\u001b[A\n",
      "Iteration:  41%|████      | 480/1178 [04:18<05:06,  2.27it/s]\u001b[A\n",
      "Iteration:  41%|████      | 481/1178 [04:18<05:02,  2.30it/s]\u001b[A\n",
      "Iteration:  41%|████      | 482/1178 [04:19<05:00,  2.32it/s]\u001b[A\n",
      "Iteration:  41%|████      | 483/1178 [04:19<04:58,  2.33it/s]\u001b[A\n",
      "Iteration:  41%|████      | 484/1178 [04:20<04:57,  2.33it/s]\u001b[A\n",
      "Iteration:  41%|████      | 485/1178 [04:20<04:55,  2.34it/s]\u001b[A\n",
      "Iteration:  41%|████▏     | 486/1178 [04:22<09:21,  1.23it/s]\u001b[A\n",
      "Iteration:  41%|████▏     | 487/1178 [04:22<07:59,  1.44it/s]\u001b[A\n",
      "Iteration:  41%|████▏     | 488/1178 [04:22<07:02,  1.63it/s]\u001b[A\n",
      "Iteration:  42%|████▏     | 489/1178 [04:23<06:23,  1.80it/s]\u001b[A\n",
      "Iteration:  42%|████▏     | 490/1178 [04:23<06:06,  1.88it/s]\u001b[A\n",
      "Iteration:  42%|████▏     | 491/1178 [04:24<05:43,  2.00it/s]\u001b[A\n",
      "Iteration:  42%|████▏     | 492/1178 [04:24<05:26,  2.10it/s]\u001b[A\n",
      "Iteration:  42%|████▏     | 493/1178 [04:25<05:15,  2.17it/s]\u001b[A\n",
      "Iteration:  42%|████▏     | 494/1178 [04:25<05:07,  2.23it/s]\u001b[A\n",
      "Iteration:  42%|████▏     | 495/1178 [04:25<05:01,  2.27it/s]\u001b[A\n",
      "Iteration:  42%|████▏     | 496/1178 [04:26<04:56,  2.30it/s]\u001b[A\n",
      "Iteration:  42%|████▏     | 497/1178 [04:26<04:53,  2.32it/s]\u001b[A\n",
      "Iteration:  42%|████▏     | 498/1178 [04:27<04:52,  2.33it/s]\u001b[A\n",
      "Iteration:  42%|████▏     | 499/1178 [04:27<04:49,  2.34it/s]\u001b[A\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Iteration:  42%|████▏     | 500/1178 [04:28<04:58,  2.27it/s]\u001b[A\n",
      "Iteration:  43%|████▎     | 501/1178 [04:28<04:54,  2.29it/s]\u001b[A\n",
      "Iteration:  43%|████▎     | 502/1178 [04:29<04:51,  2.32it/s]\u001b[A\n",
      "Iteration:  43%|████▎     | 503/1178 [04:29<04:50,  2.33it/s]\u001b[A\n",
      "Iteration:  43%|████▎     | 504/1178 [04:29<04:48,  2.34it/s]\u001b[A\n",
      "Iteration:  43%|████▎     | 505/1178 [04:30<04:47,  2.34it/s]\u001b[A\n",
      "Iteration:  43%|████▎     | 506/1178 [04:30<04:45,  2.35it/s]\u001b[A\n",
      "Iteration:  43%|████▎     | 507/1178 [04:31<04:44,  2.35it/s]\u001b[A\n",
      "Iteration:  43%|████▎     | 508/1178 [04:31<04:43,  2.36it/s]\u001b[A\n",
      "Iteration:  43%|████▎     | 509/1178 [04:31<04:43,  2.36it/s]\u001b[A\n",
      "Iteration:  43%|████▎     | 510/1178 [04:32<04:53,  2.28it/s]\u001b[A\n",
      "Iteration:  43%|████▎     | 511/1178 [04:32<04:50,  2.30it/s]\u001b[A\n",
      "Iteration:  43%|████▎     | 512/1178 [04:33<04:47,  2.31it/s]\u001b[A\n",
      "Iteration:  44%|████▎     | 513/1178 [04:33<04:44,  2.33it/s]\u001b[A\n",
      "Iteration:  44%|████▎     | 514/1178 [04:34<04:44,  2.34it/s]\u001b[A\n",
      "Iteration:  44%|████▎     | 515/1178 [04:34<04:42,  2.35it/s]\u001b[A\n",
      "Iteration:  44%|████▍     | 516/1178 [04:34<04:40,  2.36it/s]\u001b[A\n",
      "Iteration:  44%|████▍     | 517/1178 [04:35<04:38,  2.37it/s]\u001b[A\n",
      "Iteration:  44%|████▍     | 518/1178 [04:35<04:37,  2.38it/s]\u001b[A\n",
      "Iteration:  44%|████▍     | 519/1178 [04:36<04:36,  2.38it/s]\u001b[A\n",
      "Iteration:  44%|████▍     | 520/1178 [04:38<10:18,  1.06it/s]\u001b[A\n",
      "Iteration:  44%|████▍     | 521/1178 [04:38<08:35,  1.27it/s]\u001b[A\n",
      "Iteration:  44%|████▍     | 522/1178 [04:39<07:24,  1.48it/s]\u001b[A\n",
      "Iteration:  44%|████▍     | 523/1178 [04:39<06:34,  1.66it/s]\u001b[A\n",
      "Iteration:  44%|████▍     | 524/1178 [04:40<05:58,  1.82it/s]\u001b[A\n",
      "Iteration:  45%|████▍     | 525/1178 [04:40<05:34,  1.95it/s]\u001b[A\n",
      "Iteration:  45%|████▍     | 526/1178 [04:40<05:15,  2.07it/s]\u001b[A\n",
      "Iteration:  45%|████▍     | 527/1178 [04:41<05:03,  2.15it/s]\u001b[A\n",
      "Iteration:  45%|████▍     | 528/1178 [04:41<04:53,  2.21it/s]\u001b[A\n",
      "Iteration:  45%|████▍     | 529/1178 [04:42<04:48,  2.25it/s]\u001b[A\n",
      "Iteration:  45%|████▍     | 530/1178 [04:42<04:53,  2.21it/s]\u001b[A\n",
      "Iteration:  45%|████▌     | 531/1178 [04:43<04:46,  2.26it/s]\u001b[A\n",
      "Iteration:  45%|████▌     | 532/1178 [04:43<04:41,  2.30it/s]\u001b[A\n",
      "Iteration:  45%|████▌     | 533/1178 [04:43<04:37,  2.32it/s]\u001b[A\n",
      "Iteration:  45%|████▌     | 534/1178 [04:44<04:34,  2.34it/s]\u001b[A\n",
      "Iteration:  45%|████▌     | 535/1178 [04:44<04:33,  2.35it/s]\u001b[A\n",
      "Iteration:  46%|████▌     | 536/1178 [04:45<04:31,  2.37it/s]\u001b[A\n",
      "Iteration:  46%|████▌     | 537/1178 [04:45<04:30,  2.37it/s]\u001b[A\n",
      "Iteration:  46%|████▌     | 538/1178 [04:46<04:29,  2.38it/s]\u001b[A\n",
      "Iteration:  46%|████▌     | 539/1178 [04:46<04:28,  2.38it/s]\u001b[A\n",
      "Iteration:  46%|████▌     | 540/1178 [04:46<04:37,  2.30it/s]\u001b[A\n",
      "Iteration:  46%|████▌     | 541/1178 [04:47<04:35,  2.32it/s]\u001b[A\n",
      "Iteration:  46%|████▌     | 542/1178 [04:47<04:32,  2.34it/s]\u001b[A\n",
      "Iteration:  46%|████▌     | 543/1178 [04:48<04:30,  2.35it/s]\u001b[A\n",
      "Iteration:  46%|████▌     | 544/1178 [04:48<04:29,  2.35it/s]\u001b[A\n",
      "Iteration:  46%|████▋     | 545/1178 [04:49<04:28,  2.36it/s]\u001b[A\n",
      "Iteration:  46%|████▋     | 546/1178 [04:49<04:26,  2.37it/s]\u001b[A\n",
      "Iteration:  46%|████▋     | 547/1178 [04:49<04:26,  2.37it/s]\u001b[A\n",
      "Iteration:  47%|████▋     | 548/1178 [04:50<04:25,  2.37it/s]\u001b[A\n",
      "Iteration:  47%|████▋     | 549/1178 [04:50<04:25,  2.37it/s]\u001b[A\n",
      "Iteration:  47%|████▋     | 550/1178 [04:51<04:34,  2.29it/s]\u001b[A\n",
      "Iteration:  47%|████▋     | 551/1178 [04:51<04:31,  2.31it/s]\u001b[A\n",
      "Iteration:  47%|████▋     | 552/1178 [04:52<04:28,  2.33it/s]\u001b[A\n",
      "Iteration:  47%|████▋     | 553/1178 [04:52<04:26,  2.35it/s]\u001b[A\n",
      "Iteration:  47%|████▋     | 554/1178 [04:54<08:31,  1.22it/s]\u001b[A\n",
      "Iteration:  47%|████▋     | 555/1178 [04:54<07:16,  1.43it/s]\u001b[A\n",
      "Iteration:  47%|████▋     | 556/1178 [04:55<06:24,  1.62it/s]\u001b[A\n",
      "Iteration:  47%|████▋     | 557/1178 [04:55<05:47,  1.78it/s]\u001b[A\n",
      "Iteration:  47%|████▋     | 558/1178 [04:55<05:21,  1.93it/s]\u001b[A\n",
      "Iteration:  47%|████▋     | 559/1178 [04:56<05:04,  2.03it/s]\u001b[A\n",
      "Iteration:  48%|████▊     | 560/1178 [04:56<05:00,  2.06it/s]\u001b[A\n",
      "Iteration:  48%|████▊     | 561/1178 [04:57<04:48,  2.14it/s]\u001b[A\n",
      "Iteration:  48%|████▊     | 562/1178 [04:57<04:39,  2.20it/s]\u001b[A\n",
      "Iteration:  48%|████▊     | 563/1178 [04:58<04:33,  2.25it/s]\u001b[A\n",
      "Iteration:  48%|████▊     | 564/1178 [04:58<04:29,  2.28it/s]\u001b[A\n",
      "Iteration:  48%|████▊     | 565/1178 [04:58<04:26,  2.30it/s]\u001b[A\n",
      "Iteration:  48%|████▊     | 566/1178 [04:59<04:23,  2.32it/s]\u001b[A\n",
      "Iteration:  48%|████▊     | 567/1178 [04:59<04:21,  2.33it/s]\u001b[A\n",
      "Iteration:  48%|████▊     | 568/1178 [05:00<04:20,  2.34it/s]\u001b[A\n",
      "Iteration:  48%|████▊     | 569/1178 [05:00<04:19,  2.35it/s]\u001b[A\n",
      "Iteration:  48%|████▊     | 570/1178 [05:01<04:27,  2.27it/s]\u001b[A\n",
      "Iteration:  48%|████▊     | 571/1178 [05:01<04:24,  2.30it/s]\u001b[A\n",
      "Iteration:  49%|████▊     | 572/1178 [05:01<04:21,  2.31it/s]\u001b[A\n",
      "Iteration:  49%|████▊     | 573/1178 [05:02<04:19,  2.33it/s]\u001b[A\n",
      "Iteration:  49%|████▊     | 574/1178 [05:02<04:17,  2.35it/s]\u001b[A\n",
      "Iteration:  49%|████▉     | 575/1178 [05:03<04:17,  2.34it/s]\u001b[A\n",
      "Iteration:  49%|████▉     | 576/1178 [05:03<04:15,  2.36it/s]\u001b[A\n",
      "Iteration:  49%|████▉     | 577/1178 [05:04<04:15,  2.35it/s]\u001b[A\n",
      "Iteration:  49%|████▉     | 578/1178 [05:04<04:14,  2.35it/s]\u001b[A\n",
      "Iteration:  49%|████▉     | 579/1178 [05:04<04:14,  2.35it/s]\u001b[A\n",
      "Iteration:  49%|████▉     | 580/1178 [05:05<04:23,  2.27it/s]\u001b[A\n",
      "Iteration:  49%|████▉     | 581/1178 [05:05<04:19,  2.30it/s]\u001b[A\n",
      "Iteration:  49%|████▉     | 582/1178 [05:06<04:17,  2.31it/s]\u001b[A\n",
      "Iteration:  49%|████▉     | 583/1178 [05:06<04:15,  2.33it/s]\u001b[A\n",
      "Iteration:  50%|████▉     | 584/1178 [05:07<04:15,  2.33it/s]\u001b[A\n",
      "Iteration:  50%|████▉     | 585/1178 [05:07<04:13,  2.34it/s]\u001b[A\n",
      "Iteration:  50%|████▉     | 586/1178 [05:07<04:12,  2.34it/s]\u001b[A\n",
      "Iteration:  50%|████▉     | 587/1178 [05:08<04:11,  2.35it/s]\u001b[A\n",
      "Iteration:  50%|████▉     | 588/1178 [05:10<07:52,  1.25it/s]\u001b[A\n",
      "Iteration:  50%|█████     | 589/1178 [05:10<06:45,  1.45it/s]\u001b[A\n",
      "Iteration:  50%|█████     | 590/1178 [05:10<06:06,  1.60it/s]\u001b[A\n",
      "Iteration:  50%|█████     | 591/1178 [05:11<05:30,  1.77it/s]\u001b[A\n",
      "Iteration:  50%|█████     | 592/1178 [05:11<05:05,  1.92it/s]\u001b[A\n",
      "Iteration:  50%|█████     | 593/1178 [05:12<04:48,  2.03it/s]\u001b[A\n",
      "Iteration:  50%|█████     | 594/1178 [05:12<04:37,  2.11it/s]\u001b[A\n",
      "Iteration:  51%|█████     | 595/1178 [05:13<04:29,  2.16it/s]\u001b[A\n",
      "Iteration:  51%|█████     | 596/1178 [05:13<04:23,  2.21it/s]\u001b[A\n",
      "Iteration:  51%|█████     | 597/1178 [05:13<04:18,  2.25it/s]\u001b[A\n",
      "Iteration:  51%|█████     | 598/1178 [05:14<04:14,  2.28it/s]\u001b[A\n",
      "Iteration:  51%|█████     | 599/1178 [05:14<04:12,  2.29it/s]\u001b[A\n",
      "Iteration:  51%|█████     | 600/1178 [05:15<04:18,  2.23it/s]\u001b[A\n",
      "Iteration:  51%|█████     | 601/1178 [05:15<04:15,  2.26it/s]\u001b[A\n",
      "Iteration:  51%|█████     | 602/1178 [05:16<04:10,  2.30it/s]\u001b[A\n",
      "Iteration:  51%|█████     | 603/1178 [05:16<04:08,  2.32it/s]\u001b[A\n",
      "Iteration:  51%|█████▏    | 604/1178 [05:16<04:06,  2.33it/s]\u001b[A\n",
      "Iteration:  51%|█████▏    | 605/1178 [05:17<04:05,  2.33it/s]\u001b[A\n",
      "Iteration:  51%|█████▏    | 606/1178 [05:17<04:03,  2.34it/s]\u001b[A\n",
      "Iteration:  52%|█████▏    | 607/1178 [05:18<04:01,  2.36it/s]\u001b[A\n",
      "Iteration:  52%|█████▏    | 608/1178 [05:18<04:00,  2.37it/s]\u001b[A\n",
      "Iteration:  52%|█████▏    | 609/1178 [05:19<04:00,  2.37it/s]\u001b[A\n",
      "Iteration:  52%|█████▏    | 610/1178 [05:19<04:07,  2.29it/s]\u001b[A\n",
      "Iteration:  52%|█████▏    | 611/1178 [05:19<04:04,  2.32it/s]\u001b[A\n",
      "Iteration:  52%|█████▏    | 612/1178 [05:20<04:02,  2.34it/s]\u001b[A\n",
      "Iteration:  52%|█████▏    | 613/1178 [05:20<04:00,  2.34it/s]\u001b[A\n",
      "Iteration:  52%|█████▏    | 614/1178 [05:21<04:00,  2.34it/s]\u001b[A\n",
      "Iteration:  52%|█████▏    | 615/1178 [05:21<03:59,  2.35it/s]\u001b[A\n",
      "Iteration:  52%|█████▏    | 616/1178 [05:22<03:59,  2.35it/s]\u001b[A\n",
      "Iteration:  52%|█████▏    | 617/1178 [05:22<03:57,  2.36it/s]\u001b[A\n",
      "Iteration:  52%|█████▏    | 618/1178 [05:22<03:56,  2.37it/s]\u001b[A\n",
      "Iteration:  53%|█████▎    | 619/1178 [05:23<03:55,  2.37it/s]\u001b[A\n",
      "Iteration:  53%|█████▎    | 620/1178 [05:23<04:04,  2.29it/s]\u001b[A\n",
      "Iteration:  53%|█████▎    | 621/1178 [05:24<04:00,  2.32it/s]\u001b[A\n",
      "Iteration:  53%|█████▎    | 622/1178 [05:25<07:30,  1.24it/s]\u001b[A\n",
      "Iteration:  53%|█████▎    | 623/1178 [05:26<06:24,  1.44it/s]\u001b[A\n",
      "Iteration:  53%|█████▎    | 624/1178 [05:26<05:38,  1.64it/s]\u001b[A\n",
      "Iteration:  53%|█████▎    | 625/1178 [05:27<05:06,  1.80it/s]\u001b[A\n",
      "Iteration:  53%|█████▎    | 626/1178 [05:27<04:44,  1.94it/s]\u001b[A\n",
      "Iteration:  53%|█████▎    | 627/1178 [05:27<04:28,  2.05it/s]\u001b[A\n",
      "Iteration:  53%|█████▎    | 628/1178 [05:28<04:17,  2.13it/s]\u001b[A\n",
      "Iteration:  53%|█████▎    | 629/1178 [05:28<04:09,  2.20it/s]\u001b[A\n",
      "Iteration:  53%|█████▎    | 630/1178 [05:29<04:12,  2.17it/s]\u001b[A\n",
      "Iteration:  54%|█████▎    | 631/1178 [05:29<04:05,  2.23it/s]\u001b[A\n",
      "Iteration:  54%|█████▎    | 632/1178 [05:30<04:00,  2.27it/s]\u001b[A\n",
      "Iteration:  54%|█████▎    | 633/1178 [05:30<03:57,  2.30it/s]\u001b[A\n",
      "Iteration:  54%|█████▍    | 634/1178 [05:30<03:54,  2.32it/s]\u001b[A\n",
      "Iteration:  54%|█████▍    | 635/1178 [05:31<03:54,  2.32it/s]\u001b[A\n",
      "Iteration:  54%|█████▍    | 636/1178 [05:31<03:51,  2.34it/s]\u001b[A\n",
      "Iteration:  54%|█████▍    | 637/1178 [05:32<03:51,  2.34it/s]\u001b[A\n",
      "Iteration:  54%|█████▍    | 638/1178 [05:32<03:49,  2.35it/s]\u001b[A\n",
      "Iteration:  54%|█████▍    | 639/1178 [05:33<03:48,  2.35it/s]\u001b[A\n",
      "Iteration:  54%|█████▍    | 640/1178 [05:33<03:56,  2.27it/s]\u001b[A\n",
      "Iteration:  54%|█████▍    | 641/1178 [05:34<03:53,  2.30it/s]\u001b[A\n",
      "Iteration:  54%|█████▍    | 642/1178 [05:34<03:49,  2.33it/s]\u001b[A\n",
      "Iteration:  55%|█████▍    | 643/1178 [05:34<03:47,  2.35it/s]\u001b[A\n",
      "Iteration:  55%|█████▍    | 644/1178 [05:35<03:46,  2.35it/s]\u001b[A\n",
      "Iteration:  55%|█████▍    | 645/1178 [05:35<03:46,  2.36it/s]\u001b[A\n",
      "Iteration:  55%|█████▍    | 646/1178 [05:36<03:45,  2.36it/s]\u001b[A\n",
      "Iteration:  55%|█████▍    | 647/1178 [05:36<03:44,  2.37it/s]\u001b[A\n",
      "Iteration:  55%|█████▌    | 648/1178 [05:36<03:43,  2.37it/s]\u001b[A\n",
      "Iteration:  55%|█████▌    | 649/1178 [05:37<03:42,  2.37it/s]\u001b[A\n",
      "Iteration:  55%|█████▌    | 650/1178 [05:37<03:50,  2.29it/s]\u001b[A\n",
      "Iteration:  55%|█████▌    | 651/1178 [05:38<03:47,  2.31it/s]\u001b[A\n",
      "Iteration:  55%|█████▌    | 652/1178 [05:38<03:45,  2.33it/s]\u001b[A\n",
      "Iteration:  55%|█████▌    | 653/1178 [05:39<03:44,  2.34it/s]\u001b[A\n",
      "Iteration:  56%|█████▌    | 654/1178 [05:39<03:42,  2.35it/s]\u001b[A\n",
      "Iteration:  56%|█████▌    | 655/1178 [05:39<03:41,  2.36it/s]\u001b[A\n",
      "Iteration:  56%|█████▌    | 656/1178 [05:41<06:56,  1.25it/s]\u001b[A\n",
      "Iteration:  56%|█████▌    | 657/1178 [05:42<05:57,  1.46it/s]\u001b[A\n",
      "Iteration:  56%|█████▌    | 658/1178 [05:42<05:15,  1.65it/s]\u001b[A\n",
      "Iteration:  56%|█████▌    | 659/1178 [05:42<04:45,  1.82it/s]\u001b[A\n",
      "Iteration:  56%|█████▌    | 660/1178 [05:43<04:32,  1.90it/s]\u001b[A\n",
      "Iteration:  56%|█████▌    | 661/1178 [05:43<04:16,  2.02it/s]\u001b[A\n",
      "Iteration:  56%|█████▌    | 662/1178 [05:44<04:05,  2.10it/s]\u001b[A\n",
      "Iteration:  56%|█████▋    | 663/1178 [05:44<03:56,  2.17it/s]\u001b[A\n",
      "Iteration:  56%|█████▋    | 664/1178 [05:45<03:50,  2.23it/s]\u001b[A\n",
      "Iteration:  56%|█████▋    | 665/1178 [05:45<03:46,  2.27it/s]\u001b[A\n",
      "Iteration:  57%|█████▋    | 666/1178 [05:45<03:42,  2.30it/s]\u001b[A\n",
      "Iteration:  57%|█████▋    | 667/1178 [05:46<03:40,  2.32it/s]\u001b[A\n",
      "Iteration:  57%|█████▋    | 668/1178 [05:46<03:38,  2.33it/s]\u001b[A\n",
      "Iteration:  57%|█████▋    | 669/1178 [05:47<03:38,  2.33it/s]\u001b[A\n",
      "Iteration:  57%|█████▋    | 670/1178 [05:47<03:44,  2.26it/s]\u001b[A\n",
      "Iteration:  57%|█████▋    | 671/1178 [05:48<03:41,  2.28it/s]\u001b[A\n",
      "Iteration:  57%|█████▋    | 672/1178 [05:48<03:38,  2.32it/s]\u001b[A\n",
      "Iteration:  57%|█████▋    | 673/1178 [05:48<03:37,  2.32it/s]\u001b[A\n",
      "Iteration:  57%|█████▋    | 674/1178 [05:49<03:35,  2.34it/s]\u001b[A\n",
      "Iteration:  57%|█████▋    | 675/1178 [05:49<03:34,  2.34it/s]\u001b[A\n",
      "Iteration:  57%|█████▋    | 676/1178 [05:50<03:33,  2.35it/s]\u001b[A\n",
      "Iteration:  57%|█████▋    | 677/1178 [05:50<03:32,  2.35it/s]\u001b[A\n",
      "Iteration:  58%|█████▊    | 678/1178 [05:51<03:31,  2.37it/s]\u001b[A\n",
      "Iteration:  58%|█████▊    | 679/1178 [05:51<03:30,  2.38it/s]\u001b[A\n",
      "Iteration:  58%|█████▊    | 680/1178 [05:51<03:37,  2.29it/s]\u001b[A\n",
      "Iteration:  58%|█████▊    | 681/1178 [05:52<03:34,  2.32it/s]\u001b[A\n",
      "Iteration:  58%|█████▊    | 682/1178 [05:52<03:32,  2.33it/s]\u001b[A\n",
      "Iteration:  58%|█████▊    | 683/1178 [05:53<03:28,  2.37it/s]\u001b[A\n",
      "Iteration:  58%|█████▊    | 684/1178 [05:53<03:27,  2.38it/s]\u001b[A\n",
      "Iteration:  58%|█████▊    | 685/1178 [05:54<03:27,  2.37it/s]\u001b[A\n",
      "Iteration:  58%|█████▊    | 686/1178 [05:54<03:27,  2.37it/s]\u001b[A\n",
      "Iteration:  58%|█████▊    | 687/1178 [05:54<03:27,  2.37it/s]\u001b[A\n",
      "Iteration:  58%|█████▊    | 688/1178 [05:55<03:26,  2.37it/s]\u001b[A\n",
      "Iteration:  58%|█████▊    | 689/1178 [05:55<03:26,  2.37it/s]\u001b[A\n",
      "Iteration:  59%|█████▊    | 690/1178 [05:57<06:39,  1.22it/s]\u001b[A\n",
      "Iteration:  59%|█████▊    | 691/1178 [05:57<05:40,  1.43it/s]\u001b[A\n",
      "Iteration:  59%|█████▊    | 692/1178 [05:58<04:59,  1.62it/s]\u001b[A\n",
      "Iteration:  59%|█████▉    | 693/1178 [05:58<04:30,  1.79it/s]\u001b[A\n",
      "Iteration:  59%|█████▉    | 694/1178 [05:59<04:10,  1.93it/s]\u001b[A\n",
      "Iteration:  59%|█████▉    | 695/1178 [05:59<03:56,  2.04it/s]\u001b[A\n",
      "Iteration:  59%|█████▉    | 696/1178 [05:59<03:46,  2.13it/s]\u001b[A\n",
      "Iteration:  59%|█████▉    | 697/1178 [06:00<03:38,  2.20it/s]\u001b[A\n",
      "Iteration:  59%|█████▉    | 698/1178 [06:00<03:33,  2.25it/s]\u001b[A\n",
      "Iteration:  59%|█████▉    | 699/1178 [06:01<03:29,  2.28it/s]\u001b[A\n",
      "Iteration:  59%|█████▉    | 700/1178 [06:01<03:34,  2.23it/s]\u001b[A\n",
      "Iteration:  60%|█████▉    | 701/1178 [06:02<03:29,  2.27it/s]\u001b[A\n",
      "Iteration:  60%|█████▉    | 702/1178 [06:02<03:26,  2.30it/s]\u001b[A\n",
      "Iteration:  60%|█████▉    | 703/1178 [06:02<03:24,  2.33it/s]\u001b[A\n",
      "Iteration:  60%|█████▉    | 704/1178 [06:03<03:22,  2.34it/s]\u001b[A\n",
      "Iteration:  60%|█████▉    | 705/1178 [06:03<03:21,  2.35it/s]\u001b[A\n",
      "Iteration:  60%|█████▉    | 706/1178 [06:04<03:20,  2.35it/s]\u001b[A\n",
      "Iteration:  60%|██████    | 707/1178 [06:04<03:19,  2.36it/s]\u001b[A\n",
      "Iteration:  60%|██████    | 708/1178 [06:05<03:18,  2.36it/s]\u001b[A\n",
      "Iteration:  60%|██████    | 709/1178 [06:05<03:18,  2.36it/s]\u001b[A\n",
      "Iteration:  60%|██████    | 710/1178 [06:05<03:25,  2.28it/s]\u001b[A\n",
      "Iteration:  60%|██████    | 711/1178 [06:06<03:23,  2.30it/s]\u001b[A\n",
      "Iteration:  60%|██████    | 712/1178 [06:06<03:20,  2.32it/s]\u001b[A\n",
      "Iteration:  61%|██████    | 713/1178 [06:07<03:19,  2.33it/s]\u001b[A\n",
      "Iteration:  61%|██████    | 714/1178 [06:07<03:17,  2.34it/s]\u001b[A\n",
      "Iteration:  61%|██████    | 715/1178 [06:08<03:16,  2.36it/s]\u001b[A\n",
      "Iteration:  61%|██████    | 716/1178 [06:08<03:15,  2.36it/s]\u001b[A\n",
      "Iteration:  61%|██████    | 717/1178 [06:08<03:14,  2.36it/s]\u001b[A\n",
      "Iteration:  61%|██████    | 718/1178 [06:09<03:14,  2.36it/s]\u001b[A\n",
      "Iteration:  61%|██████    | 719/1178 [06:09<03:13,  2.37it/s]\u001b[A\n",
      "Iteration:  61%|██████    | 720/1178 [06:10<03:20,  2.28it/s]\u001b[A\n",
      "Iteration:  61%|██████    | 721/1178 [06:10<03:18,  2.30it/s]\u001b[A\n",
      "Iteration:  61%|██████▏   | 722/1178 [06:11<03:16,  2.32it/s]\u001b[A\n",
      "Iteration:  61%|██████▏   | 723/1178 [06:11<03:14,  2.34it/s]\u001b[A\n",
      "Iteration:  61%|██████▏   | 724/1178 [06:13<06:07,  1.24it/s]\u001b[A\n",
      "Iteration:  62%|██████▏   | 725/1178 [06:13<05:13,  1.44it/s]\u001b[A\n",
      "Iteration:  62%|██████▏   | 726/1178 [06:14<04:36,  1.64it/s]\u001b[A\n",
      "Iteration:  62%|██████▏   | 727/1178 [06:14<04:09,  1.81it/s]\u001b[A\n",
      "Iteration:  62%|██████▏   | 728/1178 [06:14<03:50,  1.95it/s]\u001b[A\n",
      "Iteration:  62%|██████▏   | 729/1178 [06:15<03:37,  2.06it/s]\u001b[A\n",
      "Iteration:  62%|██████▏   | 730/1178 [06:15<03:35,  2.08it/s]\u001b[A\n",
      "Iteration:  62%|██████▏   | 731/1178 [06:16<03:26,  2.16it/s]\u001b[A\n",
      "Iteration:  62%|██████▏   | 732/1178 [06:16<03:20,  2.23it/s]\u001b[A\n",
      "Iteration:  62%|██████▏   | 733/1178 [06:17<03:15,  2.28it/s]\u001b[A\n",
      "Iteration:  62%|██████▏   | 734/1178 [06:17<03:11,  2.31it/s]\u001b[A\n",
      "Iteration:  62%|██████▏   | 735/1178 [06:17<03:08,  2.35it/s]\u001b[A\n",
      "Iteration:  62%|██████▏   | 736/1178 [06:18<03:07,  2.36it/s]\u001b[A\n",
      "Iteration:  63%|██████▎   | 737/1178 [06:18<03:06,  2.37it/s]\u001b[A\n",
      "Iteration:  63%|██████▎   | 738/1178 [06:19<03:05,  2.38it/s]\u001b[A\n",
      "Iteration:  63%|██████▎   | 739/1178 [06:19<03:04,  2.38it/s]\u001b[A\n",
      "Iteration:  63%|██████▎   | 740/1178 [06:20<03:10,  2.30it/s]\u001b[A\n",
      "Iteration:  63%|██████▎   | 741/1178 [06:20<03:07,  2.33it/s]\u001b[A\n",
      "Iteration:  63%|██████▎   | 742/1178 [06:20<03:05,  2.35it/s]\u001b[A\n",
      "Iteration:  63%|██████▎   | 743/1178 [06:21<03:03,  2.37it/s]\u001b[A\n",
      "Iteration:  63%|██████▎   | 744/1178 [06:21<03:02,  2.38it/s]\u001b[A\n",
      "Iteration:  63%|██████▎   | 745/1178 [06:22<03:01,  2.38it/s]\u001b[A\n",
      "Iteration:  63%|██████▎   | 746/1178 [06:22<03:00,  2.39it/s]\u001b[A\n",
      "Iteration:  63%|██████▎   | 747/1178 [06:22<03:00,  2.39it/s]\u001b[A\n",
      "Iteration:  63%|██████▎   | 748/1178 [06:23<02:59,  2.40it/s]\u001b[A\n",
      "Iteration:  64%|██████▎   | 749/1178 [06:23<02:58,  2.40it/s]\u001b[A\n",
      "Iteration:  64%|██████▎   | 750/1178 [06:24<03:04,  2.32it/s]\u001b[A\n",
      "Iteration:  64%|██████▍   | 751/1178 [06:24<03:02,  2.34it/s]\u001b[A\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Iteration:  64%|██████▍   | 752/1178 [06:25<03:00,  2.36it/s]\u001b[A\n",
      "Iteration:  64%|██████▍   | 753/1178 [06:25<02:58,  2.38it/s]\u001b[A\n",
      "Iteration:  64%|██████▍   | 754/1178 [06:25<02:57,  2.39it/s]\u001b[A\n",
      "Iteration:  64%|██████▍   | 755/1178 [06:26<02:57,  2.39it/s]\u001b[A\n",
      "Iteration:  64%|██████▍   | 756/1178 [06:26<02:56,  2.39it/s]\u001b[A\n",
      "Iteration:  64%|██████▍   | 757/1178 [06:27<02:56,  2.38it/s]\u001b[A\n",
      "Iteration:  64%|██████▍   | 758/1178 [06:28<05:34,  1.26it/s]\u001b[A\n",
      "Iteration:  64%|██████▍   | 759/1178 [06:29<04:45,  1.47it/s]\u001b[A\n",
      "Iteration:  65%|██████▍   | 760/1178 [06:29<04:17,  1.62it/s]\u001b[A\n",
      "Iteration:  65%|██████▍   | 761/1178 [06:30<03:51,  1.80it/s]\u001b[A\n",
      "Iteration:  65%|██████▍   | 762/1178 [06:30<03:33,  1.95it/s]\u001b[A\n",
      "Iteration:  65%|██████▍   | 763/1178 [06:30<03:20,  2.07it/s]\u001b[A\n",
      "Iteration:  65%|██████▍   | 764/1178 [06:31<03:12,  2.15it/s]\u001b[A\n",
      "Iteration:  65%|██████▍   | 765/1178 [06:31<03:06,  2.22it/s]\u001b[A\n",
      "Iteration:  65%|██████▌   | 766/1178 [06:32<02:59,  2.29it/s]\u001b[A\n",
      "Iteration:  65%|██████▌   | 767/1178 [06:32<02:57,  2.32it/s]\u001b[A\n",
      "Iteration:  65%|██████▌   | 768/1178 [06:33<02:54,  2.34it/s]\u001b[A\n",
      "Iteration:  65%|██████▌   | 769/1178 [06:33<02:53,  2.36it/s]\u001b[A\n",
      "Iteration:  65%|██████▌   | 770/1178 [06:33<02:57,  2.29it/s]\u001b[A\n",
      "Iteration:  65%|██████▌   | 771/1178 [06:34<02:55,  2.31it/s]\u001b[A\n",
      "Iteration:  66%|██████▌   | 772/1178 [06:34<02:53,  2.34it/s]\u001b[A\n",
      "Iteration:  66%|██████▌   | 773/1178 [06:35<02:51,  2.36it/s]\u001b[A\n",
      "Iteration:  66%|██████▌   | 774/1178 [06:35<02:49,  2.38it/s]\u001b[A\n",
      "Iteration:  66%|██████▌   | 775/1178 [06:35<02:48,  2.38it/s]\u001b[A\n",
      "Iteration:  66%|██████▌   | 776/1178 [06:36<02:48,  2.39it/s]\u001b[A\n",
      "Iteration:  66%|██████▌   | 777/1178 [06:36<02:47,  2.39it/s]\u001b[A\n",
      "Iteration:  66%|██████▌   | 778/1178 [06:37<02:47,  2.39it/s]\u001b[A\n",
      "Iteration:  66%|██████▌   | 779/1178 [06:37<02:46,  2.39it/s]\u001b[A\n",
      "Iteration:  66%|██████▌   | 780/1178 [06:38<02:52,  2.31it/s]\u001b[A\n",
      "Iteration:  66%|██████▋   | 781/1178 [06:38<02:50,  2.33it/s]\u001b[A\n",
      "Iteration:  66%|██████▋   | 782/1178 [06:38<02:48,  2.35it/s]\u001b[A\n",
      "Iteration:  66%|██████▋   | 783/1178 [06:39<02:47,  2.36it/s]\u001b[A\n",
      "Iteration:  67%|██████▋   | 784/1178 [06:39<02:46,  2.37it/s]\u001b[A\n",
      "Iteration:  67%|██████▋   | 785/1178 [06:40<02:45,  2.37it/s]\u001b[A\n",
      "Iteration:  67%|██████▋   | 786/1178 [06:40<02:44,  2.38it/s]\u001b[A\n",
      "Iteration:  67%|██████▋   | 787/1178 [06:41<02:44,  2.38it/s]\u001b[A\n",
      "Iteration:  67%|██████▋   | 788/1178 [06:41<02:43,  2.39it/s]\u001b[A\n",
      "Iteration:  67%|██████▋   | 789/1178 [06:41<02:42,  2.39it/s]\u001b[A\n",
      "Iteration:  67%|██████▋   | 790/1178 [06:42<02:47,  2.31it/s]\u001b[A\n",
      "Iteration:  67%|██████▋   | 791/1178 [06:42<02:46,  2.33it/s]\u001b[A\n",
      "Iteration:  67%|██████▋   | 792/1178 [06:44<05:10,  1.24it/s]\u001b[A\n",
      "Iteration:  67%|██████▋   | 793/1178 [06:44<04:24,  1.45it/s]\u001b[A\n",
      "Iteration:  67%|██████▋   | 794/1178 [06:45<03:52,  1.65it/s]\u001b[A\n",
      "Iteration:  67%|██████▋   | 795/1178 [06:45<03:30,  1.82it/s]\u001b[A\n",
      "Iteration:  68%|██████▊   | 796/1178 [06:46<03:14,  1.96it/s]\u001b[A\n",
      "Iteration:  68%|██████▊   | 797/1178 [06:46<03:03,  2.07it/s]\u001b[A\n",
      "Iteration:  68%|██████▊   | 798/1178 [06:46<02:56,  2.16it/s]\u001b[A\n",
      "Iteration:  68%|██████▊   | 799/1178 [06:47<02:50,  2.22it/s]\u001b[A\n",
      "Iteration:  68%|██████▊   | 800/1178 [06:47<02:51,  2.20it/s]\u001b[A\n",
      "Iteration:  68%|██████▊   | 801/1178 [06:48<02:47,  2.25it/s]\u001b[A\n",
      "Iteration:  68%|██████▊   | 802/1178 [06:48<02:43,  2.29it/s]\u001b[A\n",
      "Iteration:  68%|██████▊   | 803/1178 [06:49<02:41,  2.32it/s]\u001b[A\n",
      "Iteration:  68%|██████▊   | 804/1178 [06:49<02:39,  2.34it/s]\u001b[A\n",
      "Iteration:  68%|██████▊   | 805/1178 [06:49<02:38,  2.35it/s]\u001b[A\n",
      "Iteration:  68%|██████▊   | 806/1178 [06:50<02:37,  2.37it/s]\u001b[A\n",
      "Iteration:  69%|██████▊   | 807/1178 [06:50<02:36,  2.37it/s]\u001b[A\n",
      "Iteration:  69%|██████▊   | 808/1178 [06:51<02:35,  2.37it/s]\u001b[A\n",
      "Iteration:  69%|██████▊   | 809/1178 [06:51<02:34,  2.38it/s]\u001b[A\n",
      "Iteration:  69%|██████▉   | 810/1178 [06:52<02:39,  2.31it/s]\u001b[A\n",
      "Iteration:  69%|██████▉   | 811/1178 [06:52<02:37,  2.33it/s]\u001b[A\n",
      "Iteration:  69%|██████▉   | 812/1178 [06:52<02:34,  2.36it/s]\u001b[A\n",
      "Iteration:  69%|██████▉   | 813/1178 [06:53<02:33,  2.37it/s]\u001b[A\n",
      "Iteration:  69%|██████▉   | 814/1178 [06:53<02:32,  2.38it/s]\u001b[A\n",
      "Iteration:  69%|██████▉   | 815/1178 [06:54<02:31,  2.39it/s]\u001b[A\n",
      "Iteration:  69%|██████▉   | 816/1178 [06:54<02:30,  2.40it/s]\u001b[A\n",
      "Iteration:  69%|██████▉   | 817/1178 [06:54<02:30,  2.40it/s]\u001b[A\n",
      "Iteration:  69%|██████▉   | 818/1178 [06:55<02:29,  2.41it/s]\u001b[A\n",
      "Iteration:  70%|██████▉   | 819/1178 [06:55<02:29,  2.39it/s]\u001b[A\n",
      "Iteration:  70%|██████▉   | 820/1178 [06:56<02:35,  2.31it/s]\u001b[A\n",
      "Iteration:  70%|██████▉   | 821/1178 [06:56<02:33,  2.33it/s]\u001b[A\n",
      "Iteration:  70%|██████▉   | 822/1178 [06:57<02:31,  2.35it/s]\u001b[A\n",
      "Iteration:  70%|██████▉   | 823/1178 [06:57<02:30,  2.36it/s]\u001b[A\n",
      "Iteration:  70%|██████▉   | 824/1178 [06:57<02:28,  2.38it/s]\u001b[A\n",
      "Iteration:  70%|███████   | 825/1178 [06:58<02:27,  2.39it/s]\u001b[A\n",
      "Iteration:  70%|███████   | 826/1178 [07:00<04:41,  1.25it/s]\u001b[A\n",
      "Iteration:  70%|███████   | 827/1178 [07:00<04:00,  1.46it/s]\u001b[A\n",
      "Iteration:  70%|███████   | 828/1178 [07:00<03:31,  1.65it/s]\u001b[A\n",
      "Iteration:  70%|███████   | 829/1178 [07:01<03:11,  1.82it/s]\u001b[A\n",
      "Iteration:  70%|███████   | 830/1178 [07:01<03:02,  1.91it/s]\u001b[A\n",
      "Iteration:  71%|███████   | 831/1178 [07:02<02:50,  2.03it/s]\u001b[A\n",
      "Iteration:  71%|███████   | 832/1178 [07:02<02:42,  2.12it/s]\u001b[A\n",
      "Iteration:  71%|███████   | 833/1178 [07:03<02:37,  2.19it/s]\u001b[A\n",
      "Iteration:  71%|███████   | 834/1178 [07:03<02:33,  2.25it/s]\u001b[A\n",
      "Iteration:  71%|███████   | 835/1178 [07:03<02:29,  2.29it/s]\u001b[A\n",
      "Iteration:  71%|███████   | 836/1178 [07:04<02:27,  2.32it/s]\u001b[A\n",
      "Iteration:  71%|███████   | 837/1178 [07:04<02:26,  2.33it/s]\u001b[A\n",
      "Iteration:  71%|███████   | 838/1178 [07:05<02:24,  2.35it/s]\u001b[A\n",
      "Iteration:  71%|███████   | 839/1178 [07:05<02:23,  2.35it/s]\u001b[A\n",
      "Iteration:  71%|███████▏  | 840/1178 [07:06<02:27,  2.29it/s]\u001b[A\n",
      "Iteration:  71%|███████▏  | 841/1178 [07:06<02:25,  2.31it/s]\u001b[A\n",
      "Iteration:  71%|███████▏  | 842/1178 [07:06<02:24,  2.33it/s]\u001b[A\n",
      "Iteration:  72%|███████▏  | 843/1178 [07:07<02:22,  2.34it/s]\u001b[A\n",
      "Iteration:  72%|███████▏  | 844/1178 [07:07<02:21,  2.36it/s]\u001b[A\n",
      "Iteration:  72%|███████▏  | 845/1178 [07:08<02:21,  2.36it/s]\u001b[A\n",
      "Iteration:  72%|███████▏  | 846/1178 [07:08<02:20,  2.37it/s]\u001b[A\n",
      "Iteration:  72%|███████▏  | 847/1178 [07:08<02:19,  2.37it/s]\u001b[A\n",
      "Iteration:  72%|███████▏  | 848/1178 [07:09<02:18,  2.38it/s]\u001b[A\n",
      "Iteration:  72%|███████▏  | 849/1178 [07:09<02:18,  2.38it/s]\u001b[A\n",
      "Iteration:  72%|███████▏  | 850/1178 [07:10<02:22,  2.30it/s]\u001b[A\n",
      "Iteration:  72%|███████▏  | 851/1178 [07:10<02:21,  2.31it/s]\u001b[A\n",
      "Iteration:  72%|███████▏  | 852/1178 [07:11<02:19,  2.33it/s]\u001b[A\n",
      "Iteration:  72%|███████▏  | 853/1178 [07:11<02:18,  2.34it/s]\u001b[A\n",
      "Iteration:  72%|███████▏  | 854/1178 [07:11<02:17,  2.36it/s]\u001b[A\n",
      "Iteration:  73%|███████▎  | 855/1178 [07:12<02:16,  2.36it/s]\u001b[A\n",
      "Iteration:  73%|███████▎  | 856/1178 [07:12<02:16,  2.36it/s]\u001b[A\n",
      "Iteration:  73%|███████▎  | 857/1178 [07:13<02:15,  2.37it/s]\u001b[A\n",
      "Iteration:  73%|███████▎  | 858/1178 [07:13<02:15,  2.37it/s]\u001b[A\n",
      "Iteration:  73%|███████▎  | 859/1178 [07:14<02:14,  2.37it/s]\u001b[A\n",
      "Iteration:  73%|███████▎  | 860/1178 [07:15<04:17,  1.24it/s]\u001b[A\n",
      "Iteration:  73%|███████▎  | 861/1178 [07:16<03:40,  1.44it/s]\u001b[A\n",
      "Iteration:  73%|███████▎  | 862/1178 [07:16<03:13,  1.64it/s]\u001b[A\n",
      "Iteration:  73%|███████▎  | 863/1178 [07:17<02:53,  1.81it/s]\u001b[A\n",
      "Iteration:  73%|███████▎  | 864/1178 [07:17<02:40,  1.96it/s]\u001b[A\n",
      "Iteration:  73%|███████▎  | 865/1178 [07:17<02:31,  2.07it/s]\u001b[A\n",
      "Iteration:  74%|███████▎  | 866/1178 [07:18<02:24,  2.16it/s]\u001b[A\n",
      "Iteration:  74%|███████▎  | 867/1178 [07:18<02:19,  2.22it/s]\u001b[A\n",
      "Iteration:  74%|███████▎  | 868/1178 [07:19<02:16,  2.27it/s]\u001b[A\n",
      "Iteration:  74%|███████▍  | 869/1178 [07:19<02:13,  2.31it/s]\u001b[A\n",
      "Iteration:  74%|███████▍  | 870/1178 [07:20<02:16,  2.26it/s]\u001b[A\n",
      "Iteration:  74%|███████▍  | 871/1178 [07:20<02:13,  2.30it/s]\u001b[A\n",
      "Iteration:  74%|███████▍  | 872/1178 [07:20<02:11,  2.33it/s]\u001b[A\n",
      "Iteration:  74%|███████▍  | 873/1178 [07:21<02:10,  2.34it/s]\u001b[A\n",
      "Iteration:  74%|███████▍  | 874/1178 [07:21<02:08,  2.36it/s]\u001b[A\n",
      "Iteration:  74%|███████▍  | 875/1178 [07:22<02:08,  2.37it/s]\u001b[A\n",
      "Iteration:  74%|███████▍  | 876/1178 [07:22<02:07,  2.38it/s]\u001b[A\n",
      "Iteration:  74%|███████▍  | 877/1178 [07:22<02:06,  2.38it/s]\u001b[A\n",
      "Iteration:  75%|███████▍  | 878/1178 [07:23<02:05,  2.39it/s]\u001b[A\n",
      "Iteration:  75%|███████▍  | 879/1178 [07:23<02:04,  2.39it/s]\u001b[A\n",
      "Iteration:  75%|███████▍  | 880/1178 [07:24<02:09,  2.31it/s]\u001b[A\n",
      "Iteration:  75%|███████▍  | 881/1178 [07:24<02:07,  2.33it/s]\u001b[A\n",
      "Iteration:  75%|███████▍  | 882/1178 [07:25<02:05,  2.35it/s]\u001b[A\n",
      "Iteration:  75%|███████▍  | 883/1178 [07:25<02:04,  2.37it/s]\u001b[A\n",
      "Iteration:  75%|███████▌  | 884/1178 [07:25<02:03,  2.38it/s]\u001b[A\n",
      "Iteration:  75%|███████▌  | 885/1178 [07:26<02:03,  2.37it/s]\u001b[A\n",
      "Iteration:  75%|███████▌  | 886/1178 [07:26<02:02,  2.37it/s]\u001b[A\n",
      "Iteration:  75%|███████▌  | 887/1178 [07:27<02:02,  2.38it/s]\u001b[A\n",
      "Iteration:  75%|███████▌  | 888/1178 [07:27<02:01,  2.38it/s]\u001b[A\n",
      "Iteration:  75%|███████▌  | 889/1178 [07:28<02:01,  2.38it/s]\u001b[A\n",
      "Iteration:  76%|███████▌  | 890/1178 [07:28<02:05,  2.30it/s]\u001b[A\n",
      "Iteration:  76%|███████▌  | 891/1178 [07:28<02:03,  2.32it/s]\u001b[A\n",
      "Iteration:  76%|███████▌  | 892/1178 [07:29<02:02,  2.34it/s]\u001b[A\n",
      "Iteration:  76%|███████▌  | 893/1178 [07:29<02:01,  2.35it/s]\u001b[A\n",
      "Iteration:  76%|███████▌  | 894/1178 [07:31<03:47,  1.25it/s]\u001b[A\n",
      "Iteration:  76%|███████▌  | 895/1178 [07:31<03:13,  1.46it/s]\u001b[A\n",
      "Iteration:  76%|███████▌  | 896/1178 [07:32<02:50,  1.65it/s]\u001b[A\n",
      "Iteration:  76%|███████▌  | 897/1178 [07:32<02:34,  1.82it/s]\u001b[A\n",
      "Iteration:  76%|███████▌  | 898/1178 [07:33<02:22,  1.96it/s]\u001b[A\n",
      "Iteration:  76%|███████▋  | 899/1178 [07:33<02:14,  2.08it/s]\u001b[A\n",
      "Iteration:  76%|███████▋  | 900/1178 [07:33<02:12,  2.09it/s]\u001b[A\n",
      "Iteration:  76%|███████▋  | 901/1178 [07:34<02:07,  2.17it/s]\u001b[A\n",
      "Iteration:  77%|███████▋  | 902/1178 [07:34<02:03,  2.24it/s]\u001b[A\n",
      "Iteration:  77%|███████▋  | 903/1178 [07:35<02:00,  2.28it/s]\u001b[A\n",
      "Iteration:  77%|███████▋  | 904/1178 [07:35<01:58,  2.31it/s]\u001b[A\n",
      "Iteration:  77%|███████▋  | 905/1178 [07:36<01:57,  2.32it/s]\u001b[A\n",
      "Iteration:  77%|███████▋  | 906/1178 [07:36<01:56,  2.34it/s]\u001b[A\n",
      "Iteration:  77%|███████▋  | 907/1178 [07:36<01:55,  2.35it/s]\u001b[A\n",
      "Iteration:  77%|███████▋  | 908/1178 [07:37<01:54,  2.37it/s]\u001b[A\n",
      "Iteration:  77%|███████▋  | 909/1178 [07:37<01:53,  2.37it/s]\u001b[A\n",
      "Iteration:  77%|███████▋  | 910/1178 [07:38<01:57,  2.28it/s]\u001b[A\n",
      "Iteration:  77%|███████▋  | 911/1178 [07:38<01:55,  2.30it/s]\u001b[A\n",
      "Iteration:  77%|███████▋  | 912/1178 [07:39<01:54,  2.33it/s]\u001b[A\n",
      "Iteration:  78%|███████▊  | 913/1178 [07:39<01:53,  2.34it/s]\u001b[A\n",
      "Iteration:  78%|███████▊  | 914/1178 [07:39<01:52,  2.36it/s]\u001b[A\n",
      "Iteration:  78%|███████▊  | 915/1178 [07:40<01:51,  2.37it/s]\u001b[A\n",
      "Iteration:  78%|███████▊  | 916/1178 [07:40<01:50,  2.37it/s]\u001b[A\n",
      "Iteration:  78%|███████▊  | 917/1178 [07:41<01:50,  2.37it/s]\u001b[A\n",
      "Iteration:  78%|███████▊  | 918/1178 [07:41<01:49,  2.37it/s]\u001b[A\n",
      "Iteration:  78%|███████▊  | 919/1178 [07:42<01:48,  2.38it/s]\u001b[A\n",
      "Iteration:  78%|███████▊  | 920/1178 [07:42<01:50,  2.32it/s]\u001b[A\n",
      "Iteration:  78%|███████▊  | 921/1178 [07:42<01:49,  2.34it/s]\u001b[A\n",
      "Iteration:  78%|███████▊  | 922/1178 [07:43<01:48,  2.36it/s]\u001b[A\n",
      "Iteration:  78%|███████▊  | 923/1178 [07:43<01:47,  2.36it/s]\u001b[A\n",
      "Iteration:  78%|███████▊  | 924/1178 [07:44<01:47,  2.37it/s]\u001b[A\n",
      "Iteration:  79%|███████▊  | 925/1178 [07:44<01:46,  2.37it/s]\u001b[A\n",
      "Iteration:  79%|███████▊  | 926/1178 [07:44<01:46,  2.37it/s]\u001b[A\n",
      "Iteration:  79%|███████▊  | 927/1178 [07:45<01:45,  2.38it/s]\u001b[A\n",
      "Iteration:  79%|███████▉  | 928/1178 [07:47<03:20,  1.25it/s]\u001b[A\n",
      "Iteration:  79%|███████▉  | 929/1178 [07:47<02:50,  1.46it/s]\u001b[A\n",
      "Iteration:  79%|███████▉  | 930/1178 [07:47<02:33,  1.61it/s]\u001b[A\n",
      "Iteration:  79%|███████▉  | 931/1178 [07:48<02:18,  1.78it/s]\u001b[A\n",
      "Iteration:  79%|███████▉  | 932/1178 [07:48<02:07,  1.93it/s]\u001b[A\n",
      "Iteration:  79%|███████▉  | 933/1178 [07:49<02:00,  2.04it/s]\u001b[A\n",
      "Iteration:  79%|███████▉  | 934/1178 [07:49<01:54,  2.13it/s]\u001b[A\n",
      "Iteration:  79%|███████▉  | 935/1178 [07:50<01:50,  2.19it/s]\u001b[A\n",
      "Iteration:  79%|███████▉  | 936/1178 [07:50<01:47,  2.25it/s]\u001b[A\n",
      "Iteration:  80%|███████▉  | 937/1178 [07:50<01:45,  2.29it/s]\u001b[A\n",
      "Iteration:  80%|███████▉  | 938/1178 [07:51<01:43,  2.32it/s]\u001b[A\n",
      "Iteration:  80%|███████▉  | 939/1178 [07:51<01:41,  2.34it/s]\u001b[A\n",
      "Iteration:  80%|███████▉  | 940/1178 [07:52<01:44,  2.28it/s]\u001b[A\n",
      "Iteration:  80%|███████▉  | 941/1178 [07:52<01:42,  2.30it/s]\u001b[A\n",
      "Iteration:  80%|███████▉  | 942/1178 [07:53<01:41,  2.33it/s]\u001b[A\n",
      "Iteration:  80%|████████  | 943/1178 [07:53<01:39,  2.36it/s]\u001b[A\n",
      "Iteration:  80%|████████  | 944/1178 [07:53<01:39,  2.36it/s]\u001b[A\n",
      "Iteration:  80%|████████  | 945/1178 [07:54<01:38,  2.37it/s]\u001b[A\n",
      "Iteration:  80%|████████  | 946/1178 [07:54<01:37,  2.38it/s]\u001b[A\n",
      "Iteration:  80%|████████  | 947/1178 [07:55<01:37,  2.38it/s]\u001b[A\n",
      "Iteration:  80%|████████  | 948/1178 [07:55<01:36,  2.38it/s]\u001b[A\n",
      "Iteration:  81%|████████  | 949/1178 [07:56<01:36,  2.38it/s]\u001b[A\n",
      "Iteration:  81%|████████  | 950/1178 [07:56<01:39,  2.30it/s]\u001b[A\n",
      "Iteration:  81%|████████  | 951/1178 [07:56<01:37,  2.32it/s]\u001b[A\n",
      "Iteration:  81%|████████  | 952/1178 [07:57<01:36,  2.34it/s]\u001b[A\n",
      "Iteration:  81%|████████  | 953/1178 [07:57<01:35,  2.36it/s]\u001b[A\n",
      "Iteration:  81%|████████  | 954/1178 [07:58<01:34,  2.37it/s]\u001b[A\n",
      "Iteration:  81%|████████  | 955/1178 [07:58<01:33,  2.39it/s]\u001b[A\n",
      "Iteration:  81%|████████  | 956/1178 [07:58<01:32,  2.39it/s]\u001b[A\n",
      "Iteration:  81%|████████  | 957/1178 [07:59<01:32,  2.39it/s]\u001b[A\n",
      "Iteration:  81%|████████▏ | 958/1178 [07:59<01:31,  2.40it/s]\u001b[A\n",
      "Iteration:  81%|████████▏ | 959/1178 [08:00<01:31,  2.39it/s]\u001b[A\n",
      "Iteration:  81%|████████▏ | 960/1178 [08:00<01:34,  2.31it/s]\u001b[A\n",
      "Iteration:  82%|████████▏ | 961/1178 [08:01<01:32,  2.33it/s]\u001b[A\n",
      "Iteration:  82%|████████▏ | 962/1178 [08:02<02:53,  1.24it/s]\u001b[A\n",
      "Iteration:  82%|████████▏ | 963/1178 [08:03<02:28,  1.45it/s]\u001b[A\n",
      "Iteration:  82%|████████▏ | 964/1178 [08:03<02:10,  1.64it/s]\u001b[A\n",
      "Iteration:  82%|████████▏ | 965/1178 [08:04<01:57,  1.81it/s]\u001b[A\n",
      "Iteration:  82%|████████▏ | 966/1178 [08:04<01:48,  1.95it/s]\u001b[A\n",
      "Iteration:  82%|████████▏ | 967/1178 [08:04<01:42,  2.06it/s]\u001b[A\n",
      "Iteration:  82%|████████▏ | 968/1178 [08:05<01:37,  2.15it/s]\u001b[A\n",
      "Iteration:  82%|████████▏ | 969/1178 [08:05<01:34,  2.21it/s]\u001b[A\n",
      "Iteration:  82%|████████▏ | 970/1178 [08:06<01:35,  2.19it/s]\u001b[A\n",
      "Iteration:  82%|████████▏ | 971/1178 [08:06<01:32,  2.24it/s]\u001b[A\n",
      "Iteration:  83%|████████▎ | 972/1178 [08:07<01:30,  2.28it/s]\u001b[A\n",
      "Iteration:  83%|████████▎ | 973/1178 [08:07<01:28,  2.31it/s]\u001b[A\n",
      "Iteration:  83%|████████▎ | 974/1178 [08:07<01:27,  2.33it/s]\u001b[A\n",
      "Iteration:  83%|████████▎ | 975/1178 [08:08<01:26,  2.35it/s]\u001b[A\n",
      "Iteration:  83%|████████▎ | 976/1178 [08:08<01:25,  2.36it/s]\u001b[A\n",
      "Iteration:  83%|████████▎ | 977/1178 [08:09<01:24,  2.37it/s]\u001b[A\n",
      "Iteration:  83%|████████▎ | 978/1178 [08:09<01:24,  2.38it/s]\u001b[A\n",
      "Iteration:  83%|████████▎ | 979/1178 [08:09<01:23,  2.38it/s]\u001b[A\n",
      "Iteration:  83%|████████▎ | 980/1178 [08:10<01:26,  2.30it/s]\u001b[A\n",
      "Iteration:  83%|████████▎ | 981/1178 [08:10<01:24,  2.32it/s]\u001b[A\n",
      "Iteration:  83%|████████▎ | 982/1178 [08:11<01:23,  2.35it/s]\u001b[A\n",
      "Iteration:  83%|████████▎ | 983/1178 [08:11<01:22,  2.35it/s]\u001b[A\n",
      "Iteration:  84%|████████▎ | 984/1178 [08:12<01:22,  2.36it/s]\u001b[A\n",
      "Iteration:  84%|████████▎ | 985/1178 [08:12<01:21,  2.37it/s]\u001b[A\n",
      "Iteration:  84%|████████▎ | 986/1178 [08:12<01:20,  2.37it/s]\u001b[A\n",
      "Iteration:  84%|████████▍ | 987/1178 [08:13<01:20,  2.38it/s]\u001b[A\n",
      "Iteration:  84%|████████▍ | 988/1178 [08:13<01:19,  2.38it/s]\u001b[A\n",
      "Iteration:  84%|████████▍ | 989/1178 [08:14<01:19,  2.38it/s]\u001b[A\n",
      "Iteration:  84%|████████▍ | 990/1178 [08:14<01:21,  2.30it/s]\u001b[A\n",
      "Iteration:  84%|████████▍ | 991/1178 [08:15<01:20,  2.33it/s]\u001b[A\n",
      "Iteration:  84%|████████▍ | 992/1178 [08:15<01:19,  2.35it/s]\u001b[A\n",
      "Iteration:  84%|████████▍ | 993/1178 [08:15<01:18,  2.36it/s]\u001b[A\n",
      "Iteration:  84%|████████▍ | 994/1178 [08:16<01:17,  2.36it/s]\u001b[A\n",
      "Iteration:  84%|████████▍ | 995/1178 [08:16<01:17,  2.37it/s]\u001b[A\n",
      "Iteration:  85%|████████▍ | 996/1178 [08:18<02:25,  1.25it/s]\u001b[A\n",
      "Iteration:  85%|████████▍ | 997/1178 [08:18<02:03,  1.46it/s]\u001b[A\n",
      "Iteration:  85%|████████▍ | 998/1178 [08:19<01:48,  1.66it/s]\u001b[A\n",
      "Iteration:  85%|████████▍ | 999/1178 [08:19<01:37,  1.83it/s]\u001b[A\n",
      "Iteration:  85%|████████▍ | 1000/1178 [08:20<01:33,  1.91it/s]\u001b[A\n",
      "Iteration:  85%|████████▍ | 1001/1178 [08:20<01:27,  2.03it/s]\u001b[A\n",
      "Iteration:  85%|████████▌ | 1002/1178 [08:21<01:22,  2.12it/s]\u001b[A\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Iteration:  85%|████████▌ | 1003/1178 [08:21<01:19,  2.20it/s]\u001b[A\n",
      "Iteration:  85%|████████▌ | 1004/1178 [08:21<01:17,  2.25it/s]\u001b[A\n",
      "Iteration:  85%|████████▌ | 1005/1178 [08:22<01:15,  2.29it/s]\u001b[A\n",
      "Iteration:  85%|████████▌ | 1006/1178 [08:22<01:14,  2.32it/s]\u001b[A\n",
      "Iteration:  85%|████████▌ | 1007/1178 [08:23<01:13,  2.34it/s]\u001b[A\n",
      "Iteration:  86%|████████▌ | 1008/1178 [08:23<01:12,  2.36it/s]\u001b[A\n",
      "Iteration:  86%|████████▌ | 1009/1178 [08:23<01:11,  2.37it/s]\u001b[A\n",
      "Iteration:  86%|████████▌ | 1010/1178 [08:24<01:13,  2.30it/s]\u001b[A\n",
      "Iteration:  86%|████████▌ | 1011/1178 [08:24<01:12,  2.31it/s]\u001b[A\n",
      "Iteration:  86%|████████▌ | 1012/1178 [08:25<01:11,  2.33it/s]\u001b[A\n",
      "Iteration:  86%|████████▌ | 1013/1178 [08:25<01:10,  2.36it/s]\u001b[A\n",
      "Iteration:  86%|████████▌ | 1014/1178 [08:26<01:09,  2.37it/s]\u001b[A\n",
      "Iteration:  86%|████████▌ | 1015/1178 [08:26<01:08,  2.39it/s]\u001b[A\n",
      "Iteration:  86%|████████▌ | 1016/1178 [08:26<01:07,  2.40it/s]\u001b[A\n",
      "Iteration:  86%|████████▋ | 1017/1178 [08:27<01:07,  2.39it/s]\u001b[A\n",
      "Iteration:  86%|████████▋ | 1018/1178 [08:27<01:06,  2.39it/s]\u001b[A\n",
      "Iteration:  87%|████████▋ | 1019/1178 [08:28<01:06,  2.39it/s]\u001b[A\n",
      "Iteration:  87%|████████▋ | 1020/1178 [08:28<01:08,  2.31it/s]\u001b[A\n",
      "Iteration:  87%|████████▋ | 1021/1178 [08:29<01:07,  2.33it/s]\u001b[A\n",
      "Iteration:  87%|████████▋ | 1022/1178 [08:29<01:06,  2.35it/s]\u001b[A\n",
      "Iteration:  87%|████████▋ | 1023/1178 [08:29<01:05,  2.37it/s]\u001b[A\n",
      "Iteration:  87%|████████▋ | 1024/1178 [08:30<01:05,  2.37it/s]\u001b[A\n",
      "Iteration:  87%|████████▋ | 1025/1178 [08:30<01:04,  2.38it/s]\u001b[A\n",
      "Iteration:  87%|████████▋ | 1026/1178 [08:31<01:03,  2.39it/s]\u001b[A\n",
      "Iteration:  87%|████████▋ | 1027/1178 [08:31<01:03,  2.39it/s]\u001b[A\n",
      "Iteration:  87%|████████▋ | 1028/1178 [08:31<01:02,  2.39it/s]\u001b[A\n",
      "Iteration:  87%|████████▋ | 1029/1178 [08:32<01:02,  2.40it/s]\u001b[A\n",
      "Iteration:  87%|████████▋ | 1030/1178 [08:34<01:59,  1.23it/s]\u001b[A\n",
      "Iteration:  88%|████████▊ | 1031/1178 [08:34<01:42,  1.44it/s]\u001b[A\n",
      "Iteration:  88%|████████▊ | 1032/1178 [08:34<01:29,  1.64it/s]\u001b[A\n",
      "Iteration:  88%|████████▊ | 1033/1178 [08:35<01:20,  1.81it/s]\u001b[A\n",
      "Iteration:  88%|████████▊ | 1034/1178 [08:35<01:14,  1.94it/s]\u001b[A\n",
      "Iteration:  88%|████████▊ | 1035/1178 [08:36<01:09,  2.06it/s]\u001b[A\n",
      "Iteration:  88%|████████▊ | 1036/1178 [08:36<01:06,  2.14it/s]\u001b[A\n",
      "Iteration:  88%|████████▊ | 1037/1178 [08:37<01:03,  2.21it/s]\u001b[A\n",
      "Iteration:  88%|████████▊ | 1038/1178 [08:37<01:01,  2.27it/s]\u001b[A\n",
      "Iteration:  88%|████████▊ | 1039/1178 [08:37<01:00,  2.30it/s]\u001b[A\n",
      "Iteration:  88%|████████▊ | 1040/1178 [08:38<01:01,  2.24it/s]\u001b[A\n",
      "Iteration:  88%|████████▊ | 1041/1178 [08:38<01:00,  2.28it/s]\u001b[A\n",
      "Iteration:  88%|████████▊ | 1042/1178 [08:39<00:58,  2.31it/s]\u001b[A\n",
      "Iteration:  89%|████████▊ | 1043/1178 [08:39<00:58,  2.33it/s]\u001b[A\n",
      "Iteration:  89%|████████▊ | 1044/1178 [08:40<00:57,  2.34it/s]\u001b[A\n",
      "Iteration:  89%|████████▊ | 1045/1178 [08:40<00:56,  2.36it/s]\u001b[A\n",
      "Iteration:  89%|████████▉ | 1046/1178 [08:40<00:55,  2.37it/s]\u001b[A\n",
      "Iteration:  89%|████████▉ | 1047/1178 [08:41<00:55,  2.38it/s]\u001b[A\n",
      "Iteration:  89%|████████▉ | 1048/1178 [08:41<00:54,  2.38it/s]\u001b[A\n",
      "Iteration:  89%|████████▉ | 1049/1178 [08:42<00:54,  2.37it/s]\u001b[A\n",
      "Iteration:  89%|████████▉ | 1050/1178 [08:42<00:55,  2.30it/s]\u001b[A\n",
      "Iteration:  89%|████████▉ | 1051/1178 [08:43<00:54,  2.32it/s]\u001b[A\n",
      "Iteration:  89%|████████▉ | 1052/1178 [08:43<00:53,  2.34it/s]\u001b[A\n",
      "Iteration:  89%|████████▉ | 1053/1178 [08:43<00:53,  2.35it/s]\u001b[A\n",
      "Iteration:  89%|████████▉ | 1054/1178 [08:44<00:52,  2.36it/s]\u001b[A\n",
      "Iteration:  90%|████████▉ | 1055/1178 [08:44<00:51,  2.37it/s]\u001b[A\n",
      "Iteration:  90%|████████▉ | 1056/1178 [08:45<00:51,  2.37it/s]\u001b[A\n",
      "Iteration:  90%|████████▉ | 1057/1178 [08:45<00:51,  2.37it/s]\u001b[A\n",
      "Iteration:  90%|████████▉ | 1058/1178 [08:45<00:50,  2.37it/s]\u001b[A\n",
      "Iteration:  90%|████████▉ | 1059/1178 [08:46<00:50,  2.37it/s]\u001b[A\n",
      "Iteration:  90%|████████▉ | 1060/1178 [08:46<00:51,  2.29it/s]\u001b[A\n",
      "Iteration:  90%|█████████ | 1061/1178 [08:47<00:50,  2.32it/s]\u001b[A\n",
      "Iteration:  90%|█████████ | 1062/1178 [08:47<00:49,  2.35it/s]\u001b[A\n",
      "Iteration:  90%|█████████ | 1063/1178 [08:48<00:48,  2.36it/s]\u001b[A\n",
      "Iteration:  90%|█████████ | 1064/1178 [08:49<01:30,  1.25it/s]\u001b[A\n",
      "Iteration:  90%|█████████ | 1065/1178 [08:50<01:17,  1.46it/s]\u001b[A\n",
      "Iteration:  90%|█████████ | 1066/1178 [08:50<01:07,  1.65it/s]\u001b[A\n",
      "Iteration:  91%|█████████ | 1067/1178 [08:51<01:00,  1.82it/s]\u001b[A\n",
      "Iteration:  91%|█████████ | 1068/1178 [08:51<00:56,  1.96it/s]\u001b[A\n",
      "Iteration:  91%|█████████ | 1069/1178 [08:51<00:52,  2.07it/s]\u001b[A\n",
      "Iteration:  91%|█████████ | 1070/1178 [08:52<00:51,  2.09it/s]\u001b[A\n",
      "Iteration:  91%|█████████ | 1071/1178 [08:52<00:49,  2.17it/s]\u001b[A\n",
      "Iteration:  91%|█████████ | 1072/1178 [08:53<00:47,  2.23it/s]\u001b[A\n",
      "Iteration:  91%|█████████ | 1073/1178 [08:53<00:46,  2.27it/s]\u001b[A\n",
      "Iteration:  91%|█████████ | 1074/1178 [08:54<00:45,  2.31it/s]\u001b[A\n",
      "Iteration:  91%|█████████▏| 1075/1178 [08:54<00:44,  2.32it/s]\u001b[A\n",
      "Iteration:  91%|█████████▏| 1076/1178 [08:54<00:43,  2.34it/s]\u001b[A\n",
      "Iteration:  91%|█████████▏| 1077/1178 [08:55<00:42,  2.35it/s]\u001b[A\n",
      "Iteration:  92%|█████████▏| 1078/1178 [08:55<00:42,  2.36it/s]\u001b[A\n",
      "Iteration:  92%|█████████▏| 1079/1178 [08:56<00:41,  2.37it/s]\u001b[A\n",
      "Iteration:  92%|█████████▏| 1080/1178 [08:56<00:42,  2.29it/s]\u001b[A\n",
      "Iteration:  92%|█████████▏| 1081/1178 [08:57<00:41,  2.31it/s]\u001b[A\n",
      "Iteration:  92%|█████████▏| 1082/1178 [08:57<00:41,  2.33it/s]\u001b[A\n",
      "Iteration:  92%|█████████▏| 1083/1178 [08:57<00:40,  2.35it/s]\u001b[A\n",
      "Iteration:  92%|█████████▏| 1084/1178 [08:58<00:39,  2.36it/s]\u001b[A\n",
      "Iteration:  92%|█████████▏| 1085/1178 [08:58<00:39,  2.36it/s]\u001b[A\n",
      "Iteration:  92%|█████████▏| 1086/1178 [08:59<00:38,  2.37it/s]\u001b[A\n",
      "Iteration:  92%|█████████▏| 1087/1178 [08:59<00:38,  2.37it/s]\u001b[A\n",
      "Iteration:  92%|█████████▏| 1088/1178 [08:59<00:37,  2.37it/s]\u001b[A\n",
      "Iteration:  92%|█████████▏| 1089/1178 [09:00<00:37,  2.37it/s]\u001b[A\n",
      "Iteration:  93%|█████████▎| 1090/1178 [09:00<00:38,  2.29it/s]\u001b[A\n",
      "Iteration:  93%|█████████▎| 1091/1178 [09:01<00:37,  2.32it/s]\u001b[A\n",
      "Iteration:  93%|█████████▎| 1092/1178 [09:01<00:36,  2.34it/s]\u001b[A\n",
      "Iteration:  93%|█████████▎| 1093/1178 [09:02<00:36,  2.35it/s]\u001b[A\n",
      "Iteration:  93%|█████████▎| 1094/1178 [09:02<00:35,  2.36it/s]\u001b[A\n",
      "Iteration:  93%|█████████▎| 1095/1178 [09:02<00:35,  2.37it/s]\u001b[A\n",
      "Iteration:  93%|█████████▎| 1096/1178 [09:03<00:34,  2.37it/s]\u001b[A\n",
      "Iteration:  93%|█████████▎| 1097/1178 [09:03<00:34,  2.37it/s]\u001b[A\n",
      "Iteration:  93%|█████████▎| 1098/1178 [09:05<01:04,  1.24it/s]\u001b[A\n",
      "Iteration:  93%|█████████▎| 1099/1178 [09:05<00:54,  1.45it/s]\u001b[A\n",
      "Iteration:  93%|█████████▎| 1100/1178 [09:06<00:48,  1.61it/s]\u001b[A\n",
      "Iteration:  93%|█████████▎| 1101/1178 [09:06<00:43,  1.78it/s]\u001b[A\n",
      "Iteration:  94%|█████████▎| 1102/1178 [09:07<00:39,  1.92it/s]\u001b[A\n",
      "Iteration:  94%|█████████▎| 1103/1178 [09:07<00:36,  2.05it/s]\u001b[A\n",
      "Iteration:  94%|█████████▎| 1104/1178 [09:08<00:34,  2.14it/s]\u001b[A\n",
      "Iteration:  94%|█████████▍| 1105/1178 [09:08<00:33,  2.21it/s]\u001b[A\n",
      "Iteration:  94%|█████████▍| 1106/1178 [09:08<00:31,  2.26it/s]\u001b[A\n",
      "Iteration:  94%|█████████▍| 1107/1178 [09:09<00:30,  2.29it/s]\u001b[A\n",
      "Iteration:  94%|█████████▍| 1108/1178 [09:09<00:30,  2.33it/s]\u001b[A\n",
      "Iteration:  94%|█████████▍| 1109/1178 [09:10<00:29,  2.35it/s]\u001b[A\n",
      "Iteration:  94%|█████████▍| 1110/1178 [09:10<00:29,  2.28it/s]\u001b[A\n",
      "Iteration:  94%|█████████▍| 1111/1178 [09:11<00:29,  2.30it/s]\u001b[A\n",
      "Iteration:  94%|█████████▍| 1112/1178 [09:11<00:28,  2.33it/s]\u001b[A\n",
      "Iteration:  94%|█████████▍| 1113/1178 [09:11<00:27,  2.34it/s]\u001b[A\n",
      "Iteration:  95%|█████████▍| 1114/1178 [09:12<00:27,  2.36it/s]\u001b[A\n",
      "Iteration:  95%|█████████▍| 1115/1178 [09:12<00:26,  2.36it/s]\u001b[A\n",
      "Iteration:  95%|█████████▍| 1116/1178 [09:13<00:26,  2.37it/s]\u001b[A\n",
      "Iteration:  95%|█████████▍| 1117/1178 [09:13<00:25,  2.38it/s]\u001b[A\n",
      "Iteration:  95%|█████████▍| 1118/1178 [09:13<00:25,  2.38it/s]\u001b[A\n",
      "Iteration:  95%|█████████▍| 1119/1178 [09:14<00:24,  2.38it/s]\u001b[A\n",
      "Iteration:  95%|█████████▌| 1120/1178 [09:14<00:25,  2.30it/s]\u001b[A\n",
      "Iteration:  95%|█████████▌| 1121/1178 [09:15<00:24,  2.32it/s]\u001b[A\n",
      "Iteration:  95%|█████████▌| 1122/1178 [09:15<00:23,  2.33it/s]\u001b[A\n",
      "Iteration:  95%|█████████▌| 1123/1178 [09:16<00:23,  2.35it/s]\u001b[A\n",
      "Iteration:  95%|█████████▌| 1124/1178 [09:16<00:22,  2.36it/s]\u001b[A\n",
      "Iteration:  96%|█████████▌| 1125/1178 [09:16<00:22,  2.37it/s]\u001b[A\n",
      "Iteration:  96%|█████████▌| 1126/1178 [09:17<00:21,  2.38it/s]\u001b[A\n",
      "Iteration:  96%|█████████▌| 1127/1178 [09:17<00:21,  2.37it/s]\u001b[A\n",
      "Iteration:  96%|█████████▌| 1128/1178 [09:18<00:21,  2.37it/s]\u001b[A\n",
      "Iteration:  96%|█████████▌| 1129/1178 [09:18<00:20,  2.38it/s]\u001b[A\n",
      "Iteration:  96%|█████████▌| 1130/1178 [09:19<00:20,  2.30it/s]\u001b[A\n",
      "Iteration:  96%|█████████▌| 1131/1178 [09:19<00:20,  2.32it/s]\u001b[A\n",
      "Iteration:  96%|█████████▌| 1132/1178 [09:21<00:37,  1.23it/s]\u001b[A\n",
      "Iteration:  96%|█████████▌| 1133/1178 [09:21<00:31,  1.44it/s]\u001b[A\n",
      "Iteration:  96%|█████████▋| 1134/1178 [09:22<00:26,  1.63it/s]\u001b[A\n",
      "Iteration:  96%|█████████▋| 1135/1178 [09:22<00:23,  1.80it/s]\u001b[A\n",
      "Iteration:  96%|█████████▋| 1136/1178 [09:22<00:21,  1.94it/s]\u001b[A\n",
      "Iteration:  97%|█████████▋| 1137/1178 [09:23<00:19,  2.05it/s]\u001b[A\n",
      "Iteration:  97%|█████████▋| 1138/1178 [09:23<00:18,  2.14it/s]\u001b[A\n",
      "Iteration:  97%|█████████▋| 1139/1178 [09:24<00:17,  2.21it/s]\u001b[A\n",
      "Iteration:  97%|█████████▋| 1140/1178 [09:24<00:17,  2.19it/s]\u001b[A\n",
      "Iteration:  97%|█████████▋| 1141/1178 [09:25<00:16,  2.24it/s]\u001b[A\n",
      "Iteration:  97%|█████████▋| 1142/1178 [09:25<00:15,  2.27it/s]\u001b[A\n",
      "Iteration:  97%|█████████▋| 1143/1178 [09:25<00:15,  2.30it/s]\u001b[A\n",
      "Iteration:  97%|█████████▋| 1144/1178 [09:26<00:14,  2.32it/s]\u001b[A\n",
      "Iteration:  97%|█████████▋| 1145/1178 [09:26<00:14,  2.33it/s]\u001b[A\n",
      "Iteration:  97%|█████████▋| 1146/1178 [09:27<00:13,  2.34it/s]\u001b[A\n",
      "Iteration:  97%|█████████▋| 1147/1178 [09:27<00:13,  2.34it/s]\u001b[A\n",
      "Iteration:  97%|█████████▋| 1148/1178 [09:28<00:12,  2.36it/s]\u001b[A\n",
      "Iteration:  98%|█████████▊| 1149/1178 [09:28<00:12,  2.36it/s]\u001b[A\n",
      "Iteration:  98%|█████████▊| 1150/1178 [09:28<00:12,  2.29it/s]\u001b[A\n",
      "Iteration:  98%|█████████▊| 1151/1178 [09:29<00:11,  2.31it/s]\u001b[A\n",
      "Iteration:  98%|█████████▊| 1152/1178 [09:29<00:11,  2.33it/s]\u001b[A\n",
      "Iteration:  98%|█████████▊| 1153/1178 [09:30<00:10,  2.34it/s]\u001b[A\n",
      "Iteration:  98%|█████████▊| 1154/1178 [09:30<00:10,  2.35it/s]\u001b[A\n",
      "Iteration:  98%|█████████▊| 1155/1178 [09:31<00:09,  2.35it/s]\u001b[A\n",
      "Iteration:  98%|█████████▊| 1156/1178 [09:31<00:09,  2.36it/s]\u001b[A\n",
      "Iteration:  98%|█████████▊| 1157/1178 [09:31<00:08,  2.37it/s]\u001b[A\n",
      "Iteration:  98%|█████████▊| 1158/1178 [09:32<00:08,  2.36it/s]\u001b[A\n",
      "Iteration:  98%|█████████▊| 1159/1178 [09:32<00:08,  2.37it/s]\u001b[A\n",
      "Iteration:  98%|█████████▊| 1160/1178 [09:33<00:07,  2.29it/s]\u001b[A\n",
      "Iteration:  99%|█████████▊| 1161/1178 [09:33<00:07,  2.31it/s]\u001b[A\n",
      "Iteration:  99%|█████████▊| 1162/1178 [09:34<00:06,  2.33it/s]\u001b[A\n",
      "Iteration:  99%|█████████▊| 1163/1178 [09:34<00:06,  2.34it/s]\u001b[A\n",
      "Iteration:  99%|█████████▉| 1164/1178 [09:34<00:05,  2.35it/s]\u001b[A\n",
      "Iteration:  99%|█████████▉| 1165/1178 [09:35<00:05,  2.35it/s]\u001b[A\n",
      "Iteration:  99%|█████████▉| 1166/1178 [09:37<00:09,  1.23it/s]\u001b[A\n",
      "Iteration:  99%|█████████▉| 1167/1178 [09:37<00:07,  1.43it/s]\u001b[A\n",
      "Iteration:  99%|█████████▉| 1168/1178 [09:37<00:06,  1.62it/s]\u001b[A\n",
      "Iteration:  99%|█████████▉| 1169/1178 [09:38<00:05,  1.79it/s]\u001b[A\n",
      "Iteration:  99%|█████████▉| 1170/1178 [09:38<00:04,  1.87it/s]\u001b[A\n",
      "Iteration:  99%|█████████▉| 1171/1178 [09:39<00:03,  1.99it/s]\u001b[A\n",
      "Iteration:  99%|█████████▉| 1172/1178 [09:39<00:02,  2.09it/s]\u001b[A\n",
      "Iteration: 100%|█████████▉| 1173/1178 [09:40<00:02,  2.17it/s]\u001b[A\n",
      "Iteration: 100%|█████████▉| 1174/1178 [09:40<00:01,  2.23it/s]\u001b[A\n",
      "Iteration: 100%|█████████▉| 1175/1178 [09:40<00:01,  2.27it/s]\u001b[A\n",
      "Iteration: 100%|█████████▉| 1176/1178 [09:41<00:00,  2.31it/s]\u001b[A\n",
      "Iteration: 100%|█████████▉| 1177/1178 [09:41<00:00,  2.34it/s]\u001b[A\n",
      "Iteration: 100%|██████████| 1178/1178 [09:42<00:00,  1.99it/s]\u001b[A\n",
      "Epoch: 100%|██████████| 1/1 [09:42<00:00, 582.43s/it]\n"
     ]
    }
   ],
   "source": [
    "global_step = 0\n",
    "if args.do_train:\n",
    "    model.train()\n",
    "    for _ in trange(int(args.num_train_epochs), desc=\"Epoch\"):\n",
    "        tr_loss = 0\n",
    "        nb_tr_examples, nb_tr_steps = 0, 0\n",
    "        for step, batch in enumerate(tqdm(train_dataloader, desc=\"Iteration\")):\n",
    "            batch = tuple(t.to(device) for t in batch)\n",
    "            input_ids, input_mask, segment_ids, label_ids = batch\n",
    "            loss = model(input_ids, segment_ids, input_mask, label_ids)\n",
    "            if n_gpu > 1:\n",
    "                loss = loss.mean() # mean() to average on multi-gpu.\n",
    "            if args.fp16 and args.loss_scale != 1.0:\n",
    "                # rescale loss for fp16 training\n",
    "                # see https://docs.nvidia.com/deeplearning/sdk/mixed-precision-training/index.html\n",
    "                loss = loss * args.loss_scale\n",
    "            if args.gradient_accumulation_steps > 1:\n",
    "                loss = loss / args.gradient_accumulation_steps\n",
    "            tr_loss += loss.item()\n",
    "            nb_tr_examples += input_ids.size(0)\n",
    "            nb_tr_steps += 1\n",
    "\n",
    "            if args.fp16:\n",
    "                optimizer.backward(loss)\n",
    "            else:\n",
    "                loss.backward()\n",
    "            if (step + 1) % args.gradient_accumulation_steps == 0:\n",
    "                # modify learning rate with special warm up BERT uses\n",
    "                lr_this_step = args.learning_rate * warmup_linear(global_step/t_total, args.warmup_proportion)\n",
    "                for param_group in optimizer.param_groups:\n",
    "                    param_group['lr'] = lr_this_step\n",
    "                optimizer.step()\n",
    "                optimizer.zero_grad()\n",
    "                global_step += 1\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": [
    "# Save a trained model\n",
    "model_to_save = model.module if hasattr(model, 'module') else model  # Only save the model it-self\n",
    "output_model_file = os.path.join(args.output_dir, \"pytorch_model.bin\")\n",
    "torch.save(model_to_save.state_dict(), output_model_file)\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": [
    "# Load a trained model that you have fine-tuned\n",
    "# model_state_dict = torch.load('./search_data/swagaf/output_data_all/pytorch_model.bin')\n",
    "# model = BertForMultipleChoice.from_pretrained(args.bert_model,\n",
    "#     state_dict=model_state_dict,\n",
    "#     num_choices=10)\n",
    "# model.to(device)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": [
    "# model_state_dict = torch.load('./search_data/swagaf/output_data_2/pytorch_model.bin')\n",
    "# model = BertForMultipleChoice.from_pretrained(args.bert_model,\n",
    "#     state_dict=model_state_dict,\n",
    "#     num_choices=10)\n",
    "# model.to(device)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": [
    "test['label'] = None"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": [
    "test.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {
    "hidden": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(104170, 4)"
      ]
     },
     "execution_count": 39,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": [
    "test_dfs = [test[i:i+500] for i in range(0,test.shape[0],500)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": [
    "test_orig = convert_examples_with_pool(test_dfs,4,False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {
    "hidden": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "209"
      ]
     },
     "execution_count": 64,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(test_orig)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "hidden": true,
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "test_orig"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {
    "collapsed": true,
    "hidden": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None]"
      ]
     },
     "execution_count": 66,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_list_all = []\n",
    "[test_list_all.extend(i) for i in test_orig]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": [
    "len(test_list_all)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {
    "hidden": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<__main__.InputFeatures at 0x7f24bc7c3630>"
      ]
     },
     "execution_count": 68,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_list_all[1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": [
    "# test_orig = convert_examples_to_features(test, tokenizer, args.max_seq_length, False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": [
    "args.eval_batch_size = 80"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "heading_collapsed": true
   },
   "source": [
    "## Checking MRR on Eval set"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {
    "code_folding": [],
    "hidden": true,
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "12/29/2018 10:20:29 - INFO - __main__ -   ***** Running evaluation *****\n",
      "12/29/2018 10:20:29 - INFO - __main__ -     Num examples = 5000\n",
      "12/29/2018 10:20:29 - INFO - __main__ -     Batch size = 30\n",
      "/home/ec2-user/anaconda3/envs/pytorch_p36/lib/python3.6/site-packages/torch/nn/parallel/_functions.py:58: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0\n",
      "100\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "12/29/2018 10:21:38 - INFO - __main__ -   ***** Eval results *****\n",
      "12/29/2018 10:21:38 - INFO - __main__ -     eval_accuracy = 0.5276\n",
      "12/29/2018 10:21:38 - INFO - __main__ -     eval_loss = 1.2896398434382\n",
      "12/29/2018 10:21:38 - INFO - __main__ -     eval_mrr = 0.6976818253968254\n"
     ]
    }
   ],
   "source": [
    "\n",
    "final_result = []\n",
    "\n",
    "if args.do_eval and (args.local_rank == -1 or torch.distributed.get_rank() == 0):\n",
    "#     eval_examples = read_swag_examples(os.path.join(args.data_dir, 'val.csv'), is_training = True)\n",
    "#     eval_examples = train[-4000:]\n",
    "#     eval_features = convert_examples_to_features(\n",
    "#         eval_examples, tokenizer, args.max_seq_length, True)\n",
    "    eval_features = f_list_all[4000:9000]\n",
    "    logger.info(\"***** Running evaluation *****\")\n",
    "    logger.info(\"  Num examples = %d\", len(eval_features))\n",
    "    logger.info(\"  Batch size = %d\", args.eval_batch_size)\n",
    "    all_input_ids = torch.tensor(select_field(eval_features, 'input_ids'), dtype=torch.long)\n",
    "    all_input_mask = torch.tensor(select_field(eval_features, 'input_mask'), dtype=torch.long)\n",
    "    all_segment_ids = torch.tensor(select_field(eval_features, 'segment_ids'), dtype=torch.long)\n",
    "    all_label = torch.tensor([f.label for f in eval_features], dtype=torch.long)\n",
    "    eval_data = TensorDataset(all_input_ids, all_input_mask, all_segment_ids, all_label)\n",
    "    # Run prediction for full data\n",
    "    eval_sampler = SequentialSampler(eval_data)\n",
    "    eval_dataloader = DataLoader(eval_data, sampler=eval_sampler, batch_size=args.eval_batch_size)\n",
    "\n",
    "    model.eval()\n",
    "    eval_loss, eval_accuracy = 0, 0\n",
    "    eval_mrr = 0\n",
    "    nb_eval_steps, nb_eval_examples = 0, 0\n",
    "    for input_ids, input_mask, segment_ids, label_ids in eval_dataloader:\n",
    "        input_ids = input_ids.to(device)\n",
    "        input_mask = input_mask.to(device)\n",
    "        segment_ids = segment_ids.to(device)\n",
    "        label_ids = label_ids.to(device)\n",
    "\n",
    "        with torch.no_grad():\n",
    "            tmp_eval_loss = model(input_ids, segment_ids, input_mask, label_ids)\n",
    "            logits = model(input_ids, segment_ids, input_mask)\n",
    "\n",
    "        logits = logits.detach().cpu().numpy()\n",
    "        [final_result.append(l) for l in logits]\n",
    "        \n",
    "        label_ids = label_ids.to('cpu').numpy()\n",
    "        \n",
    "        eval_mrr += get_mrr2(logits,label_ids)\n",
    "        \n",
    "        #final_result.append(logits)\n",
    "        \n",
    "        \n",
    "        if(nb_eval_steps % 100 == 0):\n",
    "            print (nb_eval_steps)\n",
    "#             print(logits)\n",
    "#             print(label_ids)\n",
    "        \n",
    "        tmp_eval_accuracy = accuracy(logits, label_ids)\n",
    "\n",
    "        eval_loss += tmp_eval_loss.mean().item()\n",
    "        eval_accuracy += tmp_eval_accuracy\n",
    "\n",
    "        nb_eval_examples += input_ids.size(0)\n",
    "        nb_eval_steps += 1\n",
    "\n",
    "    eval_loss = eval_loss / nb_eval_steps\n",
    "    eval_accuracy = eval_accuracy / nb_eval_examples\n",
    "    eval_mrr = eval_mrr/nb_eval_examples\n",
    "\n",
    "    result = {'eval_loss': eval_loss,\n",
    "              'eval_accuracy': eval_accuracy,\n",
    "#               'global_step': global_step,\n",
    "#               'loss': tr_loss/nb_tr_steps,\n",
    "              'eval_mrr': eval_mrr}\n",
    "\n",
    "    output_eval_file = os.path.join(args.output_dir, \"eval_results.txt\")\n",
    "    with open(output_eval_file, \"w\") as writer:\n",
    "        logger.info(\"***** Eval results *****\")\n",
    "        for key in sorted(result.keys()):\n",
    "            logger.info(\"  %s = %s\", key, str(result[key]))\n",
    "            writer.write(\"%s = %s\\n\" % (key, str(result[key])))\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": [
    "# Empty the GPU memory\n",
    "import gc\n",
    "gc.collect()\n",
    "torch.cuda.empty_cache()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "heading_collapsed": true
   },
   "source": [
    "## Predicting on Test-Set"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": [
    "# r_file1 = open('./search_data/test_features_list_all', 'rb') # Phase-1 Test Set\n",
    "r_file1 = open('./search_data/test_2_features_list_all_290', 'rb') # Phase-2 Test Set\n",
    "test_list_all = pickle.load(r_file1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "metadata": {
    "hidden": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(415380, 4)"
      ]
     },
     "execution_count": 73,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "metadata": {
    "hidden": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "41538"
      ]
     },
     "execution_count": 72,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(test_list_all)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 100,
   "metadata": {
    "hidden": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(290,\n",
       " {'input_ids': [101,\n",
       "   2339,\n",
       "   2106,\n",
       "   1996,\n",
       "   9747,\n",
       "   8998,\n",
       "   21754,\n",
       "   1029,\n",
       "   102,\n",
       "   2348,\n",
       "   1996,\n",
       "   2647,\n",
       "   4204,\n",
       "   2106,\n",
       "   2191,\n",
       "   2510,\n",
       "   19388,\n",
       "   1999,\n",
       "   3763,\n",
       "   2637,\n",
       "   2013,\n",
       "   2051,\n",
       "   2000,\n",
       "   2051,\n",
       "   2044,\n",
       "   1996,\n",
       "   9747,\n",
       "   8998,\n",
       "   2001,\n",
       "   2623,\n",
       "   1010,\n",
       "   1996,\n",
       "   4841,\n",
       "   2106,\n",
       "   2025,\n",
       "   2298,\n",
       "   2005,\n",
       "   2162,\n",
       "   1012,\n",
       "   2027,\n",
       "   2106,\n",
       "   1010,\n",
       "   2174,\n",
       "   1010,\n",
       "   2224,\n",
       "   1996,\n",
       "   8998,\n",
       "   2004,\n",
       "   19777,\n",
       "   2005,\n",
       "   2635,\n",
       "   3146,\n",
       "   1999,\n",
       "   10008,\n",
       "   2104,\n",
       "   2343,\n",
       "   2198,\n",
       "   7482,\n",
       "   1012,\n",
       "   102,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0],\n",
       "  'input_mask': [1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0],\n",
       "  'segment_ids': [0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   1,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0,\n",
       "   0]})"
      ]
     },
     "execution_count": 100,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# a = test_list_all[1]\n",
    "# len(a.choices_features[0]['input_ids']),a.choices_features[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 101,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": [
    "args.eval_batch_size = 140"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 102,
   "metadata": {
    "hidden": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "12/29/2018 10:48:42 - INFO - __main__ -   ***** Running evaluation *****\n",
      "12/29/2018 10:48:42 - INFO - __main__ -     Num examples = 41538\n",
      "12/29/2018 10:48:42 - INFO - __main__ -     Batch size = 140\n"
     ]
    }
   ],
   "source": [
    "# def main():\n",
    "\n",
    "final_result = []\n",
    "\n",
    "if args.do_eval and (args.local_rank == -1 or torch.distributed.get_rank() == 0):\n",
    "#     eval_examples = read_swag_examples(os.path.join(args.data_dir, 'val.csv'), is_training = True)\n",
    "#     eval_examples = train[4000:8000]\n",
    "    \n",
    "    eval_features = test_list_all\n",
    "    logger.info(\"***** Running evaluation *****\")\n",
    "    logger.info(\"  Num examples = %d\", len(eval_features))\n",
    "    logger.info(\"  Batch size = %d\", args.eval_batch_size)\n",
    "    all_input_ids = torch.tensor(select_field(eval_features, 'input_ids'), dtype=torch.long)\n",
    "    all_input_mask = torch.tensor(select_field(eval_features, 'input_mask'), dtype=torch.long)\n",
    "    all_segment_ids = torch.tensor(select_field(eval_features, 'segment_ids'), dtype=torch.long)\n",
    "#     all_label = torch.tensor([f.label for f in eval_features], dtype=torch.long)\n",
    "    eval_data = TensorDataset(all_input_ids, all_input_mask, all_segment_ids)\n",
    "    # Run prediction for full data\n",
    "    eval_sampler = SequentialSampler(eval_data)\n",
    "    eval_dataloader = DataLoader(eval_data, sampler=eval_sampler, batch_size=args.eval_batch_size)\n",
    "\n",
    "    model.eval()\n",
    "#     eval_loss, eval_accuracy = 0, 0\n",
    "#     eval_mrr = 0\n",
    "#     nb_eval_steps, nb_eval_examples = 0, 0\n",
    "    for input_ids, input_mask, segment_ids in eval_dataloader:\n",
    "        input_ids = input_ids.to(device)\n",
    "        input_mask = input_mask.to(device)\n",
    "        segment_ids = segment_ids.to(device)\n",
    "#         label_ids = label_ids.to(device)\n",
    "\n",
    "        with torch.no_grad():\n",
    "#             tmp_eval_loss = model(input_ids, segment_ids, input_mask, label_ids)\n",
    "            logits = model(input_ids, segment_ids, input_mask)\n",
    "\n",
    "        logits = logits.detach().cpu().numpy()\n",
    "        [final_result.append(l) for l in logits]\n",
    "        \n",
    "#         label_ids = label_ids.to('cpu').numpy()\n",
    "        \n",
    "#         eval_mrr += get_mrr2(logits,label_ids)\n",
    "        \n",
    "        #final_result.append(logits)\n",
    "        \n",
    "        \n",
    "#         if(nb_eval_steps % 100 == 0):\n",
    "#             print(nb_eval_steps)\n",
    "#             print(label_ids)\n",
    "        \n",
    "#         tmp_eval_accuracy = accuracy(logits, label_ids)\n",
    "\n",
    "#         eval_loss += tmp_eval_loss.mean().item()\n",
    "#         eval_accuracy += tmp_eval_accuracy\n",
    "\n",
    "#         nb_eval_examples += input_ids.size(0)\n",
    "#         nb_eval_steps += 1\n",
    "\n",
    "#     eval_loss = eval_loss / nb_eval_steps\n",
    "#     eval_accuracy = eval_accuracy / nb_eval_examples\n",
    "#     eval_mrr = eval_mrr/nb_eval_examples\n",
    "\n",
    "#     result = {'eval_loss': eval_loss,\n",
    "#               'eval_accuracy': eval_accuracy,\n",
    "#               'global_step': global_step,\n",
    "#               'loss': tr_loss/nb_tr_steps,\n",
    "#               'eval_mrr': eval_mrr}\n",
    "\n",
    "#     output_eval_file = os.path.join(args.output_dir, \"eval_results.txt\")\n",
    "#     with open(output_eval_file, \"w\") as writer:\n",
    "#         logger.info(\"***** Eval results *****\")\n",
    "#         for key in sorted(result.keys()):\n",
    "#             logger.info(\"  %s = %s\", key, str(result[key]))\n",
    "#             writer.write(\"%s = %s\\n\" % (key, str(result[key])))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 103,
   "metadata": {
    "hidden": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "41538"
      ]
     },
     "execution_count": 103,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(final_result)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {
    "hidden": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>query_id</th>\n",
       "      <th>query</th>\n",
       "      <th>passage_text</th>\n",
       "      <th>passage_id</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1135787</td>\n",
       "      <td>distance between erie in buffalo new york</td>\n",
       "      <td>Erie Canal Distance Tables The Erie Canal is t...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1135787</td>\n",
       "      <td>distance between erie in buffalo new york</td>\n",
       "      <td>What is the distance between Erie AND Buffalo?...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1135787</td>\n",
       "      <td>distance between erie in buffalo new york</td>\n",
       "      <td>The distance between Erie and Buffalo in a str...</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1135787</td>\n",
       "      <td>distance between erie in buffalo new york</td>\n",
       "      <td>Erie Canal Distances. Erie Canal Distance Tabl...</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1135787</td>\n",
       "      <td>distance between erie in buffalo new york</td>\n",
       "      <td>Erie's Metropolitan Area consists of approxima...</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   query_id                                      query  \\\n",
       "0   1135787  distance between erie in buffalo new york   \n",
       "1   1135787  distance between erie in buffalo new york   \n",
       "2   1135787  distance between erie in buffalo new york   \n",
       "3   1135787  distance between erie in buffalo new york   \n",
       "4   1135787  distance between erie in buffalo new york   \n",
       "\n",
       "                                        passage_text  passage_id  \n",
       "0  Erie Canal Distance Tables The Erie Canal is t...           0  \n",
       "1  What is the distance between Erie AND Buffalo?...           1  \n",
       "2  The distance between Erie and Buffalo in a str...           2  \n",
       "3  Erie Canal Distances. Erie Canal Distance Tabl...           3  \n",
       "4  Erie's Metropolitan Area consists of approxima...           4  "
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": [
    "from functools import reduce"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": [
    "final_result_copy = final_result.copy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": [
    "final_result_copy[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": [
    "ranks = np.argsort(final_result_copy,axis=1)[::-1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": [
    "# rank = ranks[0]\n",
    "# ranks_labels\n",
    "# [ rank,lab for (rank,label) in ranks_labels]\n",
    "# [1/(np.where(rank==label)[0][0]+1) for (rank,label) in zip(ranks,[1,2,3,4,5,6,7,8,9,0])]\n",
    "# np.sum([1/(np.where(rank==label)[0][0]+1) for (rank,label) in zip(ranks,[1,2,3,4,5,6,7,8,9,0])])\n",
    "\n",
    "# def get_mrr(logits,labels):\n",
    "    \n",
    "#     ranks = np.argsort(logits,axis=1)[::-1]\n",
    "#     sum1 = np.sum([1/(np.where(rank==label)[0][0]+1) for (rank,label) in zip(ranks,labels)])\n",
    "#     return sum1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 104,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": [
    "columns = ['query_id','1','2','3','4','5','6','7','8','9','10']\n",
    "ans_headers = ['1','2','3','4','5','6','7','8','9','10']\n",
    "ans_eval = pd.DataFrame(data = final_result,columns=ans_headers)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 82,
   "metadata": {
    "hidden": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "      <th>4</th>\n",
       "      <th>5</th>\n",
       "      <th>6</th>\n",
       "      <th>7</th>\n",
       "      <th>8</th>\n",
       "      <th>9</th>\n",
       "      <th>10</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>-3.471546</td>\n",
       "      <td>3.119705</td>\n",
       "      <td>2.748143</td>\n",
       "      <td>-2.061335</td>\n",
       "      <td>-2.470538</td>\n",
       "      <td>3.298495</td>\n",
       "      <td>3.491955</td>\n",
       "      <td>1.573026</td>\n",
       "      <td>1.000348</td>\n",
       "      <td>3.620442</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>-3.726930</td>\n",
       "      <td>-2.489444</td>\n",
       "      <td>-2.292486</td>\n",
       "      <td>-1.507174</td>\n",
       "      <td>-0.444416</td>\n",
       "      <td>2.315248</td>\n",
       "      <td>1.919158</td>\n",
       "      <td>-3.663156</td>\n",
       "      <td>0.951928</td>\n",
       "      <td>2.571643</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.069452</td>\n",
       "      <td>-4.416095</td>\n",
       "      <td>1.386632</td>\n",
       "      <td>-0.630060</td>\n",
       "      <td>1.512524</td>\n",
       "      <td>-3.993858</td>\n",
       "      <td>1.339767</td>\n",
       "      <td>0.383412</td>\n",
       "      <td>2.454320</td>\n",
       "      <td>-0.877230</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>-4.738125</td>\n",
       "      <td>0.573662</td>\n",
       "      <td>-2.688248</td>\n",
       "      <td>3.691872</td>\n",
       "      <td>-3.160819</td>\n",
       "      <td>-1.016214</td>\n",
       "      <td>-5.073241</td>\n",
       "      <td>-0.230091</td>\n",
       "      <td>1.103286</td>\n",
       "      <td>4.322522</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0.369257</td>\n",
       "      <td>-0.616475</td>\n",
       "      <td>2.434665</td>\n",
       "      <td>3.228459</td>\n",
       "      <td>1.249426</td>\n",
       "      <td>-1.635600</td>\n",
       "      <td>3.795533</td>\n",
       "      <td>0.576618</td>\n",
       "      <td>1.744282</td>\n",
       "      <td>-0.599759</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>-2.602319</td>\n",
       "      <td>-4.781091</td>\n",
       "      <td>1.013639</td>\n",
       "      <td>0.600060</td>\n",
       "      <td>0.241481</td>\n",
       "      <td>-3.583309</td>\n",
       "      <td>2.391228</td>\n",
       "      <td>1.014574</td>\n",
       "      <td>-2.044503</td>\n",
       "      <td>1.613165</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>1.652425</td>\n",
       "      <td>-4.265140</td>\n",
       "      <td>-5.054367</td>\n",
       "      <td>-5.105813</td>\n",
       "      <td>-5.111250</td>\n",
       "      <td>0.427820</td>\n",
       "      <td>1.477632</td>\n",
       "      <td>-5.023465</td>\n",
       "      <td>1.311460</td>\n",
       "      <td>-4.691817</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>-4.208994</td>\n",
       "      <td>3.630674</td>\n",
       "      <td>-4.350350</td>\n",
       "      <td>-4.796297</td>\n",
       "      <td>-4.539114</td>\n",
       "      <td>2.298984</td>\n",
       "      <td>-2.820923</td>\n",
       "      <td>-4.859423</td>\n",
       "      <td>-0.482275</td>\n",
       "      <td>0.331333</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>0.817823</td>\n",
       "      <td>1.128569</td>\n",
       "      <td>-0.540657</td>\n",
       "      <td>0.682980</td>\n",
       "      <td>1.259376</td>\n",
       "      <td>-0.471268</td>\n",
       "      <td>0.984114</td>\n",
       "      <td>0.688971</td>\n",
       "      <td>1.248173</td>\n",
       "      <td>0.190560</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>-4.781983</td>\n",
       "      <td>0.906716</td>\n",
       "      <td>-4.565093</td>\n",
       "      <td>-4.718335</td>\n",
       "      <td>-3.886076</td>\n",
       "      <td>-2.771877</td>\n",
       "      <td>-4.725819</td>\n",
       "      <td>-4.897281</td>\n",
       "      <td>-4.758878</td>\n",
       "      <td>-3.611917</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "          1         2         3         4         5         6         7  \\\n",
       "0 -3.471546  3.119705  2.748143 -2.061335 -2.470538  3.298495  3.491955   \n",
       "1 -3.726930 -2.489444 -2.292486 -1.507174 -0.444416  2.315248  1.919158   \n",
       "2  0.069452 -4.416095  1.386632 -0.630060  1.512524 -3.993858  1.339767   \n",
       "3 -4.738125  0.573662 -2.688248  3.691872 -3.160819 -1.016214 -5.073241   \n",
       "4  0.369257 -0.616475  2.434665  3.228459  1.249426 -1.635600  3.795533   \n",
       "5 -2.602319 -4.781091  1.013639  0.600060  0.241481 -3.583309  2.391228   \n",
       "6  1.652425 -4.265140 -5.054367 -5.105813 -5.111250  0.427820  1.477632   \n",
       "7 -4.208994  3.630674 -4.350350 -4.796297 -4.539114  2.298984 -2.820923   \n",
       "8  0.817823  1.128569 -0.540657  0.682980  1.259376 -0.471268  0.984114   \n",
       "9 -4.781983  0.906716 -4.565093 -4.718335 -3.886076 -2.771877 -4.725819   \n",
       "\n",
       "          8         9        10  \n",
       "0  1.573026  1.000348  3.620442  \n",
       "1 -3.663156  0.951928  2.571643  \n",
       "2  0.383412  2.454320 -0.877230  \n",
       "3 -0.230091  1.103286  4.322522  \n",
       "4  0.576618  1.744282 -0.599759  \n",
       "5  1.014574 -2.044503  1.613165  \n",
       "6 -5.023465  1.311460 -4.691817  \n",
       "7 -4.859423 -0.482275  0.331333  \n",
       "8  0.688971  1.248173  0.190560  \n",
       "9 -4.897281 -4.758878 -3.611917  "
      ]
     },
     "execution_count": 82,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ans_eval.head(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 106,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": [
    "# test2 = test.sort_values(['query_id','passage_id'])\n",
    "test2 = test.copy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 107,
   "metadata": {
    "hidden": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>query_id</th>\n",
       "      <th>query</th>\n",
       "      <th>passage_text</th>\n",
       "      <th>passage_id</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1136966</td>\n",
       "      <td>#ffffff color code</td>\n",
       "      <td>Color hex is a easy to use tool to get the col...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1136966</td>\n",
       "      <td>#ffffff color code</td>\n",
       "      <td>#ffffff Color Conversion. The hexadecimal colo...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1136966</td>\n",
       "      <td>#ffffff color code</td>\n",
       "      <td>CSS Codes; Color Preview; Color Schemes; Color...</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1136966</td>\n",
       "      <td>#ffffff color code</td>\n",
       "      <td>Color Hex Color Codes Color-hex gives informat...</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1136966</td>\n",
       "      <td>#ffffff color code</td>\n",
       "      <td>Color Hex Color Codes. Color-hex gives informa...</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   query_id               query  \\\n",
       "0   1136966  #ffffff color code   \n",
       "1   1136966  #ffffff color code   \n",
       "2   1136966  #ffffff color code   \n",
       "3   1136966  #ffffff color code   \n",
       "4   1136966  #ffffff color code   \n",
       "\n",
       "                                        passage_text  passage_id  \n",
       "0  Color hex is a easy to use tool to get the col...           0  \n",
       "1  #ffffff Color Conversion. The hexadecimal colo...           1  \n",
       "2  CSS Codes; Color Preview; Color Schemes; Color...           2  \n",
       "3  Color Hex Color Codes Color-hex gives informat...           3  \n",
       "4  Color Hex Color Codes. Color-hex gives informa...           4  "
      ]
     },
     "execution_count": 107,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test2.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 86,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": [
    "# ans_eval['query_id'] = test[test['passage_id']==0]['query_id']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 108,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": [
    "ans_eval['query_id'] = test2[test2['passage_id']==0]['query_id'].reset_index().drop(['index'],axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 109,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": [
    "ans_eval_2 = ans_eval[columns]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 110,
   "metadata": {
    "hidden": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>query_id</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "      <th>4</th>\n",
       "      <th>5</th>\n",
       "      <th>6</th>\n",
       "      <th>7</th>\n",
       "      <th>8</th>\n",
       "      <th>9</th>\n",
       "      <th>10</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1136966</td>\n",
       "      <td>-3.275982</td>\n",
       "      <td>3.426316</td>\n",
       "      <td>2.783353</td>\n",
       "      <td>-2.347359</td>\n",
       "      <td>-2.684981</td>\n",
       "      <td>2.999565</td>\n",
       "      <td>3.512079</td>\n",
       "      <td>1.558420</td>\n",
       "      <td>1.321732</td>\n",
       "      <td>3.655693</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1136956</td>\n",
       "      <td>-3.037337</td>\n",
       "      <td>-1.468039</td>\n",
       "      <td>-1.505129</td>\n",
       "      <td>-1.187077</td>\n",
       "      <td>0.123792</td>\n",
       "      <td>2.826501</td>\n",
       "      <td>2.452425</td>\n",
       "      <td>-2.953604</td>\n",
       "      <td>1.316775</td>\n",
       "      <td>3.026641</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1136841</td>\n",
       "      <td>0.394103</td>\n",
       "      <td>-3.923445</td>\n",
       "      <td>1.445503</td>\n",
       "      <td>-0.897827</td>\n",
       "      <td>1.486712</td>\n",
       "      <td>-2.480670</td>\n",
       "      <td>1.576904</td>\n",
       "      <td>0.816478</td>\n",
       "      <td>2.613828</td>\n",
       "      <td>-0.920492</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1125025</td>\n",
       "      <td>-4.534754</td>\n",
       "      <td>1.344380</td>\n",
       "      <td>-2.451988</td>\n",
       "      <td>4.039209</td>\n",
       "      <td>-3.753594</td>\n",
       "      <td>-0.442643</td>\n",
       "      <td>-4.627643</td>\n",
       "      <td>-0.441724</td>\n",
       "      <td>1.750053</td>\n",
       "      <td>4.644456</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>86062</td>\n",
       "      <td>0.909422</td>\n",
       "      <td>-0.051245</td>\n",
       "      <td>3.045459</td>\n",
       "      <td>3.725198</td>\n",
       "      <td>2.355503</td>\n",
       "      <td>-0.751920</td>\n",
       "      <td>4.015532</td>\n",
       "      <td>1.262180</td>\n",
       "      <td>2.498722</td>\n",
       "      <td>-0.085766</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   query_id         1         2         3         4         5         6  \\\n",
       "0   1136966 -3.275982  3.426316  2.783353 -2.347359 -2.684981  2.999565   \n",
       "1   1136956 -3.037337 -1.468039 -1.505129 -1.187077  0.123792  2.826501   \n",
       "2   1136841  0.394103 -3.923445  1.445503 -0.897827  1.486712 -2.480670   \n",
       "3   1125025 -4.534754  1.344380 -2.451988  4.039209 -3.753594 -0.442643   \n",
       "4     86062  0.909422 -0.051245  3.045459  3.725198  2.355503 -0.751920   \n",
       "\n",
       "          7         8         9        10  \n",
       "0  3.512079  1.558420  1.321732  3.655693  \n",
       "1  2.452425 -2.953604  1.316775  3.026641  \n",
       "2  1.576904  0.816478  2.613828 -0.920492  \n",
       "3 -4.627643 -0.441724  1.750053  4.644456  \n",
       "4  4.015532  1.262180  2.498722 -0.085766  "
      ]
     },
     "execution_count": 110,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ans_eval_2.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 89,
   "metadata": {
    "hidden": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>query_id</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "      <th>4</th>\n",
       "      <th>5</th>\n",
       "      <th>6</th>\n",
       "      <th>7</th>\n",
       "      <th>8</th>\n",
       "      <th>9</th>\n",
       "      <th>10</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1136966</td>\n",
       "      <td>-3.471546</td>\n",
       "      <td>3.119705</td>\n",
       "      <td>2.748143</td>\n",
       "      <td>-2.061335</td>\n",
       "      <td>-2.470538</td>\n",
       "      <td>3.298495</td>\n",
       "      <td>3.491955</td>\n",
       "      <td>1.573026</td>\n",
       "      <td>1.000348</td>\n",
       "      <td>3.620442</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1136956</td>\n",
       "      <td>-3.726930</td>\n",
       "      <td>-2.489444</td>\n",
       "      <td>-2.292486</td>\n",
       "      <td>-1.507174</td>\n",
       "      <td>-0.444416</td>\n",
       "      <td>2.315248</td>\n",
       "      <td>1.919158</td>\n",
       "      <td>-3.663156</td>\n",
       "      <td>0.951928</td>\n",
       "      <td>2.571643</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1136841</td>\n",
       "      <td>0.069452</td>\n",
       "      <td>-4.416095</td>\n",
       "      <td>1.386632</td>\n",
       "      <td>-0.630060</td>\n",
       "      <td>1.512524</td>\n",
       "      <td>-3.993858</td>\n",
       "      <td>1.339767</td>\n",
       "      <td>0.383412</td>\n",
       "      <td>2.454320</td>\n",
       "      <td>-0.877230</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1125025</td>\n",
       "      <td>-4.738125</td>\n",
       "      <td>0.573662</td>\n",
       "      <td>-2.688248</td>\n",
       "      <td>3.691872</td>\n",
       "      <td>-3.160819</td>\n",
       "      <td>-1.016214</td>\n",
       "      <td>-5.073241</td>\n",
       "      <td>-0.230091</td>\n",
       "      <td>1.103286</td>\n",
       "      <td>4.322522</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>86062</td>\n",
       "      <td>0.369257</td>\n",
       "      <td>-0.616475</td>\n",
       "      <td>2.434665</td>\n",
       "      <td>3.228459</td>\n",
       "      <td>1.249426</td>\n",
       "      <td>-1.635600</td>\n",
       "      <td>3.795533</td>\n",
       "      <td>0.576618</td>\n",
       "      <td>1.744282</td>\n",
       "      <td>-0.599759</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   query_id         1         2         3         4         5         6  \\\n",
       "0   1136966 -3.471546  3.119705  2.748143 -2.061335 -2.470538  3.298495   \n",
       "1   1136956 -3.726930 -2.489444 -2.292486 -1.507174 -0.444416  2.315248   \n",
       "2   1136841  0.069452 -4.416095  1.386632 -0.630060  1.512524 -3.993858   \n",
       "3   1125025 -4.738125  0.573662 -2.688248  3.691872 -3.160819 -1.016214   \n",
       "4     86062  0.369257 -0.616475  2.434665  3.228459  1.249426 -1.635600   \n",
       "\n",
       "          7         8         9        10  \n",
       "0  3.491955  1.573026  1.000348  3.620442  \n",
       "1  1.919158 -3.663156  0.951928  2.571643  \n",
       "2  1.339767  0.383412  2.454320 -0.877230  \n",
       "3 -5.073241 -0.230091  1.103286  4.322522  \n",
       "4  3.795533  0.576618  1.744282 -0.599759  "
      ]
     },
     "execution_count": 89,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ans_eval_2.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": [
    "# ans3 = pd.read_csv('./search_data/answer_236_mixed_Epoch3_.tsv',names=columns,delimiter='\\t')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 90,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": [
    "# ans3.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 111,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": [
    "ans_eval_2.to_csv('./search_data/answer2_290_Epoch2_160.tsv',sep='\\t',header=False,index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "conda_pytorch_p36",
   "language": "python",
   "name": "conda_pytorch_p36"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}