beerEval.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-02-25T13:16:01.199952Z",
     "start_time": "2020-02-25T13:16:01.181649Z"
    }
   },
   "outputs": [],
   "source": [
    "%load_ext autoreload\n",
    "%autoreload 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-02-25T13:16:01.746177Z",
     "start_time": "2020-02-25T13:16:01.382957Z"
    }
   },
   "outputs": [],
   "source": [
    "%matplotlib inline\n",
    "%config InlineBackend.figure_format = 'retina'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-02-25T13:16:02.848124Z",
     "start_time": "2020-02-25T13:16:01.748303Z"
    }
   },
   "outputs": [],
   "source": [
    "from utils.attacker import attackDatasetEditDist,changeRandomCharacters\n",
    "import pandas as pd\n",
    "import deepmatcher as dm\n",
    "import numpy as np\n",
    "from utils.deepmatcher_utils import wrapDm,getF1PrecisionRecall,getMeanConfidenceAndVariance,getTruePositiveNegative\n",
    "from tqdm import tqdm\n",
    "from sklearn.utils import shuffle\n",
    "from collections import OrderedDict"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-02-25T13:16:02.867640Z",
     "start_time": "2020-02-25T13:16:02.850070Z"
    }
   },
   "outputs": [],
   "source": [
    "##given a perturbation model and number of perturbation calculate avg f1,precision and recall \n",
    "def getAvgF1PrecisionRecall(testpath,model,nperturbation,editdist=1):\n",
    "    avgf1,avgrecall,avgprecision = 0,0,0\n",
    "    for i in tqdm(range(nperturbation)):\n",
    "        test_df = pd.read_csv(testpath)\n",
    "        original_labels = test_df.label.values\n",
    "        attacked_test = attackDatasetEditDist(test_df,['id','label'],editdist=editdist)\n",
    "        attack_preds = wrapDm(attacked_test,model)\n",
    "        f1,prec,recall = getF1PrecisionRecall(original_labels,attack_preds)\n",
    "        avgf1 += f1\n",
    "        avgprecision += prec\n",
    "        avgrecall += recall\n",
    "    return (avgf1/nperturbation,avgprecision/nperturbation,avgrecall/nperturbation)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-02-25T13:16:02.887435Z",
     "start_time": "2020-02-25T13:16:02.869013Z"
    }
   },
   "outputs": [],
   "source": [
    "def getAdversarialTrainingSet(train_df,dimension,editdist):\n",
    "    shuffled_train = shuffle(train_df)\n",
    "    samples_toalter = shuffled_train.iloc[:dimension].copy()\n",
    "    unaltered = shuffled_train.iloc[dimension:].copy()\n",
    "    attacked_train = attackDatasetEditDist(samples_toalter,['id','label'],editdist=1)\n",
    "    newTrain = pd.concat([attacked_train,unaltered])\n",
    "    return newTrain"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-02-25T13:16:02.905122Z",
     "start_time": "2020-02-25T13:16:02.889025Z"
    }
   },
   "outputs": [],
   "source": [
    "def containTokens(attributes,tokensToFind):\n",
    "    matchingAttr = 0\n",
    "    for attr in attributes:\n",
    "        tokens = list(map(lambda s:s.lower(),attr.split()))\n",
    "        if set(tokensToFind).issubset(set(tokens)):\n",
    "            matchingAttr +=1\n",
    "    return matchingAttr"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-02-25T13:16:03.044422Z",
     "start_time": "2020-02-25T13:16:03.024656Z"
    }
   },
   "outputs": [],
   "source": [
    "def countTokens(attributes):\n",
    "    allTokens = {}\n",
    "    for attr in attributes:\n",
    "        tokens = list(map(lambda s:s.lower(),attr.split()))\n",
    "        for tok in tokens:\n",
    "            if tok in allTokens:\n",
    "                allTokens[tok] += 1\n",
    "            else:\n",
    "                allTokens[tok] = 1\n",
    "    allOrderedTokens = OrderedDict(sorted(allTokens.items(), key=lambda t: t[1],reverse=True))\n",
    "    return allOrderedTokens"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Change edit distance"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-02-25T13:16:31.358293Z",
     "start_time": "2020-02-25T13:16:31.327953Z"
    }
   },
   "outputs": [],
   "source": [
    "DATASET_DIR='datasets/Structured/Beer/'"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Evaluate standard model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-02-25T09:20:26.202431Z",
     "start_time": "2020-02-25T09:20:20.830429Z"
    }
   },
   "outputs": [],
   "source": [
    "standard_model = dm.MatchingModel(attr_summarizer='hybrid')\n",
    "standard_model.load_state('models/beer_hybrid.pth')\n",
    "standard_model = standard_model.to('cuda')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-02-09T14:47:21.734208Z",
     "start_time": "2020-02-09T14:46:36.465067Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 20/20 [00:45<00:00,  2.26s/it]\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(0.6018780920046372, 0.5742302039727692, 0.6370370370370371)"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "avgf1,avgprecision,avgrecall = getAvgF1PrecisionRecall('datasets/Structured/itunes-amazon/merged_test.csv',\n",
    "                                                       itunesamazon_mod,20,editdist=1)\n",
    "avgf1,avgprecision,avgrecall"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-02-07T16:25:37.530261Z",
     "start_time": "2020-02-07T16:25:07.098842Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 20/20 [00:30<00:00,  1.52s/it]\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(0.5566625559259786, 0.5380849305572994, 0.5814814814814815)"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "avgf1,avgprecision,avgrecall = getAvgF1PrecisionRecall('datasets/Structured/itunes-amazon/merged_test.csv',\n",
    "                                                       itunesamazon_mod,20,editdist=2)\n",
    "avgf1,avgprecision,avgrecall"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-02-09T14:48:39.583230Z",
     "start_time": "2020-02-09T14:48:37.913244Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(0.8026398884457185, 0.014892625030903646)"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_df = pd.read_csv('datasets/Structured/itunes-amazon/merged_test.csv')\n",
    "getMeanConfidenceAndVariance(itunesamazon_mod,test_df)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Retrain model with 10% perturbed training data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-02-09T14:49:57.172927Z",
     "start_time": "2020-02-09T14:49:57.105512Z"
    }
   },
   "outputs": [],
   "source": [
    "train_df = pd.read_csv(DATASET_DIR+'merged_train.csv')\n",
    "adversarialTrain_10 = getAdversarialTrainingSet(train_df,int(len(train_df)/10),editdist=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-02-09T14:50:00.189059Z",
     "start_time": "2020-02-09T14:50:00.146492Z"
    }
   },
   "outputs": [],
   "source": [
    "adversarialTrain_10.to_csv('datasets/Structured/itunes-amazon/attacked_train10.csv',index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-02-09T14:50:05.971047Z",
     "start_time": "2020-02-09T14:50:03.333386Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING:deepmatcher.data.dataset:Rebuilding data cache because: ['Data file list has changed.', 'One or more data files have been modified.']\n",
      "\n",
      "Reading and processing data from \"datasets/Structured/itunes-amazon/attacked_train10.csv\"\n",
      "0% [############################# ] 100% | ETA: 00:00:00\n",
      "Reading and processing data from \"datasets/Structured/itunes-amazon/merged_valid.csv\"\n",
      "0% [############################# ] 100% | ETA: 00:00:00\n",
      "Reading and processing data from \"datasets/Structured/itunes-amazon/merged_test.csv\"\n",
      "0% [############################# ] 100% | ETA: 00:00:00\n",
      "Building vocabulary\n",
      "0% [#] 100% | ETA: 00:00:00\n",
      "Total time elapsed: 00:00:00\n",
      "\n",
      "Computing principal components\n",
      "0% [#] 100% | ETA: 00:00:00\n",
      "Total time elapsed: 00:00:00\n"
     ]
    }
   ],
   "source": [
    "train,validation,test = dm.data.process('datasets/Structured/itunes-amazon/',train='attacked_train10.csv',\n",
    "                                       validation='merged_valid.csv',test='merged_test.csv',\n",
    "                                       left_prefix='ltable_',right_prefix='rtable_',cache='editdist.pth')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-02-09T14:53:47.035491Z",
     "start_time": "2020-02-09T14:50:15.255025Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "* Number of trainable parameters: 17757810\n",
      "===>  TRAIN Epoch 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/nvidia/anaconda3/envs/torchenv/lib/python3.6/site-packages/torch/nn/functional.py:1932: UserWarning: reduction: 'mean' divides the total loss by both the batch size and the support size.'batchmean' divides only by the batch size, and aligns with the KL div math definition.'mean' will be changed to behave the same as 'batchmean' in the next major release.\n",
      "  warnings.warn(\"reduction: 'mean' divides the total loss by both the batch size and the support size.\"\n",
      "0% [██] 100% | ETA: 00:00:00\n",
      "Total time elapsed: 00:00:05\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Finished Epoch 1 || Run Time:    4.8 | Load Time:    2.1 || F1:  36.87 | Prec:  23.57 | Rec:  84.62 || Ex/s:  47.03\n",
      "\n",
      "===>  EVAL Epoch 1\n",
      "Finished Epoch 1 || Run Time:    0.6 | Load Time:    0.7 || F1:  41.54 | Prec:  26.21 | Rec: 100.00 || Ex/s:  85.24\n",
      "\n",
      "* Best F1: tensor(41.5385, device='cuda:0')\n",
      "Saving best model...\n",
      "Done.\n",
      "---------------------\n",
      "\n",
      "===>  TRAIN Epoch 2\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "0% [██] 100% | ETA: 00:00:00\n",
      "Total time elapsed: 00:00:06\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Finished Epoch 2 || Run Time:    4.8 | Load Time:    2.2 || F1:  41.62 | Prec:  26.37 | Rec:  98.72 || Ex/s:  45.71\n",
      "\n",
      "===>  EVAL Epoch 2\n",
      "Finished Epoch 2 || Run Time:    0.6 | Load Time:    0.7 || F1:  57.89 | Prec:  44.90 | Rec:  81.48 || Ex/s:  84.95\n",
      "\n",
      "* Best F1: tensor(57.8947, device='cuda:0')\n",
      "Saving best model...\n",
      "Done.\n",
      "---------------------\n",
      "\n",
      "===>  TRAIN Epoch 3\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "0% [██] 100% | ETA: 00:00:00\n",
      "Total time elapsed: 00:00:06\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Finished Epoch 3 || Run Time:    4.9 | Load Time:    2.2 || F1:  61.67 | Prec:  46.98 | Rec:  89.74 || Ex/s:  45.33\n",
      "\n",
      "===>  EVAL Epoch 3\n",
      "Finished Epoch 3 || Run Time:    0.6 | Load Time:    0.7 || F1:  62.69 | Prec:  52.50 | Rec:  77.78 || Ex/s:  84.23\n",
      "\n",
      "* Best F1: tensor(62.6866, device='cuda:0')\n",
      "Saving best model...\n",
      "Done.\n",
      "---------------------\n",
      "\n",
      "===>  TRAIN Epoch 4\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "0% [██] 100% | ETA: 00:00:00\n",
      "Total time elapsed: 00:00:06\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Finished Epoch 4 || Run Time:    4.9 | Load Time:    2.2 || F1:  66.05 | Prec:  51.82 | Rec:  91.03 || Ex/s:  45.64\n",
      "\n",
      "===>  EVAL Epoch 4\n",
      "Finished Epoch 4 || Run Time:    0.6 | Load Time:    0.7 || F1:  68.97 | Prec:  64.52 | Rec:  74.07 || Ex/s:  85.64\n",
      "\n",
      "* Best F1: tensor(68.9655, device='cuda:0')\n",
      "Saving best model...\n",
      "Done.\n",
      "---------------------\n",
      "\n",
      "===>  TRAIN Epoch 5\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "0% [██] 100% | ETA: 00:00:00\n",
      "Total time elapsed: 00:00:05\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Finished Epoch 5 || Run Time:    4.9 | Load Time:    2.1 || F1:  72.00 | Prec:  59.02 | Rec:  92.31 || Ex/s:  46.45\n",
      "\n",
      "===>  EVAL Epoch 5\n",
      "Finished Epoch 5 || Run Time:    0.6 | Load Time:    0.7 || F1:  66.67 | Prec:  58.33 | Rec:  77.78 || Ex/s:  81.93\n",
      "\n",
      "---------------------\n",
      "\n",
      "===>  TRAIN Epoch 6\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "0% [██] 100% | ETA: 00:00:00\n",
      "Total time elapsed: 00:00:06\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Finished Epoch 6 || Run Time:    4.9 | Load Time:    2.2 || F1:  78.12 | Prec:  65.79 | Rec:  96.15 || Ex/s:  44.91\n",
      "\n",
      "===>  EVAL Epoch 6\n",
      "Finished Epoch 6 || Run Time:    0.6 | Load Time:    0.7 || F1:  77.19 | Prec:  73.33 | Rec:  81.48 || Ex/s:  85.99\n",
      "\n",
      "* Best F1: tensor(77.1930, device='cuda:0')\n",
      "Saving best model...\n",
      "Done.\n",
      "---------------------\n",
      "\n",
      "===>  TRAIN Epoch 7\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "0% [██] 100% | ETA: 00:00:00\n",
      "Total time elapsed: 00:00:06\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Finished Epoch 7 || Run Time:    5.0 | Load Time:    2.3 || F1:  85.39 | Prec:  76.00 | Rec:  97.44 || Ex/s:  44.29\n",
      "\n",
      "===>  EVAL Epoch 7\n",
      "Finished Epoch 7 || Run Time:    0.6 | Load Time:    0.7 || F1:  83.33 | Prec:  75.76 | Rec:  92.59 || Ex/s:  86.20\n",
      "\n",
      "* Best F1: tensor(83.3333, device='cuda:0')\n",
      "Saving best model...\n",
      "Done.\n",
      "---------------------\n",
      "\n",
      "===>  TRAIN Epoch 8\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "0% [██] 100% | ETA: 00:00:00\n",
      "Total time elapsed: 00:00:06\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Finished Epoch 8 || Run Time:    4.9 | Load Time:    2.2 || F1:  93.41 | Prec:  87.64 | Rec: 100.00 || Ex/s:  45.50\n",
      "\n",
      "===>  EVAL Epoch 8\n",
      "Finished Epoch 8 || Run Time:    0.6 | Load Time:    0.7 || F1:  85.25 | Prec:  76.47 | Rec:  96.30 || Ex/s:  85.02\n",
      "\n",
      "* Best F1: tensor(85.2459, device='cuda:0')\n",
      "Saving best model...\n",
      "Done.\n",
      "---------------------\n",
      "\n",
      "===>  TRAIN Epoch 9\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "0% [██] 100% | ETA: 00:00:00\n",
      "Total time elapsed: 00:00:06\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Finished Epoch 9 || Run Time:    4.8 | Load Time:    2.2 || F1:  97.50 | Prec:  95.12 | Rec: 100.00 || Ex/s:  45.76\n",
      "\n",
      "===>  EVAL Epoch 9\n",
      "Finished Epoch 9 || Run Time:    0.6 | Load Time:    0.7 || F1:  82.54 | Prec:  72.22 | Rec:  96.30 || Ex/s:  85.33\n",
      "\n",
      "---------------------\n",
      "\n",
      "===>  TRAIN Epoch 10\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "0% [██] 100% | ETA: 00:00:00\n",
      "Total time elapsed: 00:00:06\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Finished Epoch 10 || Run Time:    4.9 | Load Time:    2.1 || F1:  97.50 | Prec:  95.12 | Rec: 100.00 || Ex/s:  45.78\n",
      "\n",
      "===>  EVAL Epoch 10\n",
      "Finished Epoch 10 || Run Time:    0.6 | Load Time:    0.7 || F1:  83.87 | Prec:  74.29 | Rec:  96.30 || Ex/s:  84.53\n",
      "\n",
      "---------------------\n",
      "\n",
      "===>  TRAIN Epoch 11\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "0% [██] 100% | ETA: 00:00:00\n",
      "Total time elapsed: 00:00:06\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Finished Epoch 11 || Run Time:    4.8 | Load Time:    2.1 || F1:  98.73 | Prec:  97.50 | Rec: 100.00 || Ex/s:  46.00\n",
      "\n",
      "===>  EVAL Epoch 11\n",
      "Finished Epoch 11 || Run Time:    0.6 | Load Time:    0.7 || F1:  82.54 | Prec:  72.22 | Rec:  96.30 || Ex/s:  85.29\n",
      "\n",
      "---------------------\n",
      "\n",
      "===>  TRAIN Epoch 12\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "0% [██] 100% | ETA: 00:00:00\n",
      "Total time elapsed: 00:00:05\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Finished Epoch 12 || Run Time:    4.9 | Load Time:    2.1 || F1:  98.73 | Prec:  97.50 | Rec: 100.00 || Ex/s:  46.26\n",
      "\n",
      "===>  EVAL Epoch 12\n",
      "Finished Epoch 12 || Run Time:    0.6 | Load Time:    0.7 || F1:  83.87 | Prec:  74.29 | Rec:  96.30 || Ex/s:  84.00\n",
      "\n",
      "---------------------\n",
      "\n",
      "===>  TRAIN Epoch 13\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "0% [██] 100% | ETA: 00:00:00\n",
      "Total time elapsed: 00:00:06\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Finished Epoch 13 || Run Time:    4.9 | Load Time:    2.1 || F1:  99.36 | Prec:  98.73 | Rec: 100.00 || Ex/s:  45.78\n",
      "\n",
      "===>  EVAL Epoch 13\n",
      "Finished Epoch 13 || Run Time:    0.6 | Load Time:    0.7 || F1:  83.87 | Prec:  74.29 | Rec:  96.30 || Ex/s:  84.85\n",
      "\n",
      "---------------------\n",
      "\n",
      "===>  TRAIN Epoch 14\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "0% [██] 100% | ETA: 00:00:00\n",
      "Total time elapsed: 00:00:05\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Finished Epoch 14 || Run Time:    4.4 | Load Time:    1.9 || F1:  99.36 | Prec:  98.73 | Rec: 100.00 || Ex/s:  50.71\n",
      "\n",
      "===>  EVAL Epoch 14\n",
      "Finished Epoch 14 || Run Time:    0.6 | Load Time:    0.6 || F1:  83.87 | Prec:  74.29 | Rec:  96.30 || Ex/s:  92.17\n",
      "\n",
      "---------------------\n",
      "\n",
      "===>  TRAIN Epoch 15\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "0% [██] 100% | ETA: 00:00:00\n",
      "Total time elapsed: 00:00:05\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Finished Epoch 15 || Run Time:    4.8 | Load Time:    2.1 || F1: 100.00 | Prec: 100.00 | Rec: 100.00 || Ex/s:  46.74\n",
      "\n",
      "===>  EVAL Epoch 15\n",
      "Finished Epoch 15 || Run Time:    0.6 | Load Time:    0.7 || F1:  83.87 | Prec:  74.29 | Rec:  96.30 || Ex/s:  84.14\n",
      "\n",
      "---------------------\n",
      "\n",
      "===>  TRAIN Epoch 16\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "0% [██] 100% | ETA: 00:00:00\n",
      "Total time elapsed: 00:00:05\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Finished Epoch 16 || Run Time:    4.8 | Load Time:    2.1 || F1: 100.00 | Prec: 100.00 | Rec: 100.00 || Ex/s:  46.89\n",
      "\n",
      "===>  EVAL Epoch 16\n",
      "Finished Epoch 16 || Run Time:    0.6 | Load Time:    0.7 || F1:  83.87 | Prec:  74.29 | Rec:  96.30 || Ex/s:  85.57\n",
      "\n",
      "---------------------\n",
      "\n",
      "===>  TRAIN Epoch 17\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "0% [██] 100% | ETA: 00:00:00\n",
      "Total time elapsed: 00:00:05\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Finished Epoch 17 || Run Time:    4.7 | Load Time:    2.0 || F1: 100.00 | Prec: 100.00 | Rec: 100.00 || Ex/s:  47.53\n",
      "\n",
      "===>  EVAL Epoch 17\n",
      "Finished Epoch 17 || Run Time:    0.6 | Load Time:    0.7 || F1:  83.87 | Prec:  74.29 | Rec:  96.30 || Ex/s:  85.08\n",
      "\n",
      "---------------------\n",
      "\n",
      "===>  TRAIN Epoch 18\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "0% [██] 100% | ETA: 00:00:00\n",
      "Total time elapsed: 00:00:06\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Finished Epoch 18 || Run Time:    4.9 | Load Time:    2.1 || F1: 100.00 | Prec: 100.00 | Rec: 100.00 || Ex/s:  45.83\n",
      "\n",
      "===>  EVAL Epoch 18\n",
      "Finished Epoch 18 || Run Time:    0.6 | Load Time:    0.7 || F1:  83.87 | Prec:  74.29 | Rec:  96.30 || Ex/s:  84.37\n",
      "\n",
      "---------------------\n",
      "\n",
      "===>  TRAIN Epoch 19\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "0% [██] 100% | ETA: 00:00:00\n",
      "Total time elapsed: 00:00:06\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Finished Epoch 19 || Run Time:    4.8 | Load Time:    2.1 || F1: 100.00 | Prec: 100.00 | Rec: 100.00 || Ex/s:  45.98\n",
      "\n",
      "===>  EVAL Epoch 19\n",
      "Finished Epoch 19 || Run Time:    0.6 | Load Time:    0.7 || F1:  83.87 | Prec:  74.29 | Rec:  96.30 || Ex/s:  85.03\n",
      "\n",
      "---------------------\n",
      "\n",
      "===>  TRAIN Epoch 20\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "0% [██] 100% | ETA: 00:00:00\n",
      "Total time elapsed: 00:00:06\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Finished Epoch 20 || Run Time:    4.8 | Load Time:    2.2 || F1: 100.00 | Prec: 100.00 | Rec: 100.00 || Ex/s:  45.91\n",
      "\n",
      "===>  EVAL Epoch 20\n",
      "Finished Epoch 20 || Run Time:    0.6 | Load Time:    0.7 || F1:  83.87 | Prec:  74.29 | Rec:  96.30 || Ex/s:  85.53\n",
      "\n",
      "---------------------\n",
      "\n",
      "===>  TRAIN Epoch 21\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "0% [██] 100% | ETA: 00:00:00\n",
      "Total time elapsed: 00:00:06\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Finished Epoch 21 || Run Time:    4.9 | Load Time:    2.1 || F1: 100.00 | Prec: 100.00 | Rec: 100.00 || Ex/s:  45.63\n",
      "\n",
      "===>  EVAL Epoch 21\n",
      "Finished Epoch 21 || Run Time:    0.6 | Load Time:    0.7 || F1:  83.87 | Prec:  74.29 | Rec:  96.30 || Ex/s:  86.66\n",
      "\n",
      "---------------------\n",
      "\n",
      "===>  TRAIN Epoch 22\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "0% [██] 100% | ETA: 00:00:00\n",
      "Total time elapsed: 00:00:05\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Finished Epoch 22 || Run Time:    4.8 | Load Time:    2.1 || F1: 100.00 | Prec: 100.00 | Rec: 100.00 || Ex/s:  47.09\n",
      "\n",
      "===>  EVAL Epoch 22\n",
      "Finished Epoch 22 || Run Time:    0.6 | Load Time:    0.7 || F1:  83.87 | Prec:  74.29 | Rec:  96.30 || Ex/s:  86.68\n",
      "\n",
      "---------------------\n",
      "\n",
      "===>  TRAIN Epoch 23\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "0% [██] 100% | ETA: 00:00:00\n",
      "Total time elapsed: 00:00:05\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Finished Epoch 23 || Run Time:    4.9 | Load Time:    2.1 || F1: 100.00 | Prec: 100.00 | Rec: 100.00 || Ex/s:  46.04\n",
      "\n",
      "===>  EVAL Epoch 23\n",
      "Finished Epoch 23 || Run Time:    0.6 | Load Time:    0.7 || F1:  83.87 | Prec:  74.29 | Rec:  96.30 || Ex/s:  86.00\n",
      "\n",
      "---------------------\n",
      "\n",
      "===>  TRAIN Epoch 24\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "0% [██] 100% | ETA: 00:00:00\n",
      "Total time elapsed: 00:00:05\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Finished Epoch 24 || Run Time:    4.6 | Load Time:    1.9 || F1: 100.00 | Prec: 100.00 | Rec: 100.00 || Ex/s:  49.80\n",
      "\n",
      "===>  EVAL Epoch 24\n",
      "Finished Epoch 24 || Run Time:    0.6 | Load Time:    0.7 || F1:  83.87 | Prec:  74.29 | Rec:  96.30 || Ex/s:  85.96\n",
      "\n",
      "---------------------\n",
      "\n",
      "===>  TRAIN Epoch 25\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "0% [██] 100% | ETA: 00:00:00\n",
      "Total time elapsed: 00:00:06\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Finished Epoch 25 || Run Time:    4.9 | Load Time:    2.1 || F1: 100.00 | Prec: 100.00 | Rec: 100.00 || Ex/s:  45.95\n",
      "\n",
      "===>  EVAL Epoch 25\n",
      "Finished Epoch 25 || Run Time:    0.6 | Load Time:    0.7 || F1:  83.87 | Prec:  74.29 | Rec:  96.30 || Ex/s:  86.54\n",
      "\n",
      "---------------------\n",
      "\n",
      "Loading best model...\n",
      "Training done.\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "tensor(85.2459, device='cuda:0')"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "att_model = dm.MatchingModel(attr_summarizer='hybrid')\n",
    "att_model.run_train(train,validation,epochs=25,pos_neg_ratio=5,\n",
    "                    best_save_path='models/itunes_amazon_hybrid_att10.pth')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-02-09T14:54:13.396526Z",
     "start_time": "2020-02-09T14:54:11.980086Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "===>  EVAL Epoch 8\n",
      "Finished Epoch 8 || Run Time:    0.7 | Load Time:    0.7 || F1:  85.25 | Prec:  76.47 | Rec:  96.30 || Ex/s:  81.09\n",
      "\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "tensor(85.2459, device='cuda:0')"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "att_model.run_eval(test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-02-09T14:54:30.493503Z",
     "start_time": "2020-02-09T14:54:28.652091Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(0.7836888705235009, 0.016787313181924515)"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "getMeanConfidenceAndVariance(att_model,test_df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-02-09T14:55:25.947493Z",
     "start_time": "2020-02-09T14:54:51.408210Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 20/20 [00:34<00:00,  1.73s/it]\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(0.6170878502848114, 0.4939288327705847, 0.825925925925926)"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "avgf1,avgprecision,avgrecall = getAvgF1PrecisionRecall('datasets/Structured/itunes-amazon/merged_test.csv',\n",
    "                                                       att_model,20,editdist=1)\n",
    "avgf1,avgprecision,avgrecall"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-02-09T14:56:32.268135Z",
     "start_time": "2020-02-09T14:55:56.558767Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 20/20 [00:35<00:00,  1.78s/it]\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(0.5795080719900618, 0.4545109863934491, 0.8037037037037038)"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "avgf1,avgprecision,avgrecall = getAvgF1PrecisionRecall('datasets/Structured/itunes-amazon/merged_test.csv',\n",
    "                                                       att_model,20,editdist=2)\n",
    "avgf1,avgprecision,avgrecall"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-02-07T16:36:06.592738Z",
     "start_time": "2020-02-07T16:36:06.576719Z"
    }
   },
   "source": [
    "## Retrain model with 20% perturbed training samples"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-02-09T14:56:50.761335Z",
     "start_time": "2020-02-09T14:56:50.689408Z"
    }
   },
   "outputs": [],
   "source": [
    "train_df = pd.read_csv(DATASET_DIR+'merged_train.csv')\n",
    "adversarialTrain_20 = getAdversarialTrainingSet(train_df,int(len(train_df)/5),editdist=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-02-09T14:56:51.833717Z",
     "start_time": "2020-02-09T14:56:51.792458Z"
    }
   },
   "outputs": [],
   "source": [
    "adversarialTrain_20.to_csv(DATASET_DIR+'adversarial_train20.csv',index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-02-09T14:56:55.221848Z",
     "start_time": "2020-02-09T14:56:53.080254Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING:deepmatcher.data.dataset:Rebuilding data cache because: ['Data file list has changed.', 'One or more data files have been modified.']\n",
      "\n",
      "Reading and processing data from \"datasets/Structured/itunes-amazon/adversarial_train20.csv\"\n",
      "0% [############################# ] 100% | ETA: 00:00:00\n",
      "Reading and processing data from \"datasets/Structured/itunes-amazon/merged_valid.csv\"\n",
      "0% [############################# ] 100% | ETA: 00:00:00\n",
      "Reading and processing data from \"datasets/Structured/itunes-amazon/merged_test.csv\"\n",
      "0% [############################# ] 100% | ETA: 00:00:00\n",
      "Building vocabulary\n",
      "0% [#] 100% | ETA: 00:00:00\n",
      "Total time elapsed: 00:00:00\n",
      "\n",
      "Computing principal components\n",
      "0% [#] 100% | ETA: 00:00:00\n",
      "Total time elapsed: 00:00:00\n"
     ]
    }
   ],
   "source": [
    "train,validation,test = dm.data.process('datasets/Structured/itunes-amazon/',train='adversarial_train20.csv',\n",
    "                                       validation='merged_valid.csv',test='merged_test.csv',\n",
    "                                       left_prefix='ltable_',right_prefix='rtable_',cache='editdist.pth')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-02-09T15:00:31.721923Z",
     "start_time": "2020-02-09T14:57:05.075283Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "* Number of trainable parameters: 17757810\n",
      "===>  TRAIN Epoch 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/nvidia/anaconda3/envs/torchenv/lib/python3.6/site-packages/torch/nn/functional.py:1932: UserWarning: reduction: 'mean' divides the total loss by both the batch size and the support size.'batchmean' divides only by the batch size, and aligns with the KL div math definition.'mean' will be changed to behave the same as 'batchmean' in the next major release.\n",
      "  warnings.warn(\"reduction: 'mean' divides the total loss by both the batch size and the support size.\"\n",
      "0% [██] 100% | ETA: 00:00:00\n",
      "Total time elapsed: 00:00:05\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Finished Epoch 1 || Run Time:    4.7 | Load Time:    2.1 || F1:  39.20 | Prec:  24.38 | Rec: 100.00 || Ex/s:  47.06\n",
      "\n",
      "===>  EVAL Epoch 1\n",
      "Finished Epoch 1 || Run Time:    0.6 | Load Time:    0.7 || F1:  49.52 | Prec:  33.33 | Rec:  96.30 || Ex/s:  82.09\n",
      "\n",
      "* Best F1: tensor(49.5238, device='cuda:0')\n",
      "Saving best model...\n",
      "Done.\n",
      "---------------------\n",
      "\n",
      "===>  TRAIN Epoch 2\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "0% [██] 100% | ETA: 00:00:00\n",
      "Total time elapsed: 00:00:05\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Finished Epoch 2 || Run Time:    4.6 | Load Time:    2.0 || F1:  50.17 | Prec:  33.94 | Rec:  96.15 || Ex/s:  48.66\n",
      "\n",
      "===>  EVAL Epoch 2\n",
      "Finished Epoch 2 || Run Time:    0.6 | Load Time:    0.6 || F1:  55.56 | Prec:  44.44 | Rec:  74.07 || Ex/s:  90.92\n",
      "\n",
      "* Best F1: tensor(55.5556, device='cuda:0')\n",
      "Saving best model...\n",
      "Done.\n",
      "---------------------\n",
      "\n",
      "===>  TRAIN Epoch 3\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "0% [██] 100% | ETA: 00:00:00\n",
      "Total time elapsed: 00:00:05\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Finished Epoch 3 || Run Time:    4.5 | Load Time:    1.9 || F1:  62.16 | Prec:  47.92 | Rec:  88.46 || Ex/s:  49.80\n",
      "\n",
      "===>  EVAL Epoch 3\n",
      "Finished Epoch 3 || Run Time:    0.6 | Load Time:    0.6 || F1:  59.70 | Prec:  50.00 | Rec:  74.07 || Ex/s:  92.08\n",
      "\n",
      "* Best F1: tensor(59.7015, device='cuda:0')\n",
      "Saving best model...\n",
      "Done.\n",
      "---------------------\n",
      "\n",
      "===>  TRAIN Epoch 4\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "0% [██] 100% | ETA: 00:00:00\n",
      "Total time elapsed: 00:00:06\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Finished Epoch 4 || Run Time:    5.0 | Load Time:    2.2 || F1:  70.59 | Prec:  57.14 | Rec:  92.31 || Ex/s:  44.36\n",
      "\n",
      "===>  EVAL Epoch 4\n",
      "Finished Epoch 4 || Run Time:    0.6 | Load Time:    0.7 || F1:  61.54 | Prec:  52.63 | Rec:  74.07 || Ex/s:  84.55\n",
      "\n",
      "* Best F1: tensor(61.5385, device='cuda:0')\n",
      "Saving best model...\n",
      "Done.\n",
      "---------------------\n",
      "\n",
      "===>  TRAIN Epoch 5\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "0% [██] 100% | ETA: 00:00:00\n",
      "Total time elapsed: 00:00:05\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Finished Epoch 5 || Run Time:    4.7 | Load Time:    2.0 || F1:  73.00 | Prec:  59.84 | Rec:  93.59 || Ex/s:  48.31\n",
      "\n",
      "===>  EVAL Epoch 5\n",
      "Finished Epoch 5 || Run Time:    0.6 | Load Time:    0.7 || F1:  66.67 | Prec:  60.61 | Rec:  74.07 || Ex/s:  85.20\n",
      "\n",
      "* Best F1: tensor(66.6667, device='cuda:0')\n",
      "Saving best model...\n",
      "Done.\n",
      "---------------------\n",
      "\n",
      "===>  TRAIN Epoch 6\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "0% [██] 100% | ETA: 00:00:00\n",
      "Total time elapsed: 00:00:06\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Finished Epoch 6 || Run Time:    4.9 | Load Time:    2.2 || F1:  75.26 | Prec:  62.93 | Rec:  93.59 || Ex/s:  44.95\n",
      "\n",
      "===>  EVAL Epoch 6\n",
      "Finished Epoch 6 || Run Time:    0.7 | Load Time:    0.7 || F1:  72.41 | Prec:  67.74 | Rec:  77.78 || Ex/s:  81.44\n",
      "\n",
      "* Best F1: tensor(72.4138, device='cuda:0')\n",
      "Saving best model...\n",
      "Done.\n",
      "---------------------\n",
      "\n",
      "===>  TRAIN Epoch 7\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "0% [██] 100% | ETA: 00:00:00\n",
      "Total time elapsed: 00:00:05\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Finished Epoch 7 || Run Time:    4.6 | Load Time:    2.0 || F1:  81.72 | Prec:  70.37 | Rec:  97.44 || Ex/s:  48.69\n",
      "\n",
      "===>  EVAL Epoch 7\n",
      "Finished Epoch 7 || Run Time:    0.6 | Load Time:    0.7 || F1:  76.36 | Prec:  75.00 | Rec:  77.78 || Ex/s:  84.80\n",
      "\n",
      "* Best F1: tensor(76.3636, device='cuda:0')\n",
      "Saving best model...\n",
      "Done.\n",
      "---------------------\n",
      "\n",
      "===>  TRAIN Epoch 8\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "0% [██] 100% | ETA: 00:00:00\n",
      "Total time elapsed: 00:00:06\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Finished Epoch 8 || Run Time:    4.7 | Load Time:    2.2 || F1:  88.51 | Prec:  80.21 | Rec:  98.72 || Ex/s:  46.44\n",
      "\n",
      "===>  EVAL Epoch 8\n",
      "Finished Epoch 8 || Run Time:    0.6 | Load Time:    0.6 || F1:  80.70 | Prec:  76.67 | Rec:  85.19 || Ex/s:  94.01\n",
      "\n",
      "* Best F1: tensor(80.7018, device='cuda:0')\n",
      "Saving best model...\n",
      "Done.\n",
      "---------------------\n",
      "\n",
      "===>  TRAIN Epoch 9\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "0% [██] 100% | ETA: 00:00:00\n",
      "Total time elapsed: 00:00:05\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Finished Epoch 9 || Run Time:    4.4 | Load Time:    1.9 || F1:  94.55 | Prec:  89.66 | Rec: 100.00 || Ex/s:  50.78\n",
      "\n",
      "===>  EVAL Epoch 9\n",
      "Finished Epoch 9 || Run Time:    0.6 | Load Time:    0.6 || F1:  82.14 | Prec:  79.31 | Rec:  85.19 || Ex/s:  92.71\n",
      "\n",
      "* Best F1: tensor(82.1429, device='cuda:0')\n",
      "Saving best model...\n",
      "Done.\n",
      "---------------------\n",
      "\n",
      "===>  TRAIN Epoch 10\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "0% [██] 100% | ETA: 00:00:00\n",
      "Total time elapsed: 00:00:05\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Finished Epoch 10 || Run Time:    4.4 | Load Time:    1.9 || F1:  96.30 | Prec:  92.86 | Rec: 100.00 || Ex/s:  50.46\n",
      "\n",
      "===>  EVAL Epoch 10\n",
      "Finished Epoch 10 || Run Time:    0.6 | Load Time:    0.6 || F1:  83.64 | Prec:  82.14 | Rec:  85.19 || Ex/s:  94.93\n",
      "\n",
      "* Best F1: tensor(83.6364, device='cuda:0')\n",
      "Saving best model...\n",
      "Done.\n",
      "---------------------\n",
      "\n",
      "===>  TRAIN Epoch 11\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "0% [██] 100% | ETA: 00:00:00\n",
      "Total time elapsed: 00:00:05\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Finished Epoch 11 || Run Time:    4.5 | Load Time:    1.9 || F1:  98.11 | Prec:  96.30 | Rec: 100.00 || Ex/s:  50.30\n",
      "\n",
      "===>  EVAL Epoch 11\n",
      "Finished Epoch 11 || Run Time:    0.6 | Load Time:    0.6 || F1:  83.64 | Prec:  82.14 | Rec:  85.19 || Ex/s:  91.34\n",
      "\n",
      "---------------------\n",
      "\n",
      "===>  TRAIN Epoch 12\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "0% [██] 100% | ETA: 00:00:00\n",
      "Total time elapsed: 00:00:05\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Finished Epoch 12 || Run Time:    4.5 | Load Time:    2.0 || F1:  97.50 | Prec:  95.12 | Rec: 100.00 || Ex/s:  49.36\n",
      "\n",
      "===>  EVAL Epoch 12\n",
      "Finished Epoch 12 || Run Time:    0.6 | Load Time:    0.7 || F1:  83.64 | Prec:  82.14 | Rec:  85.19 || Ex/s:  85.46\n",
      "\n",
      "---------------------\n",
      "\n",
      "===>  TRAIN Epoch 13\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "0% [██] 100% | ETA: 00:00:00\n",
      "Total time elapsed: 00:00:05\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Finished Epoch 13 || Run Time:    4.5 | Load Time:    1.9 || F1:  98.11 | Prec:  96.30 | Rec: 100.00 || Ex/s:  50.27\n",
      "\n",
      "===>  EVAL Epoch 13\n",
      "Finished Epoch 13 || Run Time:    0.6 | Load Time:    0.6 || F1:  85.19 | Prec:  85.19 | Rec:  85.19 || Ex/s:  91.94\n",
      "\n",
      "* Best F1: tensor(85.1852, device='cuda:0')\n",
      "Saving best model...\n",
      "Done.\n",
      "---------------------\n",
      "\n",
      "===>  TRAIN Epoch 14\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "0% [██] 100% | ETA: 00:00:00\n",
      "Total time elapsed: 00:00:05\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Finished Epoch 14 || Run Time:    4.4 | Load Time:    1.9 || F1:  98.73 | Prec:  97.50 | Rec: 100.00 || Ex/s:  50.61\n",
      "\n",
      "===>  EVAL Epoch 14\n",
      "Finished Epoch 14 || Run Time:    0.6 | Load Time:    0.6 || F1:  85.19 | Prec:  85.19 | Rec:  85.19 || Ex/s:  92.29\n",
      "\n",
      "---------------------\n",
      "\n",
      "===>  TRAIN Epoch 15\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "0% [██] 100% | ETA: 00:00:00\n",
      "Total time elapsed: 00:00:05\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Finished Epoch 15 || Run Time:    4.6 | Load Time:    1.9 || F1:  99.36 | Prec:  98.73 | Rec: 100.00 || Ex/s:  49.07\n",
      "\n",
      "===>  EVAL Epoch 15\n",
      "Finished Epoch 15 || Run Time:    0.6 | Load Time:    0.7 || F1:  85.19 | Prec:  85.19 | Rec:  85.19 || Ex/s:  85.39\n",
      "\n",
      "---------------------\n",
      "\n",
      "===>  TRAIN Epoch 16\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "0% [██] 100% | ETA: 00:00:00\n",
      "Total time elapsed: 00:00:05\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Finished Epoch 16 || Run Time:    4.5 | Load Time:    1.9 || F1:  99.36 | Prec:  98.73 | Rec: 100.00 || Ex/s:  49.85\n",
      "\n",
      "===>  EVAL Epoch 16\n",
      "Finished Epoch 16 || Run Time:    0.6 | Load Time:    0.6 || F1:  85.19 | Prec:  85.19 | Rec:  85.19 || Ex/s:  90.39\n",
      "\n",
      "---------------------\n",
      "\n",
      "===>  TRAIN Epoch 17\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "0% [██] 100% | ETA: 00:00:00\n",
      "Total time elapsed: 00:00:05\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Finished Epoch 17 || Run Time:    4.7 | Load Time:    2.0 || F1:  99.36 | Prec:  98.73 | Rec: 100.00 || Ex/s:  48.25\n",
      "\n",
      "===>  EVAL Epoch 17\n",
      "Finished Epoch 17 || Run Time:    0.6 | Load Time:    0.7 || F1:  85.19 | Prec:  85.19 | Rec:  85.19 || Ex/s:  84.58\n",
      "\n",
      "---------------------\n",
      "\n",
      "===>  TRAIN Epoch 18\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "0% [██] 100% | ETA: 00:00:00\n",
      "Total time elapsed: 00:00:06\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Finished Epoch 18 || Run Time:    4.9 | Load Time:    2.1 || F1:  99.36 | Prec:  98.73 | Rec: 100.00 || Ex/s:  45.93\n",
      "\n",
      "===>  EVAL Epoch 18\n",
      "Finished Epoch 18 || Run Time:    0.6 | Load Time:    0.7 || F1:  85.19 | Prec:  85.19 | Rec:  85.19 || Ex/s:  82.82\n",
      "\n",
      "---------------------\n",
      "\n",
      "===>  TRAIN Epoch 19\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "0% [██] 100% | ETA: 00:00:00\n",
      "Total time elapsed: 00:00:05\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Finished Epoch 19 || Run Time:    4.6 | Load Time:    2.0 || F1:  99.36 | Prec:  98.73 | Rec: 100.00 || Ex/s:  48.71\n",
      "\n",
      "===>  EVAL Epoch 19\n",
      "Finished Epoch 19 || Run Time:    0.6 | Load Time:    0.6 || F1:  85.19 | Prec:  85.19 | Rec:  85.19 || Ex/s:  87.91\n",
      "\n",
      "---------------------\n",
      "\n",
      "===>  TRAIN Epoch 20\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "0% [██] 100% | ETA: 00:00:00\n",
      "Total time elapsed: 00:00:05\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Finished Epoch 20 || Run Time:    4.7 | Load Time:    2.0 || F1:  99.36 | Prec:  98.73 | Rec: 100.00 || Ex/s:  47.68\n",
      "\n",
      "===>  EVAL Epoch 20\n",
      "Finished Epoch 20 || Run Time:    0.6 | Load Time:    0.7 || F1:  85.19 | Prec:  85.19 | Rec:  85.19 || Ex/s:  83.28\n",
      "\n",
      "---------------------\n",
      "\n",
      "===>  TRAIN Epoch 21\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "0% [██] 100% | ETA: 00:00:00\n",
      "Total time elapsed: 00:00:05\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Finished Epoch 21 || Run Time:    4.7 | Load Time:    2.0 || F1:  99.36 | Prec:  98.73 | Rec: 100.00 || Ex/s:  48.09\n",
      "\n",
      "===>  EVAL Epoch 21\n",
      "Finished Epoch 21 || Run Time:    0.6 | Load Time:    0.6 || F1:  85.19 | Prec:  85.19 | Rec:  85.19 || Ex/s:  89.69\n",
      "\n",
      "---------------------\n",
      "\n",
      "===>  TRAIN Epoch 22\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "0% [██] 100% | ETA: 00:00:00\n",
      "Total time elapsed: 00:00:05\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Finished Epoch 22 || Run Time:    4.7 | Load Time:    2.0 || F1:  99.36 | Prec:  98.73 | Rec: 100.00 || Ex/s:  48.03\n",
      "\n",
      "===>  EVAL Epoch 22\n",
      "Finished Epoch 22 || Run Time:    0.7 | Load Time:    0.7 || F1:  85.19 | Prec:  85.19 | Rec:  85.19 || Ex/s:  82.27\n",
      "\n",
      "---------------------\n",
      "\n",
      "===>  TRAIN Epoch 23\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "0% [██] 100% | ETA: 00:00:00\n",
      "Total time elapsed: 00:00:05\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Finished Epoch 23 || Run Time:    4.7 | Load Time:    2.0 || F1:  99.36 | Prec:  98.73 | Rec: 100.00 || Ex/s:  47.41\n",
      "\n",
      "===>  EVAL Epoch 23\n",
      "Finished Epoch 23 || Run Time:    0.6 | Load Time:    0.7 || F1:  85.19 | Prec:  85.19 | Rec:  85.19 || Ex/s:  83.07\n",
      "\n",
      "---------------------\n",
      "\n",
      "===>  TRAIN Epoch 24\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "0% [██] 100% | ETA: 00:00:00\n",
      "Total time elapsed: 00:00:05\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Finished Epoch 24 || Run Time:    4.7 | Load Time:    2.0 || F1:  99.36 | Prec:  98.73 | Rec: 100.00 || Ex/s:  47.70\n",
      "\n",
      "===>  EVAL Epoch 24\n",
      "Finished Epoch 24 || Run Time:    0.6 | Load Time:    0.7 || F1:  85.19 | Prec:  85.19 | Rec:  85.19 || Ex/s:  82.40\n",
      "\n",
      "---------------------\n",
      "\n",
      "===>  TRAIN Epoch 25\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "0% [██] 100% | ETA: 00:00:00\n",
      "Total time elapsed: 00:00:05\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Finished Epoch 25 || Run Time:    4.7 | Load Time:    2.0 || F1:  99.36 | Prec:  98.73 | Rec: 100.00 || Ex/s:  47.95\n",
      "\n",
      "===>  EVAL Epoch 25\n",
      "Finished Epoch 25 || Run Time:    0.6 | Load Time:    0.7 || F1:  85.19 | Prec:  85.19 | Rec:  85.19 || Ex/s:  82.66\n",
      "\n",
      "---------------------\n",
      "\n",
      "Loading best model...\n",
      "Training done.\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "tensor(85.1852, device='cuda:0')"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "att_model2 = dm.MatchingModel(attr_summarizer='hybrid')\n",
    "att_model2.run_train(train,validation,epochs=25,pos_neg_ratio=5,\n",
    "                    best_save_path='models/itunes_amazon_hybrid_adversarial20.pth')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-02-09T15:00:52.243085Z",
     "start_time": "2020-02-09T15:00:50.813147Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "===>  EVAL Epoch 13\n",
      "Finished Epoch 13 || Run Time:    0.7 | Load Time:    0.7 || F1:  84.00 | Prec:  91.30 | Rec:  77.78 || Ex/s:  80.33\n",
      "\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "tensor(84., device='cuda:0')"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "att_model2.run_eval(test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-02-09T15:01:04.963263Z",
     "start_time": "2020-02-09T15:01:03.129732Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(0.7836888705235009, 0.016787313181924515)"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "getMeanConfidenceAndVariance(att_model,test_df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-02-09T15:02:10.123073Z",
     "start_time": "2020-02-09T15:01:34.199975Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 20/20 [00:35<00:00,  1.79s/it]\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(0.6372940167985787, 0.5283647337019737, 0.8074074074074075)"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "avgf1,avgprecision,avgrecall = getAvgF1PrecisionRecall('datasets/Structured/itunes-amazon/merged_test.csv',\n",
    "                                                       att_model2,20,editdist=1)\n",
    "avgf1,avgprecision,avgrecall"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-02-07T16:48:43.513088Z",
     "start_time": "2020-02-07T16:48:12.457438Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 20/20 [00:31<00:00,  1.55s/it]\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(0.5977356035542621, 0.4743214801476393, 0.8111111111111112)"
      ]
     },
     "execution_count": 36,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "avgf1,avgprecision,avgrecall = getAvgF1PrecisionRecall('datasets/Structured/itunes-amazon/merged_test.csv',\n",
    "                                                       att_model,20,editdist=2)\n",
    "avgf1,avgprecision,avgrecall"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Retrain with 33% perturbed tra"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-02-09T15:07:13.659162Z",
     "start_time": "2020-02-09T15:07:13.582754Z"
    }
   },
   "outputs": [],
   "source": [
    "train_df = pd.read_csv(DATASET_DIR+'merged_train.csv')\n",
    "adversarialTrain_30 = getAdversarialTrainingSet(train_df,int(len(train_df)/3),editdist=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-02-09T15:08:00.187643Z",
     "start_time": "2020-02-09T15:08:00.143363Z"
    }
   },
   "outputs": [],
   "source": [
    "adversarialTrain_30.to_csv(DATASET_DIR+'adversarial_train30.csv',index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-02-09T15:08:13.675522Z",
     "start_time": "2020-02-09T15:08:11.399455Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING:deepmatcher.data.dataset:Rebuilding data cache because: ['Data file list has changed.', 'One or more data files have been modified.']\n",
      "\n",
      "Reading and processing data from \"datasets/Structured/itunes-amazon/adversarial_train30.csv\"\n",
      "0% [############################# ] 100% | ETA: 00:00:00\n",
      "Reading and processing data from \"datasets/Structured/itunes-amazon/merged_valid.csv\"\n",
      "0% [############################# ] 100% | ETA: 00:00:00\n",
      "Reading and processing data from \"datasets/Structured/itunes-amazon/merged_test.csv\"\n",
      "0% [############################# ] 100% | ETA: 00:00:00\n",
      "Building vocabulary\n",
      "0% [#] 100% | ETA: 00:00:00\n",
      "Total time elapsed: 00:00:00\n",
      "\n",
      "Computing principal components\n",
      "0% [#] 100% | ETA: 00:00:00\n",
      "Total time elapsed: 00:00:00\n"
     ]
    }
   ],
   "source": [
    "train,validation,test = dm.data.process('datasets/Structured/itunes-amazon/',train='adversarial_train30.csv',\n",
    "                                       validation='merged_valid.csv',test='merged_test.csv',\n",
    "                                       left_prefix='ltable_',right_prefix='rtable_',cache='editdist.pth')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-02-09T15:15:28.789991Z",
     "start_time": "2020-02-09T15:08:51.216167Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "* Number of trainable parameters: 17757810\n",
      "===>  TRAIN Epoch 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/nvidia/anaconda3/envs/torchenv/lib/python3.6/site-packages/torch/nn/functional.py:1932: UserWarning: reduction: 'mean' divides the total loss by both the batch size and the support size.'batchmean' divides only by the batch size, and aligns with the KL div math definition.'mean' will be changed to behave the same as 'batchmean' in the next major release.\n",
      "  warnings.warn(\"reduction: 'mean' divides the total loss by both the batch size and the support size.\"\n",
      "0% [████] 100% | ETA: 00:00:00\n",
      "Total time elapsed: 00:00:10\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Finished Epoch 1 || Run Time:    9.1 | Load Time:    2.1 || F1:  39.39 | Prec:  24.53 | Rec: 100.00 || Ex/s:  28.64\n",
      "\n",
      "===>  EVAL Epoch 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "0% [█] 100% | ETA: 00:00:00\n",
      "Total time elapsed: 00:00:01\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Finished Epoch 1 || Run Time:    1.1 | Load Time:    0.6 || F1:  49.06 | Prec:  32.91 | Rec:  96.30 || Ex/s:  64.06\n",
      "\n",
      "* Best F1: tensor(49.0566, device='cuda:0')\n",
      "Saving best model...\n",
      "Done.\n",
      "---------------------\n",
      "\n",
      "===>  TRAIN Epoch 2\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "0% [████] 100% | ETA: 00:00:00\n",
      "Total time elapsed: 00:00:10\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Finished Epoch 2 || Run Time:    9.1 | Load Time:    2.0 || F1:  57.83 | Prec:  42.11 | Rec:  92.31 || Ex/s:  28.78\n",
      "\n",
      "===>  EVAL Epoch 2\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "0% [█] 100% | ETA: 00:00:00\n",
      "Total time elapsed: 00:00:01\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Finished Epoch 2 || Run Time:    1.1 | Load Time:    0.6 || F1:  55.42 | Prec:  41.07 | Rec:  85.19 || Ex/s:  63.25\n",
      "\n",
      "* Best F1: tensor(55.4217, device='cuda:0')\n",
      "Saving best model...\n",
      "Done.\n",
      "---------------------\n",
      "\n",
      "===>  TRAIN Epoch 3\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "0% [████] 100% | ETA: 00:00:00\n",
      "Total time elapsed: 00:00:10\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Finished Epoch 3 || Run Time:    9.4 | Load Time:    2.2 || F1:  68.60 | Prec:  55.04 | Rec:  91.03 || Ex/s:  27.70\n",
      "\n",
      "===>  EVAL Epoch 3\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "0% [█] 100% | ETA: 00:00:00\n",
      "Total time elapsed: 00:00:01\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Finished Epoch 3 || Run Time:    1.1 | Load Time:    0.6 || F1:  62.69 | Prec:  52.50 | Rec:  77.78 || Ex/s:  62.67\n",
      "\n",
      "* Best F1: tensor(62.6866, device='cuda:0')\n",
      "Saving best model...\n",
      "Done.\n",
      "---------------------\n",
      "\n",
      "===>  TRAIN Epoch 4\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "0% [████] 100% | ETA: 00:00:00\n",
      "Total time elapsed: 00:00:10\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Finished Epoch 4 || Run Time:    9.6 | Load Time:    2.2 || F1:  75.90 | Prec:  63.25 | Rec:  94.87 || Ex/s:  27.26\n",
      "\n",
      "===>  EVAL Epoch 4\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "0% [█] 100% | ETA: 00:00:00\n",
      "Total time elapsed: 00:00:01\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Finished Epoch 4 || Run Time:    1.1 | Load Time:    0.6 || F1:  64.86 | Prec:  51.06 | Rec:  88.89 || Ex/s:  63.36\n",
      "\n",
      "* Best F1: tensor(64.8649, device='cuda:0')\n",
      "Saving best model...\n",
      "Done.\n",
      "---------------------\n",
      "\n",
      "===>  TRAIN Epoch 5\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "0% [████] 100% | ETA: 00:00:00\n",
      "Total time elapsed: 00:00:10\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Finished Epoch 5 || Run Time:    9.6 | Load Time:    2.2 || F1:  79.17 | Prec:  66.67 | Rec:  97.44 || Ex/s:  27.21\n",
      "\n",
      "===>  EVAL Epoch 5\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "0% [█] 100% | ETA: 00:00:00\n",
      "Total time elapsed: 00:00:01\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Finished Epoch 5 || Run Time:    1.1 | Load Time:    0.6 || F1:  69.44 | Prec:  55.56 | Rec:  92.59 || Ex/s:  64.20\n",
      "\n",
      "* Best F1: tensor(69.4444, device='cuda:0')\n",
      "Saving best model...\n",
      "Done.\n",
      "---------------------\n",
      "\n",
      "===>  TRAIN Epoch 6\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "0% [████] 100% | ETA: 00:00:00\n",
      "Total time elapsed: 00:00:10\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Finished Epoch 6 || Run Time:    9.6 | Load Time:    2.2 || F1:  86.19 | Prec:  75.73 | Rec: 100.00 || Ex/s:  27.27\n",
      "\n",
      "===>  EVAL Epoch 6\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "0% [█] 100% | ETA: 00:00:00\n",
      "Total time elapsed: 00:00:01\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Finished Epoch 6 || Run Time:    1.1 | Load Time:    0.6 || F1:  67.65 | Prec:  56.10 | Rec:  85.19 || Ex/s:  63.21\n",
      "\n",
      "---------------------\n",
      "\n",
      "===>  TRAIN Epoch 7\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "0% [████] 100% | ETA: 00:00:00\n",
      "Total time elapsed: 00:00:10\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Finished Epoch 7 || Run Time:    9.4 | Load Time:    2.1 || F1:  91.23 | Prec:  83.87 | Rec: 100.00 || Ex/s:  27.87\n",
      "\n",
      "===>  EVAL Epoch 7\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "0% [█] 100% | ETA: 00:00:00\n",
      "Total time elapsed: 00:00:01\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Finished Epoch 7 || Run Time:    1.1 | Load Time:    0.6 || F1:  74.19 | Prec:  65.71 | Rec:  85.19 || Ex/s:  63.68\n",
      "\n",
      "* Best F1: tensor(74.1936, device='cuda:0')\n",
      "Saving best model...\n",
      "Done.\n",
      "---------------------\n",
      "\n",
      "===>  TRAIN Epoch 8\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "0% [████] 100% | ETA: 00:00:00\n",
      "Total time elapsed: 00:00:10\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Finished Epoch 8 || Run Time:    9.6 | Load Time:    2.2 || F1:  96.89 | Prec:  93.98 | Rec: 100.00 || Ex/s:  27.27\n",
      "\n",
      "===>  EVAL Epoch 8\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "0% [█] 100% | ETA: 00:00:00\n",
      "Total time elapsed: 00:00:01\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Finished Epoch 8 || Run Time:    1.1 | Load Time:    0.6 || F1:  76.19 | Prec:  66.67 | Rec:  88.89 || Ex/s:  63.66\n",
      "\n",
      "* Best F1: tensor(76.1905, device='cuda:0')\n",
      "Saving best model...\n",
      "Done.\n",
      "---------------------\n",
      "\n",
      "===>  TRAIN Epoch 9\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "0% [████] 100% | ETA: 00:00:00\n",
      "Total time elapsed: 00:00:10\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Finished Epoch 9 || Run Time:    9.5 | Load Time:    2.2 || F1:  99.36 | Prec:  98.73 | Rec: 100.00 || Ex/s:  27.45\n",
      "\n",
      "===>  EVAL Epoch 9\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "0% [█] 100% | ETA: 00:00:00\n",
      "Total time elapsed: 00:00:01\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Finished Epoch 9 || Run Time:    1.1 | Load Time:    0.6 || F1:  76.19 | Prec:  66.67 | Rec:  88.89 || Ex/s:  63.72\n",
      "\n",
      "---------------------\n",
      "\n",
      "===>  TRAIN Epoch 10\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "0% [████] 100% | ETA: 00:00:00\n",
      "Total time elapsed: 00:00:10\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Finished Epoch 10 || Run Time:    9.5 | Load Time:    2.1 || F1:  99.36 | Prec:  98.73 | Rec: 100.00 || Ex/s:  27.47\n",
      "\n",
      "===>  EVAL Epoch 10\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "0% [█] 100% | ETA: 00:00:00\n",
      "Total time elapsed: 00:00:01\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Finished Epoch 10 || Run Time:    1.1 | Load Time:    0.6 || F1:  77.42 | Prec:  68.57 | Rec:  88.89 || Ex/s:  65.36\n",
      "\n",
      "* Best F1: tensor(77.4194, device='cuda:0')\n",
      "Saving best model...\n",
      "Done.\n",
      "---------------------\n",
      "\n",
      "===>  TRAIN Epoch 11\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "0% [████] 100% | ETA: 00:00:00\n",
      "Total time elapsed: 00:00:10\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Finished Epoch 11 || Run Time:    9.5 | Load Time:    2.2 || F1:  99.36 | Prec:  98.73 | Rec: 100.00 || Ex/s:  27.46\n",
      "\n",
      "===>  EVAL Epoch 11\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "0% [█] 100% | ETA: 00:00:00\n",
      "Total time elapsed: 00:00:01\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Finished Epoch 11 || Run Time:    1.1 | Load Time:    0.6 || F1:  75.41 | Prec:  67.65 | Rec:  85.19 || Ex/s:  64.73\n",
      "\n",
      "---------------------\n",
      "\n",
      "===>  TRAIN Epoch 12\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "0% [████] 100% | ETA: 00:00:00\n",
      "Total time elapsed: 00:00:10\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Finished Epoch 12 || Run Time:    9.5 | Load Time:    2.1 || F1: 100.00 | Prec: 100.00 | Rec: 100.00 || Ex/s:  27.57\n",
      "\n",
      "===>  EVAL Epoch 12\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "0% [█] 100% | ETA: 00:00:00\n",
      "Total time elapsed: 00:00:01\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Finished Epoch 12 || Run Time:    1.1 | Load Time:    0.6 || F1:  76.19 | Prec:  66.67 | Rec:  88.89 || Ex/s:  65.50\n",
      "\n",
      "---------------------\n",
      "\n",
      "===>  TRAIN Epoch 13\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "0% [████] 100% | ETA: 00:00:00\n",
      "Total time elapsed: 00:00:10\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Finished Epoch 13 || Run Time:    9.4 | Load Time:    2.1 || F1: 100.00 | Prec: 100.00 | Rec: 100.00 || Ex/s:  27.89\n",
      "\n",
      "===>  EVAL Epoch 13\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "0% [█] 100% | ETA: 00:00:00\n",
      "Total time elapsed: 00:00:01\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Finished Epoch 13 || Run Time:    1.1 | Load Time:    0.6 || F1:  76.19 | Prec:  66.67 | Rec:  88.89 || Ex/s:  65.28\n",
      "\n",
      "---------------------\n",
      "\n",
      "===>  TRAIN Epoch 14\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "0% [████] 100% | ETA: 00:00:00\n",
      "Total time elapsed: 00:00:10\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Finished Epoch 14 || Run Time:    9.4 | Load Time:    2.1 || F1: 100.00 | Prec: 100.00 | Rec: 100.00 || Ex/s:  27.78\n",
      "\n",
      "===>  EVAL Epoch 14\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "0% [█] 100% | ETA: 00:00:00\n",
      "Total time elapsed: 00:00:01\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Finished Epoch 14 || Run Time:    1.1 | Load Time:    0.6 || F1:  76.19 | Prec:  66.67 | Rec:  88.89 || Ex/s:  65.63\n",
      "\n",
      "---------------------\n",
      "\n",
      "===>  TRAIN Epoch 15\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "0% [████] 100% | ETA: 00:00:00\n",
      "Total time elapsed: 00:00:10\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Finished Epoch 15 || Run Time:    9.3 | Load Time:    2.1 || F1: 100.00 | Prec: 100.00 | Rec: 100.00 || Ex/s:  28.13\n",
      "\n",
      "===>  EVAL Epoch 15\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "0% [█] 100% | ETA: 00:00:00\n",
      "Total time elapsed: 00:00:01\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Finished Epoch 15 || Run Time:    1.1 | Load Time:    0.6 || F1:  76.19 | Prec:  66.67 | Rec:  88.89 || Ex/s:  64.28\n",
      "\n",
      "---------------------\n",
      "\n",
      "===>  TRAIN Epoch 16\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "0% [████] 100% | ETA: 00:00:00\n",
      "Total time elapsed: 00:00:10\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Finished Epoch 16 || Run Time:    9.5 | Load Time:    2.1 || F1: 100.00 | Prec: 100.00 | Rec: 100.00 || Ex/s:  27.88\n",
      "\n",
      "===>  EVAL Epoch 16\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "0% [█] 100% | ETA: 00:00:00\n",
      "Total time elapsed: 00:00:01\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Finished Epoch 16 || Run Time:    1.1 | Load Time:    0.6 || F1:  76.19 | Prec:  66.67 | Rec:  88.89 || Ex/s:  65.75\n",
      "\n",
      "---------------------\n",
      "\n",
      "===>  TRAIN Epoch 17\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "0% [████] 100% | ETA: 00:00:00\n",
      "Total time elapsed: 00:00:10\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Finished Epoch 17 || Run Time:    9.4 | Load Time:    2.1 || F1: 100.00 | Prec: 100.00 | Rec: 100.00 || Ex/s:  28.00\n",
      "\n",
      "===>  EVAL Epoch 17\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "0% [█] 100% | ETA: 00:00:00\n",
      "Total time elapsed: 00:00:01\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Finished Epoch 17 || Run Time:    1.1 | Load Time:    0.6 || F1:  76.19 | Prec:  66.67 | Rec:  88.89 || Ex/s:  64.88\n",
      "\n",
      "---------------------\n",
      "\n",
      "===>  TRAIN Epoch 18\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "0% [████] 100% | ETA: 00:00:00\n",
      "Total time elapsed: 00:00:10\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Finished Epoch 18 || Run Time:    9.5 | Load Time:    2.1 || F1: 100.00 | Prec: 100.00 | Rec: 100.00 || Ex/s:  27.51\n",
      "\n",
      "===>  EVAL Epoch 18\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "0% [█] 100% | ETA: 00:00:00\n",
      "Total time elapsed: 00:00:01\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Finished Epoch 18 || Run Time:    1.1 | Load Time:    0.6 || F1:  76.19 | Prec:  66.67 | Rec:  88.89 || Ex/s:  64.65\n",
      "\n",
      "---------------------\n",
      "\n",
      "===>  TRAIN Epoch 19\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "0% [████] 100% | ETA: 00:00:00\n",
      "Total time elapsed: 00:00:10\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Finished Epoch 19 || Run Time:    9.5 | Load Time:    2.1 || F1: 100.00 | Prec: 100.00 | Rec: 100.00 || Ex/s:  27.69\n",
      "\n",
      "===>  EVAL Epoch 19\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "0% [█] 100% | ETA: 00:00:00\n",
      "Total time elapsed: 00:00:01\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Finished Epoch 19 || Run Time:    1.1 | Load Time:    0.6 || F1:  76.19 | Prec:  66.67 | Rec:  88.89 || Ex/s:  65.04\n",
      "\n",
      "---------------------\n",
      "\n",
      "===>  TRAIN Epoch 20\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "0% [████] 100% | ETA: 00:00:00\n",
      "Total time elapsed: 00:00:10\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Finished Epoch 20 || Run Time:    9.4 | Load Time:    2.1 || F1: 100.00 | Prec: 100.00 | Rec: 100.00 || Ex/s:  28.02\n",
      "\n",
      "===>  EVAL Epoch 20\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "0% [█] 100% | ETA: 00:00:00\n",
      "Total time elapsed: 00:00:01\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Finished Epoch 20 || Run Time:    1.1 | Load Time:    0.6 || F1:  76.19 | Prec:  66.67 | Rec:  88.89 || Ex/s:  64.21\n",
      "\n",
      "---------------------\n",
      "\n",
      "===>  TRAIN Epoch 21\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "0% [████] 100% | ETA: 00:00:00\n",
      "Total time elapsed: 00:00:10\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Finished Epoch 21 || Run Time:    9.4 | Load Time:    2.1 || F1: 100.00 | Prec: 100.00 | Rec: 100.00 || Ex/s:  27.76\n",
      "\n",
      "===>  EVAL Epoch 21\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "0% [█] 100% | ETA: 00:00:00\n",
      "Total time elapsed: 00:00:01\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Finished Epoch 21 || Run Time:    1.1 | Load Time:    0.6 || F1:  76.19 | Prec:  66.67 | Rec:  88.89 || Ex/s:  64.58\n",
      "\n",
      "---------------------\n",
      "\n",
      "===>  TRAIN Epoch 22\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "0% [████] 100% | ETA: 00:00:00\n",
      "Total time elapsed: 00:00:10\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Finished Epoch 22 || Run Time:    9.4 | Load Time:    2.1 || F1: 100.00 | Prec: 100.00 | Rec: 100.00 || Ex/s:  28.02\n",
      "\n",
      "===>  EVAL Epoch 22\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "0% [█] 100% | ETA: 00:00:00\n",
      "Total time elapsed: 00:00:01\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Finished Epoch 22 || Run Time:    1.1 | Load Time:    0.6 || F1:  76.19 | Prec:  66.67 | Rec:  88.89 || Ex/s:  65.15\n",
      "\n",
      "---------------------\n",
      "\n",
      "===>  TRAIN Epoch 23\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "0% [████] 100% | ETA: 00:00:00\n",
      "Total time elapsed: 00:00:10\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Finished Epoch 23 || Run Time:    9.4 | Load Time:    2.1 || F1: 100.00 | Prec: 100.00 | Rec: 100.00 || Ex/s:  27.84\n",
      "\n",
      "===>  EVAL Epoch 23\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "0% [█] 100% | ETA: 00:00:00\n",
      "Total time elapsed: 00:00:01\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Finished Epoch 23 || Run Time:    1.1 | Load Time:    0.6 || F1:  76.19 | Prec:  66.67 | Rec:  88.89 || Ex/s:  64.33\n",
      "\n",
      "---------------------\n",
      "\n",
      "===>  TRAIN Epoch 24\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "0% [████] 100% | ETA: 00:00:00\n",
      "Total time elapsed: 00:00:10\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Finished Epoch 24 || Run Time:    9.5 | Load Time:    2.2 || F1: 100.00 | Prec: 100.00 | Rec: 100.00 || Ex/s:  27.54\n",
      "\n",
      "===>  EVAL Epoch 24\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "0% [█] 100% | ETA: 00:00:00\n",
      "Total time elapsed: 00:00:01\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Finished Epoch 24 || Run Time:    1.0 | Load Time:    0.6 || F1:  76.19 | Prec:  66.67 | Rec:  88.89 || Ex/s:  66.60\n",
      "\n",
      "---------------------\n",
      "\n",
      "===>  TRAIN Epoch 25\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "0% [████] 100% | ETA: 00:00:00\n",
      "Total time elapsed: 00:00:09\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Finished Epoch 25 || Run Time:    8.7 | Load Time:    1.9 || F1: 100.00 | Prec: 100.00 | Rec: 100.00 || Ex/s:  30.25\n",
      "\n",
      "===>  EVAL Epoch 25\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "0% [█] 100% | ETA: 00:00:00\n",
      "Total time elapsed: 00:00:01\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Finished Epoch 25 || Run Time:    1.0 | Load Time:    0.6 || F1:  76.19 | Prec:  66.67 | Rec:  88.89 || Ex/s:  70.93\n",
      "\n",
      "---------------------\n",
      "\n",
      "===>  TRAIN Epoch 26\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "0% [████] 100% | ETA: 00:00:00\n",
      "Total time elapsed: 00:00:09\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Finished Epoch 26 || Run Time:    8.5 | Load Time:    1.8 || F1: 100.00 | Prec: 100.00 | Rec: 100.00 || Ex/s:  30.88\n",
      "\n",
      "===>  EVAL Epoch 26\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "0% [█] 100% | ETA: 00:00:00\n",
      "Total time elapsed: 00:00:01\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Finished Epoch 26 || Run Time:    1.0 | Load Time:    0.6 || F1:  76.19 | Prec:  66.67 | Rec:  88.89 || Ex/s:  67.09\n",
      "\n",
      "---------------------\n",
      "\n",
      "===>  TRAIN Epoch 27\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "0% [████] 100% | ETA: 00:00:00\n",
      "Total time elapsed: 00:00:09\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Finished Epoch 27 || Run Time:    8.3 | Load Time:    1.7 || F1: 100.00 | Prec: 100.00 | Rec: 100.00 || Ex/s:  32.12\n",
      "\n",
      "===>  EVAL Epoch 27\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "0% [█] 100% | ETA: 00:00:00\n",
      "Total time elapsed: 00:00:01\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Finished Epoch 27 || Run Time:    1.0 | Load Time:    0.5 || F1:  76.19 | Prec:  66.67 | Rec:  88.89 || Ex/s:  72.03\n",
      "\n",
      "---------------------\n",
      "\n",
      "===>  TRAIN Epoch 28\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "0% [████] 100% | ETA: 00:00:00\n",
      "Total time elapsed: 00:00:09\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Finished Epoch 28 || Run Time:    8.5 | Load Time:    1.8 || F1: 100.00 | Prec: 100.00 | Rec: 100.00 || Ex/s:  31.08\n",
      "\n",
      "===>  EVAL Epoch 28\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "0% [█] 100% | ETA: 00:00:00\n",
      "Total time elapsed: 00:00:01\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Finished Epoch 28 || Run Time:    1.1 | Load Time:    0.6 || F1:  76.19 | Prec:  66.67 | Rec:  88.89 || Ex/s:  64.97\n",
      "\n",
      "---------------------\n",
      "\n",
      "===>  TRAIN Epoch 29\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "0% [████] 100% | ETA: 00:00:00\n",
      "Total time elapsed: 00:00:09\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Finished Epoch 29 || Run Time:    8.5 | Load Time:    1.8 || F1: 100.00 | Prec: 100.00 | Rec: 100.00 || Ex/s:  31.09\n",
      "\n",
      "===>  EVAL Epoch 29\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "0% [█] 100% | ETA: 00:00:00\n",
      "Total time elapsed: 00:00:01\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Finished Epoch 29 || Run Time:    1.0 | Load Time:    0.6 || F1:  76.19 | Prec:  66.67 | Rec:  88.89 || Ex/s:  68.79\n",
      "\n",
      "---------------------\n",
      "\n",
      "===>  TRAIN Epoch 30\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "0% [████] 100% | ETA: 00:00:00\n",
      "Total time elapsed: 00:00:09\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Finished Epoch 30 || Run Time:    8.8 | Load Time:    1.9 || F1: 100.00 | Prec: 100.00 | Rec: 100.00 || Ex/s:  29.89\n",
      "\n",
      "===>  EVAL Epoch 30\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "0% [█] 100% | ETA: 00:00:00\n",
      "Total time elapsed: 00:00:01\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Finished Epoch 30 || Run Time:    1.1 | Load Time:    0.6 || F1:  76.19 | Prec:  66.67 | Rec:  88.89 || Ex/s:  65.61\n",
      "\n",
      "---------------------\n",
      "\n",
      "Loading best model...\n",
      "Training done.\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "tensor(77.4194, device='cuda:0')"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "att_model3 = dm.MatchingModel(attr_summarizer='hybrid')\n",
    "att_model3.run_train(train,validation,epochs=30,pos_neg_ratio=5,batch_size=16,\n",
    "                    best_save_path='models/itunes_amazon_hybrid_adversarial30.pth')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-02-09T15:15:39.132619Z",
     "start_time": "2020-02-09T15:15:37.771072Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "===>  EVAL Epoch 10\n",
      "Finished Epoch 10 || Run Time:    0.6 | Load Time:    0.7 || F1:  81.97 | Prec:  73.53 | Rec:  92.59 || Ex/s:  84.46\n",
      "\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "tensor(81.9672, device='cuda:0')"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "att_model3.run_eval(test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "start_time": "2020-02-09T15:15:51.128Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 50%|█████     | 10/20 [00:17<00:17,  1.74s/it]"
     ]
    }
   ],
   "source": [
    "avgf1,avgprecision,avgrecall = getAvgF1PrecisionRecall('datasets/Structured/itunes-amazon/merged_test.csv',\n",
    "                                                       att_model3,20,editdist=1)\n",
    "avgf1,avgprecision,avgrecall"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Evaluate rankings with triangles"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## StandardModel"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-02-25T09:37:53.465697Z",
     "start_time": "2020-02-25T09:37:53.429086Z"
    }
   },
   "outputs": [],
   "source": [
    "from utils.dataset_parser import getFullDataset,generate_train_valid_test\n",
    "from utils.mojito2 import prepareDataset,aggregateRankings,explainSamples\n",
    "from utils.trianglesChart import plotMostFrequentTokens"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-02-25T13:16:14.607894Z",
     "start_time": "2020-02-25T13:16:10.533412Z"
    }
   },
   "outputs": [],
   "source": [
    "standard_model = dm.MatchingModel(attr_summarizer='hybrid')\n",
    "standard_model.load_state('models/beer_hybrid.pth')\n",
    "standard_model = standard_model.to('cuda')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-02-25T09:20:28.828008Z",
     "start_time": "2020-02-25T09:20:28.755551Z"
    }
   },
   "outputs": [],
   "source": [
    "tableA = pd.read_csv(DATASET_DIR+'tableA.csv')\n",
    "tableB = pd.read_csv(DATASET_DIR+'tableB.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-02-25T09:20:29.650748Z",
     "start_time": "2020-02-25T09:20:29.540193Z"
    }
   },
   "outputs": [],
   "source": [
    "train_df,validation_df,test_df = generate_train_valid_test(DATASET_DIR,['train.csv','valid.csv','test.csv']\n",
    "                                                      ,left_prefix='ltable_',right_prefix='rtable_',\n",
    "                                                          drop_lrid=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-02-13T14:22:10.044114Z",
     "start_time": "2020-02-13T14:22:09.545292Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(91, 169)"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "augmented_test = prepareDataset(test_df,tableA,tableB,80)\n",
    "len(test_df),len(augmented_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-02-18T10:41:53.642791Z",
     "start_time": "2020-02-18T10:41:53.605637Z"
    }
   },
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'augmented_test' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-9-53ce2604796d>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0maugmented_test\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mto_csv\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mDATASET_DIR\u001b[0m\u001b[0;34m+\u001b[0m\u001b[0;34m'augmented_test.csv'\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mindex\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mFalse\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m: name 'augmented_test' is not defined"
     ]
    }
   ],
   "source": [
    "augmented_test.to_csv(DATASET_DIR+'augmented_test.csv',index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-02-25T09:20:33.501966Z",
     "start_time": "2020-02-25T09:20:33.462585Z"
    }
   },
   "outputs": [],
   "source": [
    "augmented_test = pd.read_csv(DATASET_DIR+'augmented_test.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-02-25T09:20:48.524782Z",
     "start_time": "2020-02-25T09:20:37.440511Z"
    }
   },
   "outputs": [],
   "source": [
    "predictions0 = wrapDm(augmented_test,standard_model,ignore_columns=['ltable_id','rtable_id','id','label'])\n",
    "tp_group = augmented_test[(predictions0[:,1]>=0.5)& (augmented_test['label'] == 1)]\n",
    "tn_group = augmented_test[(predictions0[:,0] >=0.5)& (augmented_test['label']==0)]\n",
    "correctPredictions0 = pd.concat([tp_group,tn_group])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-02-25T09:21:42.722650Z",
     "start_time": "2020-02-25T09:20:57.535219Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 38/38 [00:45<00:00,  1.19s/it]\n"
     ]
    }
   ],
   "source": [
    "neg_rankings0,neg_triangles0,flippedPredictions_neg0 = explainSamples(\n",
    "    correctPredictions0,[tableA,tableB],standard_model,\n",
    "    originalClass=0,maxLenAttributeSet=4)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-02-25T09:22:13.187429Z",
     "start_time": "2020-02-25T09:22:12.932510Z"
    }
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAPsAAACMCAYAAAC6Vd/fAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8li6FKAAAQU0lEQVR4nO3dfXBV9Z3H8fcHyBo0GCsPswJirIUV60QerekiDa0rorswdN2qo2hld5GuD2xn3a2MusZOHayyreNsWXxYVHStzkrlaVpE1BRYtSHYiCgyUHZcw7pFIkICxgJ+94/zu3AJSe65l9zcJOf7mjmTc37n6XsC35zfefj9jswM51zP16vQATjnOocnu3MJ4cnuXEJ4sjuXEJ7sziWEJ7tzCdGn0AG4rmHAgAFWVlZW6DBcjjZu3LjbzAa2t4wnuwOgrKyM2traQofhciTpg0zLeDXeuYTwZHcuITzZnUsIT3bnEsKT3bmE8GR3LiE82Z1LCE925xLCk925hPBkdy4hPNmdSwhPducSwhvCuMjGjSAVOgqXyQl0EOtnducSwpPduYTwZHcuITzZnUsIT3bnEsKT3bmE8GR3LiE82Z1LCE925xIib8ku6bCkOklvS3pL0tfzta8W+62WtDXsu07SlTlsozLf8UqqknRA0qC0sqZ87tMlWz5fl/3MzEYBSJoMzAO+kb6ApD5mdigP+77WzE6kE/RKoAl4Pe4KOR7LbuAfgB9kuZ5zWeusavypwB44ctZcJ2k58F4ou05STTgTPyKpt6S/kvSTMH+OpB1h/MuS/ivbACQtlbRR0ruSZqWVXxZqHm9LekVSGTAb+H6I52JJZZJelbQpLDMsrPukpIWSfgM8IGmbpIFhXi9J21PTbVgEXCXp9CzibZL0YChfI+nCUJvZIWlqWKZ3WGZDiPmmbH9frgcys7wMwGGgDngf2AuMDeWVwH7g7DA9ElgBFIXpBcD1wB8DG0LZC8AGYAhwAzCvnf1WA1vDvuuA/qH89PCzL7AZ6A8MBD5MiyW1TBVwe9o2VwA3hPGZwNIw/iSwEugdpu8B/j6MXwosaSfOKuB24J+Be0NZU9r84+IN0wZMCeMvAquBIuACoC6UzwLuCuMnAbWpY2wRw6wwr3ZY1MTCh64+tAGozZSTnVWNrwAWSzo/zKsxs/8O498CxgIbFLW66gvsMrP/k1QiqR9wJvAsMBG4GPhFhn23Vo2/TdL0MH4mMJwo2demYjGzT9rYXgXw7TD+NPBA2rz/NLPDYXwRsAx4iOiPwhMZ4gR4GKiTND9GvA3AH4BVofwd4HMzOyjpHaAslF8KlKfdrygN66d+5wCY2aPAowDjJIsRq+vGOqWJq5m9IWkAUXJBdGZPEfCUmc1tZdXXgRuJztTriBKogug6NzZJlcAlQIWZHZBUDRRns412HDkWM/tQ0u8lfRO4ELg208pm9qmkZ4GbY8Z7MPwlB/gC+Dxs5wtJqX9PAbea2UsndGSuR+mUa3ZJ5wK9ic5MLb0CXJm6Ky3pdElnhXnriKq6a4HfApOIzmR7swyhFNgTEudc4KJQ/iYwUdLZqX2H8kagX9r6rwNXh/FrQ1xteRx4hmPP+Jn8BLiJo39824o3rpeA70kqApA0QtIpWW7D9TD5TPa+qcdfwPNE17zH/ec3s/eAu4DVkjYBLwNnhNnriKqwa8O6HwLrc4hlFdBH0hbgfqIkx8w+Jrpu/YWkt0OcEF2jT0/doANuBW4M8c0A5rSzr+VACfGq8IQ4dhNdf5/UXrxZeJzo5udbkjYDj+AdlSSejtYIY64gfQk408w25Sek7k3SOOCnZnZxoWPJxjjphJ5Vuk7SRr5K2mhm49pbNdaZPTzaOTVUc98CHks9FnNHSboDWAK0dv/BuYKKW40vNbN9RHekF5vZ14huIBWMpBfT3pJLDZMLGZOZ3W9mZ5nZkUsNSXe2EuedhYzTJVPc67g+ks4AvgN0if+oZjY981KFZ2b3AfcVOg7n4p7Zf0h0h/d3ZrZB0peBbfkLyznX0bK+Qed6Jr9B1010wg26EeGd8M1hulzSXVkH6pwrmLjV+MeI7jAfBAiP3a5udw3nXJcSN9lPNrOaFmX5aJrqnMuTuHfjd0s6h6jFFaGBxUd5i8p1vrFjodav2nuyuMl+M1HrqHMl7SRqPZWxkYdzruuIm+xmZpeExhS9zKwx1XjEOdc9xL1mXwJgZvvNrDGUvZCfkJxz+dDumT00r/wqUCrp22mzTqXj2oM7d5yDBw9SX19Pc3NzoUPpUoqLixk6dChFRUVZr5upGv8nwJ8DpwF/kVbeCPxt1ntzLqb6+nr69etHWVkZ8u/GA1EXcg0NDdTX13P22dlfRbeb7Ga2DFgmqcLM3sg1SOey1dzc7InegiT69+/Pxx9/nNP6cW/QzZJ03JnczGbmtFfnYvBEP96J/E7iJvvKtPFiYDrwvznv1TnX6WLdjTezJWnDfxA1dW33pXvnOpTUsUNMDz/8MCNHjmTIkCHccsstACxcuJDFixfn60gBqKqqYv78lh0On5hc+yUbDgzKuJRz3dyCBQtYs2YNa9asoTa8YTh79uwCR5WbuK3eGiXtS/0k6pDRP1nkerTZs2ezY8cOpkyZwp49e46Up591KysrmTNnDqNGjeL888+npqbmyDIzZsygoqKC4cOH89hjjx1Z/8EHH2T8+PGUl5dzzz33HCm/7777GDFiBBMmTGDr1q0dfjyxzuxm1i/zUs71LAsXLmTVqlW89tprrFy5ss3lDhw4QF1dHWvXrmXmzJls3rwZgE2bNvHmm2+yf/9+Ro8ezRVXXMHmzZvZtm0bNTU1mBlTp05l7dq1nHLKKTz33HPU1dVx6NAhxowZw9ixYzv0eGJX48NLNROIGsOsM7OlHRqJc93UNddcA8DEiRPZt28fn376KQDTpk2jb9++9O3bl0mTJlFTU8P69etZvXo1o0ePBqCpqYlt27bR2NjI9OnTOfnkkwGYOnVqh8cZK9klLQC+Avw8FM2W9GdmdnM7qzmXCC0fh6WmWys3M+bOnctNNx37rc2HHnoov0ES/934bwKTzewJM3sCuDyUOZd4zz8ffVtk/fr1lJaWUlpaCsCyZctobm6moaGB6upqxo8fz+TJk1m0aBFNTU0A7Ny5k127djFx4kSWLl3KZ599RmNjIytWrOjwOONW47cDw4APwvSZocy5ztGF+0osLi5m9OjRHDx4kEWLFh0pLy8vZ9KkSezevZu7776bwYMHM3jwYLZs2UJFRQUAJSUlPPPMM4wZM4arrrqKCy64gEGDBjF+/PgOj7PdDiclrSC6Ri8FxgM1YfprRF9irezwiFxBjBs3zmq7UOcVW7ZsYeTIkYUOI6PKykrmz5/PuHHHvnZSVVVFSUkJt99+e4fvs7XfTZwOJzOd2Tv2qb5zrmAyNYT5dWcF4lx3VF1d3Wp5VVVVp8YRR7s36CStDz9TL9Xsa/FyjXN54980ON6J/E4yndknhJ/+Uo3rVMXFxTQ0NNC/f39v/Rak2rMXF+fWb0zGu/GSegPvmtm5Oe3BuRwMHTqU+vr6nNtu91SpnmpykTHZzeywpK2ShpnZ/+S0F+eyVFRUlFNvLK5tcZ+zfwl4V1INsD9VaGYd/06fcy4v4ib73XmNwjmXd3GT/XIzO6ZJq6QfA/5ozrluItYnmyW9ZWZjWpRtMrPyvEXmOpV/srmbOIFPNmfqN/57wN8B50jalDarH/B6lmE65wooUzX+WeBXwDzgjrTyRjP7JG9ROec6XKaXavYCeyUdMrMP0udJetrMZuQ1Oudch4nbnv2r6ROS+gAd22eOcy6vMr0bP1dSI1Ce/m488HtgWadE6JzrEJmq8fOAeZLmAQ8AIzj6QUdvpeBcNxL3OfsOYC0wFKgDLgLewLumcq7biHvNfhtRTzUfmNkkYDTwad6ics51uLjJ3mxmzQCSTjKz94k+5+yc6ybiVuPrJZ0GLAVelrSHo51POue6gbhfhJkeRqskvUbUAeWqvEVVAJIOA+8AAg4Dt5hZ3t4SlFQNlKRecZQ0DpjvnXi6fMn6w449uF+6z8xsFICkyURvDX7jRDcqqbeZHW5j9iBJU8zsVye6H+cyiXvNnjSnAke+5CfpHyVtkLRJ0r1p5ddJqpFUJ+mR0KsPkpok/Yukt4GKdvbzIHBny0JJZZLWSXorDF8P5ZWSfi1pmaQdku6XdG2I4R1J54TlBkpaEmLeIOlPO+bX4ro1M/Mhakl0mOix4vvAXmBsKL8UeJSoet8LWAlMBEYSfc22KCy3ALg+jBvwnQz7qyb6xv2rwKQwXh3mnQwUh/HhQG0YryR6CnIGcBKwE7g3zJsDPBTGnwUmhPFhwJY2YpgF1AK1w6L2VD509aENqf8j7Q25fp+9J0qvxlcAiyWdT5TslwK/DcuVECVgOdErwxtCh4h9gV1hmcPAkpj7/RFwF8d+ArsI+FdJo8K2RqTN22BmH4U4fwesDuXvEP3RALgEOC+to8ZTJZWYWVP6js3sUaI/ZIyT/CWpHs6TvRVm9oakAcBAojP6PDN7JH0ZSbcCT5nZ3FY20dzOdXrLfb0q6UdELyqlfJ/oleQLiGoTzWnzPk8b/yJt+guO/nv2Ai6y8LjUOfBr9lZJOhfoDTQALwEzJZWEeUMkDQJeAa4M40g6XdJZOe7yR8A/pU2XAh+Z2RfAjBBLNlYDt6Ydz6gc43I9iJ/Zj+orqS6MC7ghnJ1XSxoJvBGqxU3AdWb2nqS7wvxewEHgZnJ4/8DMfikpvc/kBcASSdcTPeLc3/qabboN+FnocKQP0avOs7ONy/Ussbqlcj2fd0vVTbSRr3G6pfJqvHMJ4dX4PJP0ItDyawc/MLOXChGPSy5P9jyzo68aO1dQXo13LiE82Z1LCE925xLCk925hPAbdC4ydizU+pP2nszP7M4lhCe7cwnhye5cQniyO5cQnuzOJYQnu3MJ4cnuXEJ4sjuXEJ7sziWEJ7tzCeHJ7lxCeLI7lxCe7M4lhPcu6wCQ1AhsLXQcHWwAsLvQQXSwto7pLDMb2N6K3sTVpWzN1BVxdyOp1o/pKK/GO5cQnuzOJYQnu0t5tNAB5IEfUxq/QedcQviZ3bmE8GRPGEmXSdoqabukO1qZf5Kk58P830gq6/wosxPjmL4r6WNJdWH4m0LEmQ1JiyTtkrS5jfmS9HA45k2SxmTapid7gkjqDfwMmAKcB1wj6bwWi/01sMfMvgL8FPhx50aZnZjHBPC8mY0Kw+OdGmRungQua2f+FGB4GGYB/5Zpg57syXIhsN3MdpjZH4DngGktlpkGPBXGXwC+pfBh+i4qzjF1O2a2FviknUWmAYst8iZwmqQz2tumJ3uyDAE+TJuuD2WtLmNmh4C9QP9OiS43cY4J4C9DdfcFSWd2Tmh5Ffe4j/Bkd0mwAigzs3LgZY7WXBLFkz1ZdgLpZ7WhoazVZST1AUqBhk6JLjcZj8nMGszs8zD5ODC2k2LLpzj/lsfwZE+WDcBwSWdL+iPgamB5i2WWAzeE8SuBV61rv4yR8ZhaXMtOBbZ0Ynz5shy4PtyVvwjYa2YftbeCN4RJEDM7JOkW4CWgN7DIzN6V9EOg1syWA/8OPC1pO9ENoqsLF3FmMY/pNklTgUNEx/TdggUck6SfA5XAAEn1wD1AEYCZLQR+CVwObAcOADdm3GbX/qPtnOsoXo13LiE82Z1LCE925xLCk925hPBkdy4hPNmdSwhPducSwpPduYT4f0jKy3QqtxeFAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 144x144 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "ax,negRank0 = aggregateRankings(neg_rankings0,positive=False,maxLenAttributes=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-02-25T09:23:00.103370Z",
     "start_time": "2020-02-25T09:22:14.671851Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 38/38 [00:45<00:00,  1.19s/it]\n"
     ]
    }
   ],
   "source": [
    "pos_rankings0,pos_triangles0,flippedPredictions_pos0= explainSamples(\n",
    "    correctPredictions0,[tableA,tableB]\n",
    "    ,standard_model,originalClass=1,maxLenAttributeSet=4)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-02-25T09:23:27.137702Z",
     "start_time": "2020-02-25T09:23:26.884727Z"
    }
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAWgAAAD4CAYAAADB9HwiAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8li6FKAAAWhUlEQVR4nO3de5SVdb3H8fdHpAYEMW8tERTyQGIe5GpSRGDlpYtGh1KPtyMV0nGZtrKU0gW2dFnJUY9HSdEozFIzUtR1juKNkFJhUEQUCcXlEjMRBGEEFPB7/nieDZtxZPYwe8/+zczntdas2c/vuX2fDevDj99zU0RgZmbp2a3aBZiZWcMc0GZmiXJAm5klygFtZpYoB7SZWaJ2r3YBloZ99903evXqVe0yzNqdBQsWrIqI/Rqa54A2AHr16kVtbW21yzBrdyS98mHzPMRhZpYoB7SZWaIc0GZmifIYtJmV3ebNm1mxYgWbNm2qdinJqKmpoUePHnTs2LHkdRzQZlZ2K1asoGvXrvTq1QtJ1S6n6iKC1atXs2LFCnr37l3yeh7iMLOy27RpE/vss4/DOSeJffbZp8n/o3BAm1lFOJx3tCvfhwPazCxRHoM2s4rTpeXtTcfExp9jf+211/KrX/2KdevWMXr0aK677jpuuOEGOnfuzBlnnFHWeopNmjSJLl26cMEFFzR7Ww5oM2uTpkyZwkMPPcRDDz207S7Z8ePHV7mqpvEQh5m1OePHj2f58uUcf/zxrFmzZlv7pEmTmDx5MgAjR47kvPPOY8CAARx++OHMmzdv2zKnn346w4YNo0+fPtx0003b1r/yyisZOnQo/fv3Z+LEidvaL7/8cvr27cvw4cNZunRp2Y7DPWgza3NuuOEG7r//fh599FHuu+++D11uw4YNLFy4kDlz5jB27FgWL14MwKJFi3jiiSd45513GDhwIF/5yldYvHgxy5YtY968eUQEJ5xwAnPmzGGPPfbg9ttvZ+HChWzZsoVBgwYxePDgshyHA9oAWPCPBWUfJ7T2o5Qx4RSdcsopAIwYMYJ169axdu1aAE488UQ6depEp06dGDVqFPPmzWPu3LnMmjWLgQMHAlBXV8eyZctYv349o0ePpnPnzgCccMIJZavPAW1m7Vb9S98K0w21RwQTJkzg7LPP3mHeNddcU7H6PAZtZu3WHXfcAcDcuXPp1q0b3bp1A2DmzJls2rSJ1atXM3v2bIYOHcqxxx7LtGnTqKurA+C1115j5cqVjBgxgrvvvpuNGzeyfv167r333rLV5x60mVVcqkMgNTU1DBw4kM2bNzNt2rRt7f3792fUqFGsWrWKSy65hO7du9O9e3eWLFnCsGHDAOjSpQu33norgwYN4qSTTuKII45g//33Z+jQoWWrTxFpfnHWstRdwdmNL2fWkPoBvGTJEvr161elakozcuRIJk+ezJAhQ3ZoL+d1zPU19L1IWhARQxpa3kMcZmaJ8hCHmbVLs2fPbrB90qRJLVrHzrgHbWYV4eHTHe3K9+GANrOyq6mpYfXq1Q7pXOF50DU1NU1az0McZlZ2PXr0YMWKFbz55pvVLiUZhTeqNIUDuhWQ9HXgLqBfRLwgqRewBFgKCHgHOAt4Nf/pHRHrita/G7gtIu5o4dKtnerYsWOT3hxiDfMQR+twCjA3/13wUkQMiIgjgOnATyJiA/AAMLqwkKRuwHCgfFfPm1mLcEAnTlIXsoD9NnDyhyy2J1B4ZNdt9ZYbDTyQh7eZtSIe4kjficD9EfF3SaslDQZWA4dIWgh0BToDn86XfwC4WdI+EbGaLKyvq0bhZtY87kGn7xTg9vzz7Wwf5igMcRwCnA9MBYiI94B7gDGS9gUGkoX2B0gaJ6lWUi3uX5slxz3ohEnaGzga+FdJAXQAAri+3qL3AL8pmr4NuITsBOLMiNjc0PYjYip5sKu7fD2UWWLcg07bGOB3EXFwRPSKiJ7Ay0DPessNB14qmp4N9AHOIQtrM2uF3INO2ynAL+q1zQAmsH0MWsB7wHcKC0TE+5L+BHwL+EsL1WpmZean2Rngp9lZ86T6ONHWwE+zMzNrhRzQZmaJckCbmSXKAW1mligHtJlZohzQZmaJckCbmSXKAW1mlijfSWgADO4+mNqJtdUuw8yKuAdtZpYoB7SZWaIc0GZmiXJAm5klygFtZpYoB7SZWaIc0GZmiXJAm5klygFtZpYoB7SZWaIc0GZmiXJAm5klygFtZpYoB7SZWaIc0GZmiXJAm5klygFtZpYoB7SZWaIc0GZmiXJAm5klygFtZpYoB7SZWaIc0GZmiXJAm5klygFtZpYoB7SZWaIUEdWuwRKg7grOrnYV1pbERGdLKSQtiIghDc1zD9rMLFEOaDOzRDmgzcwS5YA2M0uUA9rMLFEOaDOzRDmgzcwS5YA2M0uUA9rMLFEOaDOzRDmgEyPpp5Kek7RI0kJJn5Z0vqTOJaxb1xI1mlnL2L3aBdh2koYBXwUGRcS7kvYFPgLcAdwKbKhmfWbWstyDTssBwKqIeBcgIlYBY4DuwKOSHpU0VtI1hRUkfVfS1fU3JOlHkubnPfFLW+oAzKx8HNBpmQX0lPR3SVMkfT4irgX+AYyKiFHAH4GvSeqYr3MWMK14I5KOAfoARwIDgMGSRtTfmaRxkmol1bpvbpYeB3RCIqIOGAyMA94E7pD0Hw0s8wjwVUmHAh0j4tl6mzom/3kaeAo4lCyw6+9vakQMiYghNDrCbWYtzWPQiYmIrcBsYLakZ4EzG1jsZuAnwAvAbxqYL+CKiLixUnWaWeW5B50QSZ+UVNzTHQC8AqwHuhYaI+JJoCfw78BtDWzqAWCspC75dg+UtH/FCjezinAPOi1dgP+RtBewBXiRbLjjFOB+Sf/Ix6EhG4seEBFr6m8kImZJ6gc8LgmgDjgNWNkCx2BmZdLkV15J+hjQMyIWVaYkK4Wk+4CrI+LhsmzPr7yyMvMrr0rT7FdeSZotaU9Je5OddLpJ0lXlLNJKI2kvSX8HNpYrnM0sTaUOcXSLiHWSvgPcEhETJbkHXQURsRboW+06zKzySj1JuLukA4BvAfdVsB4zM8uVGtA/I7sy4KWImC/pE8CyypVlZmYlDXFExJ3AnUXTy4F/q1RRZmZW+knCvpIelrQ4n+4v6eLKlmZm1r6VOsRxEzAB2AyQX2J3cqWKMjOz0q/i6BwR8/KbHgq2VKAeq5LB3QdTO7G22mWYWZFSe9CrJB0CBICkMcDrFavKzMxK7kGfA0wFDpX0GvAycGrFqjIzs5IDOiLii5L2AHaLiPWSeleyMDOz9q7UIY4ZABHxTkSsz9v+VJmSzMwMGulB5w+E/xTQTdI3imbtCdRUsjAzs/ausSGOT5K9xHQv4GtF7euB71aqKDMzaySgI2ImMFPSsIh4vIVqMjMzSj9JOE7SB3rMETG2zPWYmVmu1IAufoJdDTCa7E3TZmZWIaU+LGlG8bSk24C5FanIzMyAXX9pbB/ALyE1M6ugknrQktaT3eat/Pc/gQsrWJeZWbtX6hBH10oXYmZmOyr1JCH5jSrDyXrQj0XE3RWryszMSn5g/xRgPPAssBgYL+n6ShZmZtbeldqDPhroFxGFx41OB56rWFVmZlbyVRwvAgcVTffM28zMrEIae1jSvWRjzl2BJZLm5dOfBuZVvjwzs/arsSGOyS1ShZmZfUBjD0v6S0sVYmZmO2psiGNuRAwvulFl2yyyt6zsWdHqzMzascZ60MPz375RxcyshTV6FYekDpJeaIlizMxsu0YDOiK2AkslHdTYsmZmVj6l3qjyMeC5/DK7dwqNEXFCRaoyM7OSA/qSilZhZmYfUGpAfzkidni8qKRfAL4Mz8ysQpQ/XmPnC0lPRcSgem2LIqJ/xSqzFqXuCs6udhVmbUdMbDxbASQtiIghDc1r7Dro7wH/CRwiaVHRrK7A30qs08zMdkFjQxx/AP4PuAK4qKh9fUS8VbGqzMys0RtV3gbelrQlIl4pnifpdxFxekWrMzNrx0p93Oiniick7Q4MLn85ZmZWsNOAljQhfw5Hf0nrCj/AG8DMFqnQzKydamyI4wrgCklXAL8E+gI1hdkVrs3MrF0r9Tro5cAcoAewEDgKeJzsVVhmZlYBpY5Bfx8YCrwSEaOAgcDailVlZmYlB/SmiNgEIOmjEfEC8MnKlWVmZqUOcayQtBdwN/CgpDXAK42sY2ZmzVBSDzoiRkfE2oiYRPbgpF8DX9/ZOpK2Sloo6RlJT0n6TPPLbZyk2ZKW5vteKGnMLmxjZKXrlTRJ0gZJ+xe11VVyn2bWupTag96mCe8p3BgRAwAkHUt2N+LnixeQtHtEbGlqDSU4NSJqm7H+SKCOJtzOvovHsgr4IXBhYwuaWftT6hh0c+0JrIFtvdPHJN0DPJ+3nSZpXt7jvTF/i8s3JV2Vzz9P0vL88yck/bWpBUi6W9ICSc9JGlfUflzew39G0sOSegHjgR/k9XxOUi9Jj0halC9zUL7ubyXdIOlJ4JeSlknaL5+3m6QXC9MfYhpwkqS9m1BvnaQr8/aHJB2Z/69huaQT8mU65MvMz2tu8DFIksZJqpVUy4amfqNmVmlN7kE3QSdJC8mumz6AHS/JGwQcHhEvS+oHnAR8NiI2S5oCnArMAn6cL/85YLWkA/PPcxrZ9+8lbcw/fyEiVgNjI+ItSZ2A+ZJmkP0DdRMwIq9l73yZG4C6iJgMIOleYHpETJc0FriW7UM8PYDPRMRWSW/ntV8DfBF4JiLe3EmddWQhfR4wsd68D9SbH8cewCMR8SNJdwGXAV8CDgOmA/cA3wbejoihkj4K/FXSrIh4uXgHETEVmAr50+zMLCmVDOjiIY5hwC2SDs/nzSsKiy+Q3TY+XxJAJ2BlRPxTUhdJXYGeZA9uGkEW0H9uZN8NDXF8X9Lo/HNPoA+wHzCnUMtOHgA1DPhG/vl3ZDftFNyZvxYMsrCdSRbQY4HfNFInZGG/UNLkEupdDbwH3J+3Pwu8m//D9izQK28/huzuz8L4e7d8/R0C2szSVsmA3iYiHpe0L1kgQtFrswCR9U4nNLDq34CzgKXAY2ShN4xs3LZkkkaS9WiHRcQGSbPZfkdkcxW/AuxVSW9IOho4kqw3vVMRsVbSH4BzSqx3c2x/iPf7wLv5dt7Pn5EC2Xd6bkQ80KwjM7OqapExaEmHAh3IeoD1PQyMKVzNIGlvSQfn8x4DLiAb0ngaGEXWY3y7iSV0A9bkYXco2Z2QAE8AIyT1Luw7b19P9szrgr8BJ+efT83r+jA3A7eyY8+6MVcBZ7P9H8wPq7dUDwDfk9QRQFJfSXs0cRtmVmWVDOhOhUvdgDuAMxsKrIh4HrgYmKXspQAPko1ZQxaEPcmGIbYCrwJzd6GW+4HdJS0Bfk4WzOTjw+OAP0t6Jq8T4F5gdOEkIXAucFZe3+lkY8Yf5h6gC6UNb5DXsQq4C/jozuptgpvJTsA+JWkxcCMt9L8lMyufkl55ZaWTNAS4OiI+V+1amsKvvDIrr4q/8sqaRtJFwPcoYezZzKwxrTag80vMetdrvrCaJ8Yi4udkQxLbSPop8M16i94ZEZe3WGFm1iq12oCOiNGNL1V9eRA7jM2syVrqTkIzM2siB7SZWaIc0GZmiXJAm5klqtWeJLTyGtx9MLUTm/OEVjMrN/egzcwS5YA2M0uUA9rMLFEOaDOzRDmgzcwS5YA2M0uUA9rMLFEOaDOzRDmgzcwS5YA2M0uUA9rMLFEOaDOzRDmgzcwS5YA2M0uUA9rMLFEOaDOzRDmgzcwS5YA2M0uUA9rMLFEOaDOzRDmgzcwS5YA2M0uUA9rMLFEOaDOzRDmgzcwS5YA2M0uUIqLaNVgC1F3B2dWuwqz1iYnNy1BJCyJiSEPz3IM2M0uUA9rMLFEOaDOzRDmgzcwS5YA2M0uUA9rMLFEOaDOzRDmgzcwS5YA2M0uUA9rMLFEO6JykrZIWSnpG0lOSPlPh/c2WVFs0PUTS7Eru08xaFwf0dhsjYkBEHAFMAK4ox0YlddjJ7P0lHV+O/ZhZ2+OAbtiewJrChKQfSZovaZGkS4vaT5M0L+9531gIY0l1kv5L0jPAsJ3s50rgp/UbJfWS9Fjek9/Wm5c0UtJfJM2UtFzSzyWdmtfwrKRD8uX2kzQjr3m+pM+W52sxs5a0e7ULSEgnSQuBGuAA4GgASccAfYAjAQH3SBoBvAmcBHw2IjZLmgKcCtwC7AE8GRE/bGSfjwOjJY0C1he1rwS+FBGbJPUBbgMKT7s6AugHvAUsB26OiCMlnQecC5wP/DdwdUTMlXQQ8EC+zg4kjQPGAdCttC/JzFqOA3q7jRExAEDSMOAWSYcDx+Q/T+fLdSEL7P7AYGC+JIBOZMEKsBWYUeJ+LwMuBi4sausIXCdpQL6tvkXz5kfE63mdLwGz8vZngVH55y8Ch+V1AewpqUtE1BXvOCKmAlMhf9yomSXFAd2AiHhc0r7AfmS95isi4sbiZSSdC0yPiAkNbGJTRGwtcV+PSLoMOKqo+QfAG2S95d2ATUXz3i36/H7R9Pts//PcDTgqIorXM7NWxmPQDZB0KNABWE02PDBWUpd83oGS9gceBsbkn5G0t6SDd3GXlwE/LpruBrweEe8Dp+e1NMUssuGOwvEM2MW6zKyK3IPerjAGDVmv+cy8FzxLUj/g8XzIoA44LSKel3RxPn83YDNwDvBKU3ccEf8r6c2ipinADElnAPcD7zRxk98Hrpe0iOzPeA4wvql1mVl1+ZVXBviVV2a7yq+8MjNrhzzEUWGS7gJ612u+MCIeqEY9ZtZ6OKArLCJGV7sGM2udPMRhZpYoB7SZWaIc0GZmiXJAm5klyicJDYDB3QdTO7G28QXNrMW4B21mligHtJlZohzQZmaJckCbmSXKAW1mligHtJlZohzQZmaJckCbmSXKAW1mligHtJlZohzQZmaJckCbmSXKAW1mlii/1dsAkLQeWFrtOlrAvsCqahdRYT7G1uXgiNivoRl+3KgVLP2wV7+3JZJq2/px+hjbDg9xmJklygFtZpYoB7QVTK12AS2kPRynj7GN8ElCM7NEuQdtZpYoB7SZWaIc0Iak4yQtlfSipIuqXU85SJomaaWkxUVte0t6UNKy/PfHqlljc0nqKelRSc9Lek7SeXl7WzvOGknzJD2TH+eleXtvSU/mf2/vkPSRatdabg7odk5SB+B64HjgMOAUSYdVt6qy+C1wXL22i4CHI6IP8HA+3ZptAX4YEYcBRwHn5H92be043wWOjogjgAHAcZKOAn4BXB0R/wKsAb5dxRorwgFtRwIvRsTyiHgPuB04sco1NVtEzAHeqtd8IjA9/zwd+HqLFlVmEfF6RDyVf14PLAEOpO0dZ0REXT7ZMf8J4GjgT3l7qz/Ohjig7UDg1aLpFXlbW/TxiHg9//xP4OPVLKacJPUCBgJP0gaPU1IHSQuBlcCDwEvA2ojYki/SJv/eOqCtXYrs+tI2cY2ppC7ADOD8iFhXPK+tHGdEbI2IAUAPsv/1HVrlklqEA9peA3oWTffI29qiNyQdAJD/XlnleppNUkeycP59RPw5b25zx1kQEWuBR4FhwF6SCs8TapN/bx3QNh/ok58R/whwMnBPlWuqlHuAM/PPZwIzq1hLs0kS8GtgSURcVTSrrR3nfpL2yj93Ar5ENt7+KDAmX6zVH2dDfCehIenLwDVAB2BaRFxe5ZKaTdJtwEiyx1K+AUwE7gb+CBwEvAJ8KyLqn0hsNSQNBx4DngXez5t/QjYO3ZaOsz/ZScAOZJ3KP0bEzyR9guyk9t7A08BpEfFu9SotPwe0mVmiPMRhZpYoB7SZWaIc0GZmiXJAm5klygFtZpYoB7SZWaIc0GZmifp//mePMlHAtawAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 288x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "ax,posRank0 = aggregateRankings(pos_rankings0,positive=True,maxLenAttributes=1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Mine patterns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-02-25T09:42:31.256844Z",
     "start_time": "2020-02-25T09:42:30.939970Z"
    }
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAlAAAAFUCAYAAADrmwLOAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8li6FKAAAbSElEQVR4nO3de5RlZXnn8e9PQCRoBGKrKJcGh6XjYARtASMTRaNDvMRrkgHjOA4KM9EZTJxxWMbEBC8LWV7GZCYaFJVMmHg3ouAFUSHe0G5EQIGlIaggQhtvOBMvwDN/7F1SDVVUvXVO9T779PezVq06e59dXc/u7qrzO+9+32enqpAkSdLq3WnoAiRJksbGACVJktTIACVJktTIACVJktTIACVJktTIACVJktRo55UOSHIX4AJg1/7491TVy5IcALwD+BVgC/CsqvrZHf1Z97jHPWrjxo0TFy1JkrTetmzZ8t2q2rDUcysGKOCnwKOr6sdJdgE+neTDwB8Cr6+qdyR5E3Ac8MY7+oM2btzI5s2bG8uXJEna/pJ8Y7nnVryEV50f95u79B8FPBp4T7//DOApE9YpSZI0CquaA5VkpyQXAzcA5wL/APygqm7qD7kGuO/6lChJkjRbVhWgqurmqjoE2Ac4DHjAar9BkuOTbE6yeevWrWssU5IkaXY0rcKrqh8AnwQeDuyRZGEO1T7Atct8zWlVtamqNm3YsOQ8LEmSpFFZMUAl2ZBkj/7xbsBjgcvpgtQz+sOeDXxgvYqUJEmaJatZhbc3cEaSnegC17uq6kNJvgq8I8krgC8Bp69jnZIkSTNjxQBVVZcAhy6x/yq6+VCSJEk7FDuRS5IkNTJASZIkNTJASZIkNTJASZIkNVrNKrxR2XjS2UOXsCpXn/KEVR03hvNZ7blIkjQvHIGSJElqZICSJElqZICSJElqZICSJElqZICSJElqZICSJElqZICSJElqZICSJElqZICSJElqZICSJElqZICSJElqZICSJElqZICSJElqZICSJElqZICSJElqZICSJElqZICSJElqZICSJElqZICSJElqZICSJElqZICSJElqZICSJElqZICSJElqZICSJElqZICSJElqZICSJElqZICSJElqZICSJElqZICSJElqZICSJElqZICSJElqtGKASrJvkk8m+WqSryQ5sd//p0muTXJx//H49S9XkiRpeDuv4pibgBdV1UVJ7gZsSXJu/9zrq+o161eeJEnS7FkxQFXVdcB1/eMbk1wO3He9C5MkSZpVTXOgkmwEDgUu7He9IMklSd6aZM8p1yZJkjSTVh2gktwVeC/wwqr6EfBG4H7AIXQjVK9d5uuOT7I5yeatW7dOoWRJkqRhrSpAJdmFLjydWVXvA6iq66vq5qq6BXgzcNhSX1tVp1XVpqratGHDhmnVLUmSNJjVrMILcDpweVW9btH+vRcd9lTgsumXJ0mSNHtWswrvEcCzgEuTXNzvewlwTJJDgAKuBk5YlwolSZJmzGpW4X0ayBJPnTP9ciRJkmafncglSZIaGaAkSZIaGaAkSZIaGaAkSZIaGaAkSZIaGaAkSZIaGaAkSZIaGaAkSZIaGaAkSZIaGaAkSZIaGaAkSZIaGaAkSZIaGaAkSZIaGaAkSZIaGaAkSZIaGaAkSZIaGaAkSZIaGaAkSZIaGaAkSZIaGaAkSZIaGaAkSZIaGaAkSZIaGaAkSZIaGaAkSZIaGaAkSZIaGaAkSZIaGaAkSZIaGaAkSZIaGaAkSZIaGaAkSZIaGaAkSZIaGaAkSZIaGaAkSZIaGaAkSZIaGaAkSZIarRigkuyb5JNJvprkK0lO7PfvleTcJF/rP++5/uVKkiQNbzUjUDcBL6qqBwJHAM9P8kDgJOC8qjoIOK/fliRJmnsrBqiquq6qLuof3whcDtwXeDJwRn/YGcBT1qtISZKkWdI0ByrJRuBQ4ELgXlV1Xf/Ud4B7TbUySZKkGbXqAJXkrsB7gRdW1Y8WP1dVBdQyX3d8ks1JNm/dunWiYiVJkmbBqgJUkl3owtOZVfW+fvf1Sfbun98buGGpr62q06pqU1Vt2rBhwzRqliRJGtRqVuEFOB24vKpet+ips4Bn94+fDXxg+uVJkiTNnp1XccwjgGcBlya5uN/3EuAU4F1JjgO+AfzO+pQoSZI0W1YMUFX1aSDLPP2Y6ZYjSZI0++xELkmS1MgAJUmS1MgAJUmS1MgAJUmS1MgAJUmS1MgAJUmS1MgAJUmS1MgAJUmS1MgAJUmS1MgAJUmS1MgAJUmS1MgAJUmS1MgAJUmS1MgAJUmS1MgAJUmS1MgAJUmS1MgAJUmS1MgAJUmS1MgAJUmS1MgAJUmS1MgAJUmS1MgAJUmS1MgAJUmS1MgAJUmS1MgAJUmS1MgAJUmS1MgAJUmS1MgAJUmS1MgAJUmS1MgAJUmS1MgAJUmS1MgAJUmS1MgAJUmS1MgAJUmS1MgAJUmS1MgAJUmS1GjFAJXkrUluSHLZon1/muTaJBf3H49f3zIlSZJmx2pGoN4OHL3E/tdX1SH9xznTLUuSJGl2rRigquoC4HvboRZJkqRRmGQO1AuSXNJf4ttzahVJkiTNuLUGqDcC9wMOAa4DXrvcgUmOT7I5yeatW7eu8dtJkiTNjjUFqKq6vqpurqpbgDcDh93BsadV1aaq2rRhw4a11ilJkjQz1hSgkuy9aPOpwGXLHStJkjRvdl7pgCR/CzwKuEeSa4CXAY9KcghQwNXACetYoyRJ0kxZMUBV1TFL7D59HWqRJEkaBTuRS5IkNTJASZIkNTJASZIkNTJASZIkNTJASZIkNTJASZIkNTJASZIkNTJASZIkNTJASZIkNTJASZIkNTJASZIkNTJASZIkNTJASZIkNTJASZIkNTJASZIkNTJASZIkNTJASZIkNTJASZIkNTJASZIkNTJASZIkNTJASZIkNTJASZIkNdp56AK0Y9l40tlDl7Ciq095wtAlSJJmnCNQkiRJjQxQkiRJjQxQkiRJjQxQkiRJjQxQkiRJjQxQkiRJjQxQkiRJjQxQkiRJjQxQkiRJjQxQkiRJjQxQkiRJjQxQkiRJjQxQkiRJjVYMUEnemuSGJJct2rdXknOTfK3/vOf6lilJkjQ7VjMC9Xbg6NvsOwk4r6oOAs7rtyVJknYIKwaoqroA+N5tdj8ZOKN/fAbwlCnXJUmSNLPWOgfqXlV1Xf/4O8C9plSPJEnSzJt4EnlVFVDLPZ/k+CSbk2zeunXrpN9OkiRpcGsNUNcn2Rug/3zDcgdW1WlVtamqNm3YsGGN306SJGl2rDVAnQU8u3/8bOAD0ylHkiRp9q2mjcHfAp8D7p/kmiTHAacAj03yNeA3+m1JkqQdws4rHVBVxyzz1GOmXIskSdIo2IlckiSpkQFKkiSpkQFKkiSpkQFKkiSp0YqTyCUtb+NJZw9dwoquPuUJqzpuDOcCqz8fSVpPjkBJkiQ1MkBJkiQ1MkBJkiQ1MkBJkiQ1MkBJkiQ1MkBJkiQ1MkBJkiQ1MkBJkiQ1spGmpLlkY1BJ68kRKEmSpEYGKEmSpEYGKEmSpEYGKEmSpEYGKEmSpEYGKEmSpEYGKEmSpEb2gZKkERhDXyt7WmlH4giUJElSIwOUJElSIwOUJElSIwOUJElSIwOUJElSIwOUJElSIwOUJElSIwOUJElSIxtpSpK2OxuDauwcgZIkSWpkgJIkSWpkgJIkSWpkgJIkSWo00STyJFcDNwI3AzdV1aZpFCVJkjTLprEK76iq+u4U/hxJkqRR8BKeJElSo0kDVAEfS7IlyfHTKEiSJGnWTXoJ78iqujbJPYFzk1xRVRcsPqAPVscD7LfffhN+O0mSpOFNNAJVVdf2n28A3g8ctsQxp1XVpqratGHDhkm+nSRJ0kxYc4BKsnuSuy08Bh4HXDatwiRJkmbVJJfw7gW8P8nCn/N/quojU6lKkiRphq05QFXVVcCDp1iLJEnSKNjGQJIkqZEBSpIkqZEBSpIkqZEBSpIkqZEBSpIkqZEBSpIkqZEBSpIkqZEBSpIkqZEBSpIkqZEBSpIkqZEBSpIkqZEBSpIkqZEBSpIkqdHOQxcgSdKYbTzp7KFLWJWrT3nCqo6bt/NZL45ASZIkNTJASZIkNTJASZIkNTJASZIkNTJASZIkNTJASZIkNTJASZIkNTJASZIkNTJASZIkNTJASZIkNTJASZIkNTJASZIkNTJASZIkNTJASZIkNTJASZIkNTJASZIkNTJASZIkNTJASZIkNTJASZIkNTJASZIkNTJASZIkNTJASZIkNZooQCU5OsmVSb6e5KRpFSVJkjTL1hygkuwE/C/gN4EHAsckeeC0CpMkSZpVk4xAHQZ8vaquqqqfAe8AnjydsiRJkmbXJAHqvsC3Fm1f0++TJEmaa6mqtX1h8gzg6Kp6br/9LODwqnrBbY47Hji+37w/cOXayx3MPYDvDl3EFHk+s2uezgU8n1k3T+czT+cCns+s2L+qNiz1xM4T/KHXAvsu2t6n37eNqjoNOG2C7zO4JJuratPQdUyL5zO75ulcwPOZdfN0PvN0LuD5jMEkl/C+CByU5IAkdwb+LXDWdMqSJEmaXWsegaqqm5K8APgosBPw1qr6ytQqkyRJmlGTXMKjqs4BzplSLbNs1Jcgl+D5zK55OhfwfGbdPJ3PPJ0LeD4zb82TyCVJknZU3spFkiSpkQFKkiSpkQFKkpaQ5E5Jfm3oOtZLknsPXYM0Zs6B2gEkCfBM4MCqOjnJfsC9q+oLA5e2Q0uy1x09X1Xf2161aGlJvlRVhw5dx3pIclFVPWToOiaRZFfg6cBGFi2KqqqTh6pJt5r3156JVuHNoyQfBJZNlVX1W9uxnGn5S+AW4NHAycCNwHuBhw1Z1FokuZGl/30CVFX98nYuaRJb6M4lwH7A9/vHewDfBA4YrrR2c/qzc16SpwPvq/l7t5mhC5iCDwA/pPtZ+unAtUwsydOAVwP3pPv3GePvtcXm5rVnKQao23tN//lpwL2Bv+m3jwGuH6SiyR1eVQ9J8iWAqvp+3/x0dKrqbkPXMC1VdQBAkjcD7+/bgpDkN4GnDFnbGr1m5UNG5wTgD4Gbk/wz439BW+zNQxcwBftU1dFDFzFFpwJPqqrLhy5kSubmtWcpBqjbqKrzAZK89jZt5z+YZPNAZU3q50l2oh8dSLKB7l3B6CW5J3CXhe2q+uaA5azVEVX1vIWNqvpwklOHLGgtFn52AJLsBuxXVWO89+UvzFNgv62q+suha5iCzyZ5UFVdOnQhU3L9HIUnmOPXHnAS+R3ZPcmBCxtJDgB2H7CeSfw58H7gnkleCXwaeNWwJU0myW8l+Rrwj8D5wNXAhwctau2+neSlSTb2H38EfHvootYqyZOAi4GP9NuHJBnlbZ7S+b0kf9xv75vksKHr0i8cCWxJcmWSS5JcmuSSoYuawOYk70xyTJKnLXwMXdQE5u61ZzEnkS8jydF0nVOvohu23x84oao+Omhha5TkAcBj6M7lvLG/y0nyZbrr6h+vqkOTHAX8XlUdN3BpzfrJ5C8Dfp3undoFwMljnUSeZAvdv82nFiZgJ7m0qh40bGXtkryRfg5HVf3LJHsCH6uquZjDMXZJ9l9qf1V9Y3vXMg1J3rbE7qqq/7Ddi5mSeXvtWcwAdQf6FR4P6DevqKpRTVKc51VeC3f27oPUoVV1S5IvV9WDh65trZLsXlX/d+g6JpXk81V1xOIVbEkuqapfHbq2Vgsr1W5zLqP+fzZvkjwY+Nf95t9X1ZeHrEe3WuY16Maq+vl2L2YdOAdqGUl+iW7y6P5V9bwkByW5f1V9aOjaGixe5QW3rpBK//jApb5oJH6Q5K50ozVnJrkBGGX46HsNvQW4K7Bf/4JwQlX9/rCVrdlXkhwL7JTkIOC/AJ8duKa1mus5HGOX5ETgecD7+l1/k+S0qvqLActasyR3AY4D/hXbzu0c6wjURcC+bLvC+DtJrgeeV1VbhixuUo5ALSPJO+kCyL+rqoP7QPXZqjpk4NLWpH8ncBDb/lCev/xXzLYkuwM/ofuhfCZwd+DMqvqnQQtbgyQXAs8Azlo0ynFZVR08bGVr0/+s/BHwuH7XR4FXVNVPhqtqbZI8E/hd4KHA2+n+nV5aVe8esi51+vlOD18Yue1/L3xujKOdAEneDVwBHEu37P+ZwOVVdeKgha1Rv8L4PQtTX5I8jq5v19uAN1TV4UPWNykD1DIWXSIa/dB9kucCJwL70E3uPYIuDD5m0MIEdAGqqg6fk/9rOwGvrqr/OnQt07JoDgfAJ+ZpDsfYJbkUeNhCOO9HcL44xvl2cGvj1oVL3kl2obssecTQta3FUnMfF53bxWMdkFjgJbzl/axfir0wdH8/xtuo7US6xmWfr6qj+heEUa6ESPLpqjpyiYaaY+7P863+Ml71vzBPBEb5Il1VNyc5cug6puyXgIXLeLsNXIu29TbgwiTvp/sd8GTg9GFLmsjC3KAfJDkY+A5dU82xui7Jfwfe0W//LnB9/0Zr9JfCDVBLSBLgTXTLsPdNcibwCODfD1nXBH5SVT9JQpJdq+qKJPcfuqi1qKoj+8/z1J/nPwJvAO4LXAt8DHj+oBVN5kt924J3s2heWlW9b/kvmU1J/gT4bbruyQHeluTdVfWKYSsTQFW9Lsmn6NoZFPCcqvrSsFVN5LR+pedLgbPo5kX+ybAlTeRYuhXGf9dvf6bftxPwO0MVNS1ewltGPzT8KLrLXaEbvfnuoEWtUf/u7DnAC+mWl38f2KWqHj9oYWvUv3v5SlU9YMWDtd3N01LsJFcCD150iWg34OKqGuUbkHmU5CF0q/BuAT5TVRcNXJJ2EI5ALe8iuhsgnj10IZOqqqf2D/80ySfpJlx/ZMCSJtJfJroyyX4j7Ty+jSR/vsTuHwKbq+oD27ueSfTh9pKqev3QtUzJt+kWXixMgN+VbpRQM2DeRgiTvAo4tap+0G/vCbyoql46bGVtMp/3xbwdR6CWkeQK4F8A36C7DLEwx2aUqzvmTZILgEOBL7DtZaLR/WAmOY2u39jCyq6n03VY/xXgqqp64VC1rUWSL1TVqLt1J/kLuheA/ejmD57bbz8W+EJVjbk79NyYtxHCxQtJFu27qKoeMlRNa5Hkkf3DJe8pW1V/MEhhU+YI1PL+zdAF6A798dAFTNGvAo+oqpvhF92v/55uXscY7/H1mST/E3gn24bbMV1aWbjv5Ra6W1Es+NT2L0V3YN5GCHfq56n+FH4RCHcduKZmc3pP2dsxQC1jrLcC2FFU1fn9bRwOqqqP972Hdhq6rjXak26y6A/77d2BvfpLlWNc+bmwNPnkRfuKbv7dKFTVGUPXoOUtGiH8IV3j1m1GCIesbUJnAuctmkf4HGDM/xd3T3JgVV0Fo7+n7O0YoDRKSZ4HHA/sBdyPbgXbm7i1X8+YnApc3K8mCt098V7VNwX8+JCFrUVVHTV0DdOS5InAy+nuhbkz426XMU/mcoSwql7dNwdd+D328rHef7X3B8CnkmxzT9lhS5oe50BplJJcDBwGXDj2G9YCJLkP8Cy6/k93Ba6pqguGrWrtkjyB29+O4uTlv2I2Jfk63TyOS8tfljOlX7Dw11X1zKFr0fIy8nvK3hFHoDRWP62qn3UtuyDJztzBqo9Ztkyn+M8xokteiyV5E13zyaPo7vH3DMZ7WeVbwGWGp9nTX+LeP8mdq+pnQ9cziXlrEJzk0VX1iSS3XWxxvySj7Am3FAOUxur8JC8BdkvyWOD3gQ8OXNNazU2n+N6v9bdquKSq/izJa4EPD13UGr0YOCfJ+Sy6E0FVvW64krTIVXSLFs5i2wULo/r3mcMGwY8EPgE8aYnniltv/jxqBiiN1Ul0dy2/lO6a+jl0ox1jNDed4nv/3H/+f/2lyX8C9h6wnkm8Evgx3aXIOw9ci27vH/qPOwGjDh/z1CC4ql6W5E7Ah6vqXUPXs14MUBqlqrolyRnAhXTvaK4c8WWWa5LsQXe7g3OTfJ+u/9hYfag/n1PpJvnCeMPtfarq4KGL0NKq6s+GrmFa5q1BcP87+sXA3AYoJ5FrlPpJym+ie/cZ4ADghKoa66Ui4BcN6O4OfGSs8zr63jX/ie72GkXX0+qNC80OxyTJqcDHq+pjQ9ei2+vvrHC7F7GqGuv8wblpEAyQ5BTgu9y+J9z3BitqigxQGqW+U/wTq+rr/fb9gLPnYfh77JK8C7iRW7sPHwvcvapGd/PQflLv7nTzn37OSCf1zqskD120eRe6Lv43VdWLByppIos6eG9joTHl2CT5R5YOuAcOUM7UeQlPY3XjQnjqXUX3oq3hHVxVD1y0/ckkXx2smglU1d2S7AUcxKKWDJoNVbXlNrs+k2SsKz7nrUEwwAPpFvgcya2j0W8atKIpMkBprDYnOYfu+nrR3VD0iwvLZudlmexIXZTkiKr6PECSw7m18eGoLNNi4rOMs2Hr3OnD7YI7AQ+luwQ+SnPWIBi6Luo/AhZumH5sv290o9FLMUBprO4CXE+3XBZgK7Ab3bLZuVkmOyZJLqX7u98F+GySb/bb+wNXDFnbBOatxcS82UL3fyzATXQ34T5u0Iom83z6BsEAVfW1JPcctqSJzM1o9FIMUBqlqnrO0DXodp44dAHrYN5aTMyVqjpg6BqmbG4aBPfmZjR6KQYojVJ/U8r/DGxk0f/jsa5WmQdzegPueWsxMVeS7EK34vPX+12fAv6qqn4+WFGTmYsGwXM6Gn07rsLTKCX5MnA6XSPNWxb2j3W1imbfPLSYmDdJ3kL3In1Gv+tZwM1V9dzhqlq7vvnkccDj6C5LfhR4y9h63PUT4Zc1L2+2DFAapSQXVtXhQ9chaThJvlxVD15pn7Qe7jR0AdIavSHJy5I8PMlDFj6GLkrSdnVz3wMOgCQHAjcPWM9EkjwxyZeSfC/Jj5LcmORHQ9elpTkHSmP1ILrh+kdz6yW86rcl7Rj+G93Krqv67Y3AmBeY/A/gacClY7tstyMyQGmsfhs40Lko0g7tM8Bf0fVJ+gHdnKHPDVrRZL4FXGZ4GgcDlMbqMmAP4IahC5E0mL+ma9T48n77WOB/073BGqMXA+ckOZ/u9kEAVNXrhitJyzFAaaz2AK5I8kW2/UVjGwNpxzFvjRpfCfyYrlHwnQeuRSswQGmsXjZ0AZIGN2+NGu9TVQcPXYRWxwClUbLfk7TjmuNGjeckeVxVfWzoQrQy+0BpVJJ8uqqOTHIj297iIEBV1S8PVJqk7WReGzX2v9d2p5uW8HP8vTbTDFCSJM2IJHsBB9HNgwIccZ9VXsKTJGkGJHkucCKwD3AxcATwWbo2DZoxdiKXJGk2nAg8DPhGVR0FHAr8cNiStBwDlCRJs+EnVfUTgCS7VtUVwP0HrknL8BKeJEmz4ZokewB/B5yb5PvAKCfE7wicRC5J0oxJ8kjg7sBHvGXVbDJASZIkNXIOlCRJUiMDlCRJUiMDlCRJUiMDlCRJUiMDlCRJUqP/DxF4BL5q26iFAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 720x360 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plotMostFrequentTokens(flippedPredictions_pos0,('Beer_Name',),10)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## RNN Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-02-18T10:58:26.728179Z",
     "start_time": "2020-02-18T10:58:26.523906Z"
    }
   },
   "outputs": [],
   "source": [
    "rnn_model = dm.MatchingModel(attr_summarizer='rnn')\n",
    "rnn_model.load_state('models/beer_rnn.pth')\n",
    "rnn_model = rnn_model.to('cuda')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-02-18T11:01:06.181515Z",
     "start_time": "2020-02-18T11:01:05.481313Z"
    }
   },
   "outputs": [],
   "source": [
    "predictions_rnn0 = wrapDm(augmented_test,rnn_model,ignore_columns=['ltable_id','rtable_id','id','label'])\n",
    "tp_group = augmented_test[(predictions_rnn0[:,1]>=0.5)& (augmented_test['label'] == 1)]\n",
    "tn_group = augmented_test[(predictions_rnn0[:,0] >=0.5)& (augmented_test['label']==0)]\n",
    "correctPredictions_rnn0 = pd.concat([tp_group,tn_group])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-02-18T11:01:40.263869Z",
     "start_time": "2020-02-18T11:01:19.013243Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 50/50 [00:21<00:00,  2.37it/s]\n"
     ]
    }
   ],
   "source": [
    "neg_rankings_rnn0,neg_triangles_rnn0,flippedPredictions_rnn_neg0 = explainSamples(\n",
    "    correctPredictions_rnn0,[tableA,tableB],rnn_model,\n",
    "    originalClass=0,maxLenAttributeSet=4)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-02-18T11:02:03.086739Z",
     "start_time": "2020-02-18T11:01:40.880187Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 50/50 [00:22<00:00,  2.27it/s]\n"
     ]
    }
   ],
   "source": [
    "pos_rankings0_rnn,pos_triangles0_rnn,flippedPredictions_pos0_rnn= explainSamples(\n",
    "    correctPredictions_rnn0,[tableA,tableB]\n",
    "    ,rnn_model,originalClass=1,maxLenAttributeSet=4)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-02-18T11:02:26.921379Z",
     "start_time": "2020-02-18T11:02:26.476785Z"
    }
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAvQAAAI/CAYAAAAY801hAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8li6FKAAAgAElEQVR4nOzde7RddX3v/fcHQk0ggIfbOFwNWhA8HAgkIOFWgor61Kr0YJGCekxbxIci+lREqg7xFAdWqVhqeQA5KGhF1FQujgqpQAQEyQUDUQERPDxSrUC4SIBADN/njzV3stjs7EvYO2vP7PdrjIzM+ZuX33eunbHzWb/1m3OlqpAkSZLUThv1ugBJkiRJ685AL0mSJLWYgV6SJElqMQO9JEmS1GIGekmSJKnFDPSSJElSi03qdQFSr2yzzTY1bdq0XpchSZI0pMWLFz9SVdsOtM1Arwlr2rRpLFq0qNdlSJIkDSnJA2vb5pQbSZIkqcUM9JIkSVKLGeglSZKkFnMOvSRJknpi5cqVPPjgg6xYsaLXpYwbkydPZqeddmKTTTYZ9jEGekmSJPXEgw8+yOabb860adNI0utyeq6qWLZsGQ8++CC77rrrsI9zyo0kSZJ6YsWKFWy99daG+UYStt566xF/YmGglyRJUs8Y5l9oXV4PA70kSZImrHPPPZc999yTHXfckb/+678G4Pzzz+fSSy8d037POOMMzj777FE5l3PoJUmSND6M9mh91ZC7nHfeeXz/+9/n+9///uovnDzxxBNHt44x5gi9JEmSJqQTTzyR+++/nze/+c089thjq9u7R88PP/xwTjnlFKZPn85ee+3FggULVu/zrne9i1mzZrHbbrvxpS99afXxn/vc59h///3Ze++9+eQnP7m6/dOf/jS77747hxxyCPfcc8+oXYcj9JIkSZqQzj//fK655hpuuOEGvvvd7651v6effpolS5Zw4403MmfOHH7yk58AcOedd/KjH/2Ip556in333Zc//uM/5ic/+Qn33nsvCxYsoKp461vfyo033shmm23GN77xDZYsWcLvf/979ttvP2bMmDEq12GglyRJkgZx7LHHAnDYYYfxu9/9jscffxyAt73tbUyZMoUpU6Ywe/ZsFixYwM0338y8efPYd999AVi+fDn33nsvTz75JEcddRSbbropAG9961tHrT4DvSauxYtHf66e1m4Y8xglSRqP+j95pm99oPaq4vTTT+d973vfC7Z94QtfGLP6nEMvSZIkDeLyyy8H4Oabb2bLLbdkyy23BODKK69kxYoVLFu2jPnz57P//vvzxje+kYsvvpjly5cD8B//8R889NBDHHbYYVxxxRU888wzPPnkk1x99dWjVp8j9JIkSdIgJk+ezL777svKlSu5+OKLV7fvvffezJ49m0ceeYRPfOIT7LDDDuywww7cddddzJo1C4CpU6fyta99jf32249jjjmGffbZh+222479999/1OpL+TG4JqiZSS3qdRETib9rJEn93HXXXey55569LmNQhx9+OGeffTYzZ858QfsZZ5zB1KlT+fCHPzzqfQ70uiRZXFUzB9rfKTeSJElSiznlRpIkSVqL+fPnD9h+xhlnrNc6BuMIvSRJktRiBnpJkiT1jPdzvtC6vB4GekmSJPXE5MmTWbZsmaG+UVUsW7aMyZMnj+g459BLkiSpJ3baaScefPBBHn744V6XMm5MnjyZnXbaaUTHjFmgT7IKWAoEWAX8dVXdMob9zQem9j3OJ8lM4OyqOnys+mz62QS4rar2W9/X3FXDfGB74Jmm6cyq+vYIz3E48NwY/4zOAD4CTKuqh5q25VU1daz6lCRJ49cmm2zCrrvu2usyWm8sp9w8U1XTq2of4HTgrNE4aZKNB9m8XZI3j0Y/I3AI8MNmechrTjJWb6KOa/qePtIw3zgcOGgkB6zjtTwC/M06HCdJkqQBrK859FsAj/WtJDk1ycIkdyb5VFf78UkWJFmS5IK+8J5keZJ/SHIHMGuQfj4HfKx/Y5JpSW5Kcnvz56Cm/fAkP0hyZZL7k3wmyXFNDUuTvKrZb9skc5uaFyY5uOv0bwK+N9g1N/3clOQq4Gdru9Yk70jy+Wb7KUnub5ZfmeSHA/QxqCRXJFmc5KdJTuhqf1PzOtyR5Lok04ATgQ819RzavGbXNz+j65Ls0hz7lSTnJ7kN+GySe5Ns22zbKMkv+tbX4mLgmCRbjaDe5Uk+17R/P8kBSeY3P7O3Nvts3OzT9+/qfSN9vSRJktpoLOfQT0myBJhMZzrIEQBJjgR2Aw6gMzXlqiSHAQ8DxwAHV9XKJOcBxwGXApvRmdYy1MjurcBRSWYDT3a1PwS8oapWJNkNuAzo+6atfYA9gUeB+4GLquqAJKcAJwMfBP4ROKeqbm6C7bXNMQCzgb43JQNec2M/YK+q+mWSPddyrfPoTEkBOBRYlmTHZvnGIa79X5L0Tbl5XVUtA+ZU1aNJpgALk8yl8ybuS8BhTS1bNfucDyyvqrMBklwNXFJVlySZA5wLvL05/07AQVW1KskTTe1fAF4P3FFVg02EW04n1J8CfLLfthfV21zHZsD1VXVqku8AZwJvAF4DXAJcBfwF8ERV7Z/kZcAPk8yrql92d9C8UTgBYJchXlBJkqQ2GMtA/0xVTQdIMgu4NMlewJHNnx83+02lE/D3BmbQCXIAU+gEcejMR587zH7PBD4OnNbVtgnwxSTTm3Pt3rVtYVX9pqnzPjqhGjpz4Wc3y68HXtPUBbBFkqnAlsCjVfX0ENcMsKArXL5uoGutqv9MMjXJ5sDOwNeBw+gE+n8d4rqPq6pF/do+kOSoZnlnOq/ztsCNfbVU1aNrOd8s4E+b5a8Cn+3a9q2qWtUsXwxcSSfQzwG+PESd0HlzsCTJ2cOodxnwHHBN074UeLZ5I7QUmNa0HwnsneToZn3L5vgXBPqquhC4EGBm4i31kiSp9dbLU26q6tYk29AJkwHOqqoLuvdJcjKdEeHTBzjFiq4AOVRf1yc5Eziwq/lDwG/pjMZvBKzo2vZs1/LzXevPs+b12Qg4sKq6jyPJMXRG6weqo/uaAZ7qPpS1X+stwHuBe4Cb6ITkWYxw3nk6N7m+HphVVU+nc+PsyJ6BtHarr6WqfpXkt0mOoPOpy3FDHVxVjyf5OnDSMOtdWWueZ7X6Z1RVz2fNPP4AJ1fVgD8PSZKkDdV6mUOfZA9gYzqjrdcCc5oRbpLsmGQ74Drg6GaZJFslecU6dnkma6auQGe09jdV9TzwrqaWkZhHZ/pN3/VMbxbXNn++/zX3N9i13gR8mM4Umx/T+ZTg2ap6YoQ1bwk81oTjPVjzBudHwGFJdu3ru2l/Eti86/hbgHc2y8c1da3NRcDXeOHI/VA+D7yPNW+a1lbvcF0LvD+dpw6RZPckm43wHJIkSa2zPubQQ2f09D1N2JvXzCG/tZlushw4vqp+luTjzfaNgJV0RnAfGGnHVfVvSbrncZ8HzE3ybjpTN54a+Mi1+gDwz0nupPOa3ZjkJOAPq+rurv0GvOauqTp99Q12rTfRmW5yY3Psr4C7GblrgBOT3EVntP9HTd8PN/PI/7Xp+yE689GvBr6d5G103rycDHw5yal07m947yB9XUVnqs1wptvQ1PFIMx/+Q4PVOwIX0Zl+c3s6L/jDrJnzL0mStMGK38y1bpIcQueNyIm9rqXX0nnm/zlVdWivaxmJmcmLbjrQGPJ3jSRJ6yzJ4r7vW+rPb4pdR1V1M3Bzr+votSQfBd7PMObOS5IkafS1boS+mabR/yvFTpsIN0O25dqTfAx4R7/mb1XVp3tRz9o4Qr+etex3jSRJ48lgI/StC/TSaDHQr2f+rpEkaZ0NFujX1zfFSpIkSRoDBnpJkiSpxQz0kiRJUosZ6CVJkqQWM9BLkiRJLeZz6DVxzZgBi3zOjSRJajdH6CVJkqQWM9BLkiRJLWaglyRJklrMQC9JkiS1mIFekiRJajEDvSRJktRiBnpJkiSpxQz0kiRJUosZ6CVJkqQWM9BLkiRJLWaglyRJklrMQC9JkiS1mIFekiRJajEDvSRJktRiBnpJkiSpxQz0kiRJUosZ6CVJkqQWM9BLkiRJLWaglyRJklrMQC9JkiS1mIFekiRJajEDvSRJktRiBnpJkiSpxQz0kiRJUosZ6CVJkqQWM9BLkiRJLTap1wVIPbN4MSS9rmJ8qep1BZIkaYQcoZckSZJazEAvSZIktZiBXpIkSWoxA70kSZLUYgZ6SZIkqcUM9JIkSVKLGeglSZKkFjPQS5IkSS1moJckSZJazEAvSZIktdiECvRJViVZkuSOJLcnOWg99Ts/yT1N30uSHL0O5zh8rOtNckaSp5Ns19W2fCz77Orne0l2apa3SbIyyYn99vk/SZY2r+HSJG9r2m9I8sZ++34wyf+7PmqXJEnqpQkV6IFnqmp6Ve0DnA6c1X+HJJPGqO/jmr6nV9W31+H4w4ERBfp1vJZHgL9Zh+PWWZIpwNZV9WDT9A7gR8CxA+w+u6qmA0cD5zZtlwHv7LffO5t2SZKkDdpEC/TdtgAeg9Wj3zcluQr4WdN2fJIFzWjwBUk2TvKOJJ9vtp+S5P5m+ZVJfjjSApJckWRxkp8mOaGr/U3NJwh3JLkuyTTgROBDTT2HJpmW5Pokdzb77NIc+5Uk5ye5DfhsknuTbNts2yjJL/rW1+Ji4JgkW42g3uVJPte0fz/JAc2nEvcneWuzz8bNPgubmt/XderDgfld68fSeVOxY9+o/QBW//yAbwN/nOQPmr6mATsANw1ynZIkSRuEsRqNHq+mJFkCTAa2B47o2rYfsFdV/TLJnsAxwMFVtTLJecBxwDzgI83+hwLLkuzYLN84RN//kuSZZvl1VbUMmFNVjzYj1AuTzKXzJutLwGFNLVs1+5wPLK+qswGSXA1cUlWXJJlDZ7T67c35dwIOqqpVSZ5oav8C8Hrgjqp6eJA6l9MJ9acAn+y37UX1NtexGXB9VZ2a5DvAmcAbgNcAlwBXAX8BPFFV+yd5GfDDJPOq6pfAm4ErmuvaGdi+qhYk+Wbzc/iHrhpuSBLglcCfATQ1LWjOcyWd0flvVlX1v7jmjcgJALsM8iJIkiS1xUQboe+bcrMH8Cbg0iYcAixowiXA64AZdELrkmb9lVX1n8DUJJsDOwNfBw6jE+iHGg3unnKzrGn7QJI76Ewv2RnYDTgQuLGvlqp6dC3nm9X0D/BV4JCubd+qqlXN8sXAu5vlOcCXh6gTOm8O3tNcZ7eB6gV4DrimWV4K/KCqVjbL05r2I4F3N6/nbcDWXccfDNzcLB8DfLNZ/gYvnnYzu6r2Av478MUkU5v27mk3a51uU1UXVtXMqpo52McUkiRJbTHRRuhXq6pbk2wD9OW6p7o2h87o9+kDHHoL8F7gHjohfg6dcD2ieedJDqczYj6rqp5OMp/OJwejYfW1VNWvkvw2yRHAAXRG6wdVVY8n+Tpw0jDrXdk1Gv488Gxznue75vEHOLmqru3uK8krgV9V1XNN07HAf03SV+cOSXarqnv71Xhfkt/S+RRgAZ2R+XOS7AdsWlWLh7pOSZKkDcFEG6FfLckewMbAsgE2Xwcc3fe0lyRbJXlFs+0m4MN0ptj8GJgNPFtVT4ywhC2Bx5pwvAedkXnojH4flmTXvr6b9ieB7hHzW1gzIn0cg39CcBHwNV44cj+UzwPvY82bvrXVO1zXAu9PsglAkt2TbEZnmsw1fW3A1KrasaqmVdU0Ojcuv+jm2OZnsyvwAEBVLQduoPOJhDfDSpKkCWOiBfopzU2lS4DLgfcMFHCr6mfAx4F5Se4E/p3OnHvoBOed6UyLWQX8ijXTRUbiGmBSkruAz9AJ8jTz208A/rWZ3nJ5s//VwFF9N8UCJwPvbep7F50572tzFTCV4U23oanjEeA7wMsGq3cELqJzw/HtSX4CXEDnzcKbWDNd59imz25zeWGgv6H5+d0AfLSqftu17TJgHwz0kiRpAskA9w1qA5NkJnBOVR3a61q69d0cW1Uze9H/zKQW9aLj8czfB5IkjUtJFq8tM03YOfQTRZKPAu9nGHPn17eqehboSZiXJEnaUDhCP4qaRzbu2q/5tP43gvZako/R+fKmbt+qqk/3op5ecYR+AP4+kCRpXBpshN5ArwnLQD8Afx9IkjQuDRboJ9pNsZIkSdIGxUAvSZIktZiBXpIkSWoxA70kSZLUYj62UhPXjBmwyNtiJUlSuzlCL0mSJLWYgV6SJElqMQO9JEmS1GIGekmSJKnFDPSSJElSixnoJUmSpBYz0EuSJEktZqCXJEmSWsxAL0mSJLWYgV6SJElqMQO9JEmS1GIGekmSJKnFDPSSJElSixnoJUmSpBYz0EuSJEktZqCXJEmSWsxAL0mSJLWYgV6SJElqMQO9JEmS1GIGekmSJKnFDPSSJElSixnoJUmSpBYz0EuSJEktZqCXJEmSWsxAL0mSJLWYgV6SJElqMQO9JEmS1GKTel2A1DOLF0PS6yrGl6peVyBJkkbIEXpJkiSpxQz0kiRJUosZ6CVJkqQWM9BLkiRJLWaglyRJklrMQC9JkiS1mIFekiRJajEDvSRJktRiBnpJkiSpxQz041iSVUmWJLkjye1JDhrj/uYnWdS1PjPJ/LHss+lnkyS3d62/PUkl2aOrbVqSZ7pej1uSvDrJpkmWJdmi3zmvSHLMWNcuSZLUawb68e2ZqppeVfsApwNnjcZJk2w8yObtkrx5NPoZgUOAH3atHwvc3Pzd7b6u1+MS4G+r6mngWuCovp2SbNmc8+oxrVqSJGkcMNC3xxbAY30rSU5NsjDJnUk+1dV+fJIFzUj2BX3hPcnyJP+Q5A5g1iD9fA74WP/GZoT8puaTgtWfFiQ5PMkPklyZ5P4kn0lyXFPD0iSvavbbNsncpuaFSQ7uOv2bgO81+02lE8b/AnjnMF+Py/rtexRwbRP2JUmSNmgG+vFtShPM7wYuAv4OIMmRwG7AAcB0YEaSw5LsCRwDHFxV04FVwHHNuTYDbquqfarq5kH6vBV4Lsnsfu0PAW+oqv2aPs7t2rYPcCKwJ/AuYPeqOqCp+eRmn38Ezqmq/YH/0WzrMxuY3yy/Dbimqn4OLEsyo2u/VzWvx33A/wN8vmm/FtgvydbN+jvphPwXSXJCkkVJFj08yIsgSZLUFpN6XYAG9UwTzEkyC7g0yV7Akc2fHzf7TaUT8PcGZgALkwBMoRPEoRPu5w6z3zOBjwOndbVtAnwxSd8bhd27ti2sqt80dd4HzGval9IJ6wCvB17T1AWwRTMavyXwaNdo+rF0wj/AN5r1xc36fV2vxzHAhcCbquq5JFcBRyeZC+xLJ+S/SFVd2BzHzKSG93JIkiSNXwb6lqiqW5NsA2wLBDirqi7o3ifJycAlVXX6AKdYUVWrhtnX9UnOBA7sav4Q8Fs6o/EbASu6tj3btfx81/rzrPk3thFwYFV1H9cXzK9tlrcCjgD+ezphe2Ogkpw6QJlXAV/uWr8M+ASd1+bKqlo5nGuVJElqO6fctETzxJeNgWV0AvCcZoSbJDsm2Q64js4o9XZN+1ZJXrGOXZ4JfKRrfUvgN1X1PJ1pNYPdWDuQeayZfkMz0g9d8+eBo4GvVtUrqmpaVe0M/BI4dIDzHQLc17U+n86nFCexluk2kiRJGyJH6Me3KUmWNMsB3tOMss9r5svf2kxhWQ4cX1U/S/LxZvtGwEo6AfeBkXZcVf+WpHua+XnA3CTvBq4BnhrhKT8A/HOSO+n8u7sxyUnAH1bV3c0+xwJ/3++4uV3tr2pejwDPAX/ZVe/zSb4N/BnwgxHWJkmS1FqpchqxeiPJIXTeiJzYi/5nJrVo6N0mFn8fSJI0LiVZXFUzB9rmCL16pnnazmBP3JEkSdIQDPQTUJLvALv2az6tqgZ8MowkSZLGLwP9BFRVRw29lyRJktrAp9xIkiRJLWaglyRJklrMQC9JkiS1mIFekiRJajEDvSRJktRiPuVGE9eMGbDIr5aSJEnt5gi9JEmS1GIGekmSJKnFDPSSJElSixnoJUmSpBYz0EuSJEktZqCXJEmSWsxAL0mSJLWYgV6SJElqMQO9JEmS1GIGekmSJKnFDPSSJElSixnoJUmSpBYz0EuSJEktZqCXJEmSWsxAL0mSJLWYgV6SJElqMQO9JEmS1GIGekmSJKnFDPSSJElSixnoJUmSpBYz0EuSJEktZqCXJEmSWsxAL0mSJLWYgV6SJElqMQO9JEmS1GIGekmSJKnFJvW6AKlnFi+GpNdVjC9Vva5AkiSNkCP0kiRJUosZ6CVJkqQWM9BLkiRJLWaglyRJklrMQC9JkiS1mIFekiRJajEDvSRJktRiBnpJkiSpxQz0GpeSvD1JJdmjWZ+W5JkkS5LckeSWJK9OsmmSZUm26Hf8FUmO6U31kiRJ64+BXuPVscDNzd997quq6VW1D3AJ8LdV9TRwLXBU305JtgQOAa5ej/VKkiT1hIFe406SqXQC+V8A71zLblsAjzXLl/Xb7yjg2ibsS5IkbdAm9boAaQBvA66pqp8302lmAMuAVyVZAmwObAq8ttn/WuCiJFtX1TI64f6LvShckiRpfXOEXuPRscA3muVvsGbaTd+Um1cBHwQuBKiq54CrgKOTbAPsSyfkv0iSE5IsSrLo4bG8AkmSpPUkVdXrGqTVkmwFPAg8DBSwcfP3HwFXV9VezX5TgGVVtWmzfgTwCeByYL+qOmGovmYmtWhMrqLF/H0gSdK4lGRxVc0caJsj9Bpvjga+WlWvqKppVbUz8Etg5377HQLc17U+H9gNOInOnHpJkqQJwTn0Gm+OBf6+X9tc4HTWzKEP8Bzwl307VNXzSb4N/Bnwg/VUqyRJUs855UYTllNuBuDvA0mSxiWn3EiSJEkbKAO9JEmS1GIGekmSJKnFDPSSJElSixnoJUmSpBYz0EuSJEktZqCXJEmSWsxAL0mSJLWYgV6SJElqsUm9LkDqmRkzYJHfFStJktrNEXpJkiSpxQz0kiRJUosZ6CVJkqQWM9BLkiRJLWaglyRJklrMQC9JkiS1mIFekiRJajEDvSRJktRiBnpJkiSpxQz0kiRJUosZ6CVJkqQWM9BLkiRJLWaglyRJklrMQC9JkiS1mIFekiRJajEDvSRJktRiBnpJkiSpxQz0kiRJUosZ6CVJkqQWM9BLkiRJLWaglyRJklrMQC9JkiS1mIFekiRJajEDvSRJktRiBnpJkiSpxQz0kiRJUotN6nUBUs8sXgxJr6vQaKnqdQWSJPWEI/SSJElSixnoJUmSpBYz0EuSJEktZqCXJEmSWsxAL0mSJLWYgV6SJElqMQO9JEmS1GIGekmSJKnFDPSSJElSixno1XNJPpbkp0nuTLIkyWuTfDDJpsM4dvn6qFGSJGm8mtTrAjSxJZkFvAXYr6qeTbIN8AfA5cDXgKd7WZ8kSdJ45wi9em174JGqehagqh4BjgZ2AG5IckOSOUm+0HdAkr9Kck7/EyU5NcnCZqT/U+vrAiRJknrJQK9emwfsnOTnSc5L8kdVdS7wa2B2Vc0Gvgn8SZJNmmPeC1zcfZIkRwK7AQcA04EZSQ5bb1chSZLUIwZ69VRVLQdmACcADwOXJ/mfA+xzPfCWJHsAm1TV0n6nOrL582PgdmAPOgH/BZKckGRRkkUPj/bFSJIk9YBz6NVzVbUKmA/MT7IUeM8Au10E/C1wN/DlAbYHOKuqLhiirwuBCwFmJvUSypYkSRoXHKFXTyV5dZLukfTpwAPAk8DmfY1VdRuwM/DnwGUDnOpaYE6Sqc15d0yy3ZgVLkmSNE44Qq9emwr8U5KXA78HfkFn+s2xwDVJft3Mo4fOXPrpVfVY/5NU1bwkewK3JgFYDhwPPLQerkGSJKlnUjWyWQdJ/guwc1XdOTYlSQNL8l3gnKq6bjTONzOpRaNxIo0PI/xdJklSmyRZXFUzB9o2rCk3SeYn2SLJVnRuOPxSks+PZpHS2iR5eZKfA8+MVpiXJEnaUAx3ys2WVfW7JH8JXFpVn0ziCL3Wi6p6HNi913VIkiSNR8O9KXZSku2BPwO+O4b1SJIkSRqB4Qb6/0XnKSL3VdXCJK8E7h27siRJkiQNx4hvipU2FN4Uu4Hxd5kkaQM2GjfF7p7kuiQ/adb3TvLx0SxSkiRJ0sgNd8rNl4DTgZUAzSMr3zlWRUmSJEkanuEG+k2rakG/tt+PdjGSJEmSRma4j618JMmrgAJIcjTwmzGrSlofZsyARc6ilyRJ7TbcQH8ScCGwR5L/AH4JHDdmVUmSJEkaluEG+qqq1yfZDNioqp5MsutYFiZJkiRpaMOdQz8XoKqeqqonm7Zvj01JkiRJkoZr0BH6JHsA/w3YMsmfdm3aApg8loVJkiRJGtpQU25eDbwFeDnwJ13tTwJ/NVZFSZIkSRqeQQN9VV0JXJlkVlXdup5qkiRJkjRMw70p9oQkLxqRr6o5o1yPJEmSpBEYbqD/btfyZOAo4NejX44kSZKkkRhWoK+qud3rSS4Dbh6TiiRJkiQN23AfW9nfbsB2o1mIJEmSpJEb1gh9kieBAtL8/Z/AaWNYlyRJkqRhGO6Um83HuhBJkiRJIzfcm2JpvljqEDoj9DdV1RVjVpUkSZKkYRnWHPok5wEnAkuBnwAnJvnnsSxMkiRJ0tCGO0J/BLBnVRVAkkuAn45ZVZIkSZKGZbhPufkFsEvX+s5NmyRJkqQeGnSEPsnVdObMbw7clWRBs/5aYMHYlydJkiRpMENNuTl7vVQhSZIkaZ0MGuir6gfrqxBJkiRJIzfUlJubq+qQri+WWr0JqKraYkyrkyRJkjSooUboD2n+9oulJEmSpHFoyKfcJNk4yd3roxhJkiRJIzNkoK+qVcA9SXYZal9JkiRJ69dwv1jqvwA/bR5b+VRfY1W9dUyqkiRJkjQsww30nxjTKiRJkiStk+EG+v+rqk7rbkjy94CPtZQkSZJ6aMg59I03DND25tEsRJIkSdLIDfUc+vcD/zfwqiR3dm3aHLhlLAuTxtzixZD0ugpJ61PV0PtIUssMNeXm68D3gLOAj3a1P1lVj45ZVZIkSZKGZagvlnoCeCLJ76vqge5tSb5aVe8a0/6I9vMAAB2wSURBVOokSZIkDWq4c+j/W/dKkknAjNEvR5IkSdJIDBrok5ye5Elg7yS/6/sD/Ba4cr1UKEmSJGmthppycxZwVpKzgM8CuwOT+zaPcW2SJEmShjDc59DfD9wI7AQsAQ4EbgWOGKO6JEmSJA3DcOfQfwDYH3igqmYD+wKPj1lVkiRJkoZluIF+RVWtAEjysqq6G3j12JUlSZIkaTiGO+XmwSQvB64A/j3JY8ADQxwjSZIkaYwNa4S+qo6qqser6gzgE8D/Bt4+loW1UZJVSZYkuSPJ7UkOWk/9zk9yT9P3kiRHr8M5Dh/repOckeTpJNt1tS0fyz4lSZI2dMMdoV+tqn4wFoVsIJ6pqukASd5I5xt2/6h7hySTqur3Y9D3cVW16CUcfziwHLhluAes47U8AvwNcNoIj5MkSdIAhjuHXiO3BfAYrB79vinJVcDPmrbjkyxoRtQvSLJxknck+Xyz/ZQk9zfLr0zyw5EWkOSKJIuT/DTJCV3tb2o+QbgjyXVJpgEnAh9q6jk0ybQk1ye5s9lnl+bYryQ5P8ltwGeT3Jtk22bbRkl+0be+FhcDxyTZagT1Lk/yuab9+0kOaD6VuD/JW5t9Nm72WdjU/L6Rvl6SJEltNOIReg1qSpIldJ7Vvz0vfKznfsBeVfXLJHsCxwAHV9XKJOcBxwHzgI80+x8KLEuyY7N84xB9/0uSZ5rl11XVMmBOVT2aZAqwMMlcOm/ivgQc1tSyVbPP+cDyqjobIMnVwCVVdUmSOcC5rJlmtRNwUFWtSvJEU/sXgNcDd1TVw4PUuZxOqD8F+GS/bS+qt7mOzYDrq+rUJN8BzgTeALwGuAS4CvgL4Imq2j/Jy4AfJplXVb/s7qB5o3ACwC5DvKCSJEltYKAfXd1TbmYBlybZq9m2oCtcvg6YQSe0AkwBHqqq/0wyNcnmwM7A14HD6AT6fx2i74Gm3HwgyVHN8s7AbsC2wI19tVTVo2s53yzgT5vlr9L5YrE+36qqVc3yxXS+NfgLwBzgy0PUCZ03B0uSnD2MepcBzwHXNO1LgWebN0JLgWlN+5F0vtG47/6BLZvjXxDoq+pC4EKAmYlfjiZJklrPQD9GqurWJNvQCdAAT3VtDp3R79MHOPQW4L3APcBNdELyLDrzzoctyeF0RsxnVdXTSeaz5lt+X6rV11JVv0ry2yRHAAfQGa0fVFU9nuTrwEnDrHdlVfWF7+eBZ5vzPJ+k799wgJOr6tqXdGWSJEkt4xz6MZJkD2BjOiPM/V0HHN33tJckWyV5RbPtJuDDdKbY/BiYTWdE+okRlrAl8FgTjveg8+2+AD8CDkuya1/fTfuTwOZdx98CvLNZPq6pa20uAr7GC0fuh/J54H2seVO5tnqH61rg/Uk2AUiye5LNRngOSZKk1jHQj64pfY+OBC4H3jNQwK2qnwEfB+YluRP4dzpz7qETnHemMy1mFfAr4OZ1qOUaYFKSu4DP0AnyNPPbTwD+NckdTZ0AVwNH9d0UC5wMvLep71105ryvzVXAVIY33YamjkeA7wAvG6zeEbiIzg3Htyf5CXABfgIlSZImgKyZySCtmyQzgXOq6tBe1zISM5OX9JxPSS3k/3mSWirJ4qqaOdA2RzD1kiT5KPB+hjF3XpIkSaPPEfoWaR7ZuGu/5tPG242gST4GvKNf87eq6tO9qGdtHKGXJiD/z5PUUoON0BvoNWEZ6KUJyP/zJLXUYIHem2IlSZKkFjPQS5IkSS1moJckSZJazEAvSZIktZiBXpIkSWoxn0OviWvGDFjkc24kSVK7OUIvSZIktZiBXpIkSWoxA70kSZLUYgZ6SZIkqcUM9JIkSVKLGeglSZKkFjPQS5IkSS1moJckSZJazEAvSZIktZiBXpIkSWoxA70kSZLUYgZ6SZIkqcUM9JIkSVKLGeglSZKkFjPQS5IkSS1moJckSZJazEAvSZIktZiBXpIkSWoxA70kSZLUYgZ6SZIkqcUM9JIkSVKLGeglSZKkFjPQS5IkSS1moJckSZJazEAvSZIktZiBXpIkSWqxSb0uQOqZxYsh6XUVE0dVryuQJGmD5Ai9JEmS1GIGekmSJKnFDPSSJElSixnoJUmSpBYz0EuSJEktZqCXJEmSWsxAL0mSJLWYgV6SJElqMQO9JEmS1GIG+gksyaokS5LckeT2JAeNcX/zkyzqWp+ZZP5Y9ilJkrShM9BPbM9U1fSq2gc4HThrNE6aZONBNm+X5M2j0Y8kSZIM9FpjC+CxvpUkpyZZmOTOJJ/qaj8+yYJmZP+CvvCeZHmSf0hyBzBrkH4+B3ysf2OSaUluaj4pWP1pQZLDk/wgyZVJ7k/ymSTHNTUsTfKqZr9tk8xtal6Y5ODReVkkSZLGt0m9LkA9NSXJEmAysD1wBECSI4HdgAOAAFclOQx4GDgGOLiqViY5DzgOuBTYDLitqv5miD5vBY5KMht4sqv9IeANVbUiyW7AZcDMZts+wJ7Ao8D9wEVVdUCSU4CTgQ8C/wicU1U3J9kFuLY5RpIkaYNmoJ/Ynqmq6QBJZgGXJtkLOLL58+Nmv6l0Av7ewAxgYRKAKXSCOMAqYO4w+z0T+DhwWlfbJsAXk0xvzrV717aFVfWbps77gHlN+1JgdrP8euA1TV0AWySZWlXLuztOcgJwAsAuwyxWkiRpPDPQC4CqujXJNsC2dEblz6qqC7r3SXIycElVnT7AKVZU1aph9nV9kjOBA7uaPwT8ls5o/EbAiq5tz3YtP9+1/jxr/g1vBBxYVd3HDdT3hcCFADOTGk69kiRJ45lz6AVAkj2AjYFldKarzEkytdm2Y5LtgOuAo5tlkmyV5BXr2OWZwEe61rcEflNVzwPvamoZiXl0pt/0Xc/0daxLkiSpVRyhn9j65tBDZ1T+Pc0o+7wkewK3NlNYlgPHV9XPkny82b4RsBI4CXhgpB1X1b8lebir6TxgbpJ3A9cAT43wlB8A/jnJnXT+Xd8InDjSuiRJktomVc460MQ0M6lFQ++m0eLvGkmS1lmSxVU1c6BtTrmRJEmSWswpNxp1Sb4D7Nqv+bSqurYX9UiSJG3IDPQadVV1VK9rkCRJmiicciNJkiS1mIFekiRJajEDvSRJktRiBnpJkiSpxQz0kiRJUov5lBtNXDNmwCK/WkqSJLWbI/SSJElSixnoJUmSpBYz0EuSJEktZqCXJEmSWsxAL0mSJLWYgV6SJElqMQO9JEmS1GIGekmSJKnFDPSSJElSixnoJUmSpBYz0EuSJEktZqCXJEmSWsxAL0mSJLWYgV6SJElqMQO9JEmS1GIGekmSJKnFDPSSJElSixnoJUmSpBYz0EuSJEktZqCXJEmSWsxAL0mSJLWYgV6SJElqMQO9JEmS1GIGekmSJKnFDPSSJElSixnoJUmSpBab1OsCpJ5ZvBiSXlchSUOr6nUFksYxR+glSZKkFjPQS5IkSS1moJckSZJazEAvSZIktZiBXpIkSWoxA70kSZLUYgZ6SZIkqcUM9JIkSVKLGeglSZKkFjPQa0hJPpbkp0nuTLIkyWuTfDDJpsM4dvkw+zg/ycHN8qQkDyf5TL995ie5p6nhriQnNO1fTvK+fvu+Pcn3hn+VkiRJ7WSg16CSzALeAuxXVXsDrwd+BXwQGDLQj8CBwI+a5TcAPwfekST99juuqqYDBwN/n+QPgMuAd/bb751NuyRJ0gbNQK+hbA88UlXPAlTVI8DRwA7ADUluSDInyRf6DkjyV0nO6X+iJKcmWdiM9H+qq31P4OdVtappOhb4R+D/A2atpa6pwFPAKuA6YI8k2zfn24zOG48rXsqFS5IktYGBXkOZB+yc5OdJzkvyR1V1LvBrYHZVzQa+CfxJkk2aY94LXNx9kiRHArsBBwDTgRlJDms2vxm4ptlvMp0wfjWdEfZj+9XzL0nuBO4B/q6qVjVvBOYCf9bs8yfA/Kr63ei8BJIkSeOXgV6DqqrlwAzgBOBh4PIk/3OAfa4H3pJkD2CTqlra71RHNn9+DNwO7EEn4AO8kSbQ05nec0NVPUMnpL89ycZd5zmumfqzC/DhJK9o2run3ax1uk2SE5IsSrLo4WG+BpIkSePZpF4XoPGvGQGfD8xPshR4zwC7XQT8LXA38OUBtgc4q6oueEFj58bal1fVr5umY4FDkvyfZn1r4Ajg3/vV9HCS24HXAg8AtwDbJ9kHOIgXz6nvO+5C4EKAmUmt/aolSZLawRF6DSrJq5Ps1tU0nU6AfhLYvK+xqm4Ddgb+nIFHx68F5iSZ2px3xyTbAbOBG5q2LYBDgV2qalpVTQNO4sXTbvreCOwL3Nf0X8DlwCXA96pqxUu4bEmSpNZwhF5DmQr8U5KXA78HfkFn+s2xwDVJft3Mo4fOXPrpVfVY/5NU1bzm5tdbmwfXLAeOpzN//tvNbkcB1/fdgNu4Evhskpc16/+S5BngZcBXqmpx176XAR8BPvpSL1qSJKkt0hnYlF66JN8Fzqmq60ZwzO3Aa6tq5dhVNrCZSS1a351K0rrw/2ppwkuyuKpmDrTNKTd6yZK8PMnPgWdGEuYBqmq/XoR5SZKkDYVTbvSSVdXjwO69rkOSJGkicoRekiRJajEDvSRJktRiBnpJkiSpxQz0kiRJUosZ6CVJkqQWM9BLkiRJLeZjKzVxzZgBi/xqKUmS1G6O0EuSJEktZqCXJEmSWsxAL0mSJLWYgV6SJElqMQO9JEmS1GIGekmSJKnFDPSSJElSixnoJUmSpBYz0EuSJEktZqCXJEmSWsxAL0mSJLWYgV6SJElqMQO9JEmS1GIGekmSJKnFDPSSJElSixnoJUmSpBYz0EuSJEktZqCXJEmSWsxAL0mSJLWYgV6SJElqMQO9JEmS1GIGekmSJKnFDPSSJElSixnoJUmSpBYz0EuSJEktZqCXJEmSWsxAL0mSJLXYpF4XIPXM4sWQ9LoKSf1V9boCSWoVR+glSZKkFjPQS5IkSS1moJckSZJazEAvSZIktZiBXpIkSWoxA70kSZLUYgZ6SZIkqcUM9JIkSVKLGeglSZKkFjPQjyNJViVZkuSOJLcnOWiM+5ufZFHX+swk88eyz6afTZLc3ix/LMlPk9zZXPtrm/YPJtl0GOdaPtb1SpIkjWcG+vHlmaqaXlX7AKcDZ43GSZNsPMjm7ZK8eTT6GYFDgB8mmQW8BdivqvYGXg/8qtnng8CQgV6SJGmiM9CPX1sAj/WtJDk1ycJmJPtTXe3HJ1nQjG5f0BfekyxP8g9J7gBmDdLP54CP9W9MMi3JTc0nBas/LUhyeJIfJLkyyf1JPpPkuKaGpUle1ey3bZK5Tc0Lkxzcdfo3Ad8DtgceqapnAarqkar6dZIPADsANyS5IcmcJF/oqu2vkpwzQM0DvkaSJEkbMgP9+DKlCeZ3AxcBfweQ5EhgN+AAYDowI8lhSfYEjgEOrqrpwCrguOZcmwG3VdU+VXXzIH3eCjyXZHa/9oeAN1TVfk0f53Zt2wc4EdgTeBewe1Ud0NR8crPPPwLnVNX+wP9otvWZDcwH5gE7J/l5kvOS/BFAVZ0L/BqYXVWzgW8Cf5Jkk+b49wIXdxe7tteo/8UmOSHJoiSLHh7kRZEkSWqLSb0uQC/wTBPMaaajXJpkL+DI5s+Pm/2m0gmvewMzgIVJAKbQCeLQCfdzh9nvmcDHgdO62jYBvpik743C7l3bFlbVb5o676MTzAGW0gnr0Jk+85qmLoAtkkwFtgQeraqnm+NnAIc2x12e5KNV9ZXu4qpqeZLrgbckuQvYpKqW9ruGtb1GN/Y714XAhQAzkxridZEkSRr3DPTjVFXdmmQbYFsgwFlVdUH3PklOBi6pqtMHOMWKqlo1zL6uT3ImcGBX84eA39IZjd8IWNG17dmu5ee71p9nzb+pjYADq6r7OJIcA1zb1fcqOqP185MsBd4DfGWAMi8C/ha4G/jyANsHfI0kSZI2dE65GaeS7AFsDCyjE4DnNCPcJNkxyXbAdcDRzTJJtkryinXs8kzgI13rWwK/qarn6UyrGezG2oHMY830G5qRflgzf54kr06yW9cx04EHmuUngc37NlTVbcDOwJ8Dlw3Q39peI0mSpA2aI/Tjy5QkS5rlAO9pRrDnNfPlb22msCwHjq+qnyX5eLN9I2AlcBJrQvGwVdW/JemeVn4eMDfJu4FrgKdGeMoPAP+c5E46/85uTHIS8IdVdXezz1Tgn5K8HPg98AvghGbbhcA1SX7dzKOHzlz66VW1+mbhrvoHfI1YMwVJkiRpg5QqpxFr/UhyCJ03Iieu4/HfpXOj7XWjUc/MpBYNvZuk9c3/lyTpRZIsrqqZA21zyo3Wm6q6eV3CfJKXJ/k5nZuGRyXMS5IkbSiccjMBJPkOsGu/5tOq6tqB9h9vqupxXviUHUmSJDUM9BNAVR3V6xokSZI0NpxyI0mSJLWYgV6SJElqMQO9JEmS1GIGekmSJKnFDPSSJElSi/mUG01cM2bAIr9aSpIktZsj9JIkSVKLGeglSZL0/7d398Ga1nUdx98fd9UYlhh5yHEA5UEY2so2WBkRodWMsGkUSlKHGpQKckgpwKSn0R6cGk1wbBoViSADLEbQrSmEIGY3QGUXFxahHQlhkMFF0AiyQOHbH9fv7N4dzz6csw/3+d3n/frnXPfvuq/7+t7nxzX7uX/ne1+oYwZ6SZIkqWMGekmSJKljBnpJkiSpYwZ6SZIkqWMGekmSJKljBnpJkiSpYwZ6SZIkqWMGekmSJKljBnpJkiSpYwZ6SZIkqWMGekmSJKljBnpJkiSpYwZ6SZIkqWMGekmSJKljBnpJkiSpYwZ6SZIkqWMGekmSJKljBnpJkiSpYwZ6SZIkqWMGekmSJKljBnpJkiSpYwZ6SZIkqWMGekmSJKljBnpJkiSpY4vHXYA0NmvXQjLuKiRJUs+qxl2BK/SSJElSzwz0kiRJUscM9JIkSVLHDPSSJElSxwz0kiRJUscM9JIkSVLHDPSSJElSxwz0kiRJUscM9JIkSVLHDPSSJElSxyY60Cd5Nsm6JHcmuSPJq3fTeW9OsqGde12SN8/hNVbs6nqTvD/Jd5L80MjYU7vynCPn+eckByb5uSRfbnN0T5Kz2v6Tkyzdjte5OcnyXV+xJEnS/LR43AXsYv9TVcsAkvwM8KfAT44+IcniqvreLjj3aVW1ZgeOXwE8Bdy6vQfM8b08BpwHvHeWx81Zkj2AfYGNwMXAMVX19SQvBA5uTzsZ+Efgnt1VlyRJUo8meoV+mh8Evg2bVr9XJ1lJC4xJfinJl9qK+ieSLEpyapIL2/5zktzftg9NcstsC0jy2SRrk3wlyZkj4ye1vyDcmeTGJAcDvw78Vqvn+CQHJ7kpyV3tOS9tx16W5ONJvgh8MMlXk+zf9j0vyX1Tj7fgUuAtSfaZRb1PJflQG/+XJMe0lfL7k7yxPWdRe87treazRl56BXAzsBfDh8rHAarq6ara0P4y8UbgQ+39H5bkjpHzHz76eGT8xCS3td/l1UmWbG0+JEmSJsGkB/o9WiD8d+AS4I9H9h0FnFNVRyT5YeAtwHFtRf9Z4DRgNXB8e/7xwONJDmjbq7Zx7itGWm72bWNnVNXRwHLg3Un2bWH7k8AvVNWPA6dW1QPAx4GLqmpZVa0G/gK4vKpeAVwBfHTkXAcCr66qc4G/bbUDvB64s6q+uZU6n2II9efMsO/76m3jewI3VdWPAE8CfwL8NHAK8EftOb8CPFFVrwReCfxakkPavjcA11XVt4CVwINJrkpyWpLnVdWtbfw97f3/B/BEkmXt+HcAfz1aaJL9gN8HXl9VRwFrgHOnv6EkZyZZk2TN1n4pkiRJvVhILTfHAn+T5Efbvi9V1dfa9k8BRwO3JwHYA3i0qr6RZEmSvYCDgCuBExgC/TXbOPdMLTfvTnJK2z4IOBzYH1g1VUsLuTM5Fvj5tv0p4IMj+66uqmfb9qXA54CPAGcwLfhuwUeBdUn+fDvqfRx4Briuja8Hnq6q7yZZz+aWmROBV4x8f2DvdvzXgOOA89v7/dUkP8bw4eN8hg8Gb5+hxkuAdyQ5l+HD1zHT9r8KWArc0ubwBcBt01+kqi5maPNheVIz/zokSZL6MemBfpOquq2t4k61n/z3yO4wrH7/zgyH3sqwIryBYcX+DIZwfd5szp9kBUNoPbaqvpPkZuAHZvMaW7HpvVTVQ0k2JnkdQ+g9bcuHbTrmP5NcCZy9nfV+t6qmwvBzwNPtdZ5LMvXfVIB3VdXnR8+V5FDgoap6ZuT864H1ST7FEPjfPkOZnwHeB9wErK2qx6ftD3BDVb1tW+9XkiRpkkx6y80mSY4EFtH6tae5EXhz2t1ekuyT5GVt32qGleNVwJeB1zKsSD8xyxL2Br7dwvGRDCvKAF8ATphqRxnpZX+Socd8yq3AW9v2VDvQllzC0HozunK/LRcCZ7H5Q96W6t1enwfemeT5AEmOSLInrd2mjS1pHxymLAMebNv/7/1X1f+21/wYM//V4QvAcUle3l57zyRHzLJmSZKk7kx6oJ/qoV8H/B1w+kwBt6ruYei/vj7JXcANwEva7tUM7Sar2rEPAf82h1quAxYnuRf4M4YASutvPxO4JsmdrU6AfwBOmfpSLPAuhpaTu4BfZuae9ykrgSVsX7sNrY7HgGuBF26t3lm4hOELx3ckuRv4BMOHhZPY3K4T4LfTbvEJ/CGbV+c/Dbwnwy0tD2tjVzD8ReD6Ger/Zjv2qvY7ug04cpY1S5IkdSebOyc0KTLcl/2iqjp+m0/ejTLclvKWqprTfeOTnA/sXVV/sDPqWZ7s0H1FJUmS2E1ZOsnaLWWoBdNDv1AkuQB4J9vRO7+7VdXTDHfMmbUk1wKHAa/bqUVJkiR1zhX6HdBC5iHTht87/Yug45bk94BTpw1fXVUfGEc984Ur9JIkaYfNgxV6A70WLAO9JEnaYfMg0E/6l2IlSZKkiWaglyRJkjpmoJckSZI6ZqCXJEmSOuZtK7VwHX00rPFrsZIkqW+u0EuSJEkdM9BLkiRJHTPQS5IkSR0z0EuSJEkdM9BLkiRJHTPQS5IkSR0z0EuSJEkdM9BLkiRJHTPQS5IkSR0z0EuSJEkdM9BLkiRJHTPQS5IkSR0z0EuSJEkdS1WNuwZpLJI8CWwYdx3aafYDHht3EdopnMvJ4nxODudyvF5WVfvPtGPx7q5Emkc2VNXycRehnSPJGudzMjiXk8X5nBzO5fxly40kSZLUMQO9JEmS1DEDvRayi8ddgHYq53NyOJeTxfmcHM7lPOWXYiVJkqSOuUIvSZIkdcxArwUpyUlJNiS5L8kF465Hc5fkgSTrk6xLsmbc9Wh2klya5NEkd4+M7ZPkhiRfbT9fNM4atX22MJfvT/Jwuz7XJfnZcdao7ZfkoCT/muSeJF9Jck4b9/qchwz0WnCSLAL+EngDsBR4W5Kl461KO+i1VbXM26l16TLgpGljFwA3VtXhwI3tsea/y/j+uQS4qF2fy6rqn3ZzTZq77wHnVdVS4FXA2e3fSq/PechAr4XoGOC+qrq/qp4BPg28acw1SQtSVa0CvjVt+E3A5W37cuDk3VqU5mQLc6lOVdUjVXVH234SuBc4AK/PeclAr4XoAOChkcdfb2PqUwHXJ1mb5MxxF6Od4sVV9Ujb/gbw4nEWox32G0nuai05tmd0KMnBwE8AX8Trc14y0Evq3Wuq6iiGFqqzk5ww7oK089RwKzZvx9avjwGHAcuAR4APj7cczVaSJcBngN+sqv8a3ef1OX8Y6LUQPQwcNPL4wDamDlXVw+3no8C1DC1V6tvGJC8BaD8fHXM9mqOq2lhVz1bVc8An8frsSpLnM4T5K6rqmjbs9TkPGei1EN0OHJ7kkCQvAN4KrBxzTZqDJHsm2WtqGzgRuHvrR6kDK4HT2/bpwOfGWIt2wFTwa07B67MbSQL8FXBvVV04ssvrcx7yfyylBandOu0jwCLg0qr6wJhL0hwkOZRhVR5gMXClc9mXJFcBK4D9gI3A+4DPAn8PvBR4EPjFqvLLlvPcFuZyBUO7TQEPAGeN9F9rHkvyGmA1sB54rg3/LkMfvdfnPGOglyRJkjpmy40kSZLUMQO9JEmS1DEDvSRJktQxA70kSZLUMQO9JEmS1DEDvSRJktQxA70kSZLUMQO9JEmS1LH/AwqqlUXMWOtwAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 720x720 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt,negRanking0_rnn =aggregateRankings(neg_rankings_rnn0,positive=False,maxLenAttributes=2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-02-18T11:03:05.188150Z",
     "start_time": "2020-02-18T11:03:04.743326Z"
    }
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAvQAAAI/CAYAAAAY801hAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8li6FKAAAgAElEQVR4nOzde7RdZX3/+/cHSE0gEH/cxiGABikIlkIgAYlcSqhSObUoLRb5BWtNK8FjEW1FpOoI/ooDKlQstRxABha8ICqVi6NCKhC5ai4YiIqIYDlSLZeASoAAhu/5Y82dLDY7+xKyszKz368xMpjrmc+cz3euHeNnPeuZc6eqkCRJktROm/S6AEmSJElrz0AvSZIktZiBXpIkSWoxA70kSZLUYgZ6SZIkqcUM9JIkSVKLbdbrAqRe2XbbbWvKlCm9LkOSJGlIixcvfqyqthton4FeY9aUKVNYtGhRr8uQJEkaUpIH17TPJTeSJElSixnoJUmSpBYz0EuSJEkt5hp6SZIk9cTzzz/PQw89xIoVK3pdygZj/Pjx7LTTTowbN27YxxjoJUmS1BMPPfQQW265JVOmTCFJr8vpuapi2bJlPPTQQ+yyyy7DPs4lN5IkSeqJFStWsM022xjmG0nYZpttRvyNhYFekiRJPWOYf7G1eT8M9JIkSRqzzjvvPPbcc0923HFH/uZv/gaACy64gMsuu2xUxz399NM555xz1sm5XEMvSZKkDUI+sW5n62tuDdnn/PPP59vf/jbf/va3V/3CyRNPPHGd1jHanKGXJEnSmHTiiSfywAMPcOSRR/LEE0+sau+ePT/ssMM4+eSTmTp1KnvttRcLFixY1eed73wnM2bMYLfdduNzn/vcquPPPvts9t9/f/bee2/mzp27qv2Tn/wku+++OwcffDD33nvvOrsOZ+glSZI0Jl1wwQVcd9113HTTTXzzm99cY7+nn36aJUuWcPPNNzN79mx+8IMfAHD33Xfz3e9+l6eeeop9992XP/7jP+YHP/gB9913HwsWLKCqOOqoo7j55pvZYost+MpXvsKSJUv47W9/y3777ce0adPWyXUY6CVJkqRBHHfccQAceuih/OY3v+FXv/oVAG9961uZMGECEyZMYObMmSxYsIBbb72VefPmse+++wKwfPly7rvvPp588kmOPvpoNt98cwCOOuqodVafgV5j1uJfLF7na/Ukqe2Gs+ZYGmv6P3mm7/VA7VXFaaedxpw5c1607zOf+cyo1ecaekmSJGkQV1xxBQC33norkyZNYtKkSQBcffXVrFixgmXLljF//nz2339//uiP/ohLLrmE5cuXA/Df//3fPPLIIxx66KFcddVVPPPMMzz55JNce+2166w+Z+glSZKkQYwfP559992X559/nksuuWRV+957783MmTN57LHH+PjHP87kyZOZPHky99xzDzNmzABg4sSJfPGLX2S//fbj2GOPZZ999mH77bdn//33X2f1pcqv1jQ2ZXKKOUP3k6SxxCU3Wp/uuece9txzz16XMajDDjuMc845h+nTp7+o/fTTT2fixIl86EMfWudjDvS+JFlcVdMH6u+SG0mSJKnFXHIjSZIkrcH8+fMHbD/99NPXax2DcYZekiRJajEDvSRJknrG+zlfbG3eDwO9JEmSemL8+PEsW7bMUN+oKpYtW8b48eNHdJxr6CVJktQTO+20Ew899BCPPvpor0vZYIwfP56ddtppRMeMqUCfZCWwFAiwEvibqrp9PYw7H9gBeKZpOqOqvj7CcxwGPDea9SY5HfgwMKWqHmnallfVxNEas2vsbwHvqaqHkmwL/BI4qaou6OrzX8CTdH52mwIfq6qrk9wEnFVV13f1/QDw2qp672jXLkmS1s64cePYZZddel1G6421JTfPVNXUqtoHOA04s3+HJKP1IWdWM/bUkYb5xmHAG0ZywFpey2PA363FcWstyQRgm6p6qGl6O/Bd4LgBus+sqqnAMcB5TdvlwDv69XtH0y5JkrRRG2uBvttWwBPQmf1OckuSa4AfNW3HJ1mQZEmSC5NsmuTtST7d7D85yQPN9muS3DbSApJclWRxkh8mOaGr/c1J7kxyV5IbkkwBTgQ+2NRzSJIpSW5McnfT51XNsf+W5IIk3wM+leS+JNs1+zZJ8tO+12twCXBskq1HUO/yJGc37d9OckCS+UkeSHJU02fTps/CpubuX+l0GDC/6/VxdD5U7JhkTd85rfr5AV8H/jjJ7zRjTQEmA7cMcp2SJEkbhTG15AaYkGQJMJ7OEpjDu/btB+xVVT9LsidwLHBQVT2f5HxgFjCPzpIUgEOAZUl2bLZvHmLsLyXpW3Lzh1W1DJhdVY83M9QLk1xJ50PW54BDm1q2bvpcACyvqnMAklwLXFpVlyaZTWe2+m3N+XcC3lBVK5P8uqn9M8AbgbuqarCFasvphPqTgbn99r2k3uY6tgBurKpTknwDOAN4E/A64FLgGuCvgF9X1f5JXgHclmReVf0MOBK4qrmunYEdqmpBkq82P4d/6qrhpiQBXgP8OUBT04LmPFfTmZ3/ag1wh03zQaTzYWTSIO+CJElSS4y1Gfq+JTd7AG8GLmvCIcCCJlwC/CEwjU5oXdK8fk1V/Q8wMcmWwM7Al4FD6QT6oWaDu5fcLGva3p/kLjrLS3YGdgMOBG7uq6WqHl/D+WY04wN8ATi4a9/Xqmpls30J8BfN9mzg80PUCZ0PB+9qrrPbQPUCPAdc12wvBb5TVc8321Oa9iOAv2jez+8B23QdfxBwa7N9LPDVZvsrvHTZzcyq2gv4feCzSfrW93cvu1njcpuquqiqplfVdDZf8xsgSZLUFmNthn6Vqrqjufmyb/nJU127Q2f2+7QBDr0deDdwL50QP5tOuB7RuvPmJtc3AjOq6unmxtmRPaNozVZdS1X9PMnDSQ4HDqAzWz+oqvpVki8D7xtmvc93zYa/ADzbnOeFrnX8oXOT66obV5vzvgb4eVU91zQdB/xfSfrqnJxkt6q6r1+N9yd5mM63AAvozMyfm2Q/YPOqWjzUdUqSJG0MxtoM/SpJ9qDzpJRlA+y+ATgmyfZN362TvLrZdwvwITpLbL4PzASerapfj7CEScATTTjeg87MPHRmvw9Nskvf2E37k0D3jPntrJ6RnsXg3xBcDHyRF8/cD+XTwBxWf+hbU73DdT3w3iTjAJLsnmQLOstkrutrAyZW1Y5VNaWqptC5cfklN8c2P5tdgAcBqmo5cBOdbyS8GVaSJI0ZYy3QT2huKl0CXAG8a6CAW1U/Aj4GzEtyN/CfdNbcQyc470xnWcxK4OesXi4yEtcBmyW5BziLTpCnWd9+AvDvzfKWK5r+1wJH990UC5wEvLup75101ryvyTXARIa33IamjseAbwCvGKzeEbiYzg3Hdyb5AXAhnQ8Lb2b1cp3jmjG7XcmLA/1Nzc/vJuAjVfVw177LgX0w0EuSpDEk/maujV+S6cC5VXVIr2vp1ndzbFVN78n4k1PMGbqfJI0lNddcIG2IkixeU2Yas2vox4okHwHeyzDWzq9vVfUs0JMwL0mStLEw0K9DzSMb+/+6s1P73wi6PlXVWXSWyKyS5KN0fnlTt69V1SfXW2GSJElaJwz061BVHd3rGoajCe6Gd0mSpI3AWLspVpIkSdqoGOglSZKkFjPQS5IkSS1moJckSZJazEAvSZIktZhPudGYNW3yNBbNXdTrMiRJkl4WZ+glSZKkFjPQS5IkSS1moJckSZJazEAvSZIktZiBXpIkSWoxA70kSZLUYgZ6SZIkqcUM9JIkSVKLGeglSZKkFjPQS5IkSS1moJckSZJazEAvSZIktZiBXpIkSWoxA70kSZLUYgZ6SZIkqcUM9JIkSVKLGeglSZKkFjPQS5IkSS1moJckSZJazEAvSZIktZiBXpIkSWoxA70kSZLUYgZ6SZIkqcUM9JIkSVKLGeglSZKkFjPQS5IkSS2Wqup1DVJPZHKKOb2uQpI2fjXXrCG9XEkWV9X0gfY5Qy9JkiS1mIFekiRJajEDvSRJktRiBnpJkiSpxQz0kiRJUosZ6CVJkqQWM9BLkiRJLWaglyRJklrMQC9JkiS1mIFekiRJarFRC/RJViZZkuSuJHcmecNojdWMNz/Joq7X05PMH80xm3HGJbmz2V6v19xVw/wk9zZjL0lyzFqc47D18DM6PcnTSbbvals+mmNKkiRt7EZzhv6ZqppaVfsApwFnrouTJtl0kN3bJzlyXYwzAgcDtzXbQ15zks1GqY5ZzdhTq+rra3H8YcCIAv1aXstjwN+txXGSJEkawPpacrMV8ETfiySnJFmY5O4kn+hqPz7JgmaW+cK+8J5keZJ/SnIXMGOQcc4GPtq/McmUJLc0s+arZs6bWenvJLk6yQNJzkoyq6lhaZJdm37bJbmyqXlhkoO6Tv9m4FuDXXMzzi1JrgF+tKZrTfL2JJ9u9p+c5IFm+zVJbhtgjEEluSrJ4iQ/THJCV/ubm/fhriQ3JJkCnAh8sKnnkOY9u7H5Gd2Q5FXNsf+W5IIk3wM+leS+JNs1+zZJ8tO+12twCXBskq1HUO/yJGc37d9OckDzrcQDSY5q+mza9On7ezVnpO+XJElSG43WbDHAhCRLgPHADsDhAEmOAHYDDgACXJPkUOBR4FjgoKp6Psn5wCzgMmAL4HtVNdTM7h3A0UlmAk92tT8CvKmqViTZDbgcmN7s2wfYE3gceAC4uKoOSHIycBLwAeCfgXOr6tYm2F7fHAMwE+j7UDLgNTf2A/aqqp8l2XMN1zoP+HDT/xBgWZIdm+2bh7j2LyV5ptn+w6paBsyuqseTTAAWJrmSzoe4zwGHNrVs3fS5AFheVecAJLkWuLSqLk0yGzgPeFtz/p2AN1TVyiS/bmr/DPBG4K6qenSQOpfTCfUnA3P77XtJvc11bAHcWFWnJPkGcAbwJuB1wKXANcBfAb+uqv2TvAK4Lcm8qvpZ9wDNB4XOh4VJQ7yjkiRJLTCagf6ZqpoKkGQGcFmSvYAjmj/fb/pNpBPw9wam0QlyABPoBHGAlcCVwxz3DOBjwKldbeOAzyaZ2pxr9659C6vql02d99MJ1QBL6YR16ATV1zV1AWyVZCKdSPh4VT09xDUDLOgKl3840LVW1f8kmZhkS2Bn4MvAoXQC/b8Pcd2zqmpRv7b3Jzm62d6Zzvu8HXBzXy1V9fgazjcD+NNm+wvAp7r2fa2qVjbblwBX0wn0s4HPD1EndD4cLElyzjDqXQY8B1zXtC8Fnm0+CC0FpjTtRwB7Z/X9A5Oa418U6KvqIuAigExODaNWSZKkDdpoBvpVquqOJNvSCZMBzqyqC7v7JDmJzozwaQOcYkVXgBxqrBuTnAEc2NX8QeBhOrPxmwAruvY927X9QtfrF1j9/mwCHFhV3ceR5Fg6s/UD1dF9zQBPdR/Kmq/1duDdwL3ALXRC8gxGuO48yWF0PojMqKqn07lBePxIzjGIVddSVT9P8nCSw+l86zJrqIOr6ldJvgy8b5j1Pl9VfeF71c+oql7I6nX8AU6qqgF/HpIkSRur9bKGPskewKZ0ZluvB2Y3M9wk2TGdp57cABzTbJNk6ySvXsshz2D10hXozNb+sqpeAN7Z1DIS8+gsv+m7nqnN5prWz/e/5v4Gu9ZbgA/RWWLzfTrfEjxbVb8eYc2TgCeacLwHqz/gfBc4NMkufWM37U8CW3YdfzvwjmZ7VlPXmlwMfJEXz9wP5dPAHFZ/aFpTvcN1PfDeJOMAkuyeZIsRnkOSJKl11scaeujMnr6rCXvzmjXkdzTLTZYDx1fVj5J8rNm/CfA8nRncB0c6cFX9R5LuddznA1cm+Qs6SzeeGvjINXo/8K9J7qbznt2c5H3A71bVj7v6DXjNXUt1+uob7FpvobPc5Obm2J8DP2bkrgNOTHIPndn+7zZjP9qsI//3ZuxH6KxHvxb4epK30vnwchLw+SSn0Lm/4d2DjHUNnaU2w1luQ1PHY816+A8OVu8IXExn+c2d6bzhj7J6zb8kSdJGK6tXMmgkkhxM54PIib2updeSTKdz0/Ahva5lJDI5hc/CkaRRV3PNGtLLlWRxVU0faN96WUO/MaqqW4Fbe11HryX5CPBehrF2XpIkSete6wJ9s0xjl37Np46FmyE3xGuvqrOAs7rbknwUeHu/rl+rqk+ut8IkSZLGCJfcaMxyyY0krR8uuZFevsGW3Kyv3xQrSZIkaRQY6CVJkqQWM9BLkiRJLWaglyRJklqsdU+5kdaVaZOnsWjuol6XIUmS9LI4Qy9JkiS1mIFekiRJajEDvSRJktRiBnpJkiSpxQz0kiRJUosZ6CVJkqQWM9BLkiRJLWaglyRJklrMQC9JkiS1mIFekiRJajEDvSRJktRiBnpJkiSpxQz0kiRJUosZ6CVJkqQWM9BLkiRJLWaglyRJklrMQC9JkiS1mIFekiRJajEDvSRJktRiBnpJkiSpxQz0kiRJUosZ6CVJkqQWM9BLkiRJLWaglyRJklrMQC9JkiS1mIFekiRJajEDvSRJktRiqape1yD1RCanmNPrKqTRUXP9t12SNiZJFlfV9IH2OUMvSZIktZiBXpIkSWoxA70kSZLUYgZ6SZIkqcUM9JIkSVKLGeglSZKkFjPQS5IkSS1moJckSZJazEAvSZIktdhGHeiTrEyyJMldSe5M8ob1NO78JPc2Yy9JcsxanOOw0a43yelJnk6yfVfb8tEcs2ucbyXZKclbkny/+Rn9KMmcZv/bkrxuGOeZn2TA35omSZI0FmzW6wJG2TNVNRUgyR8BZwJ/0N0hyWZV9dtRGHtWVS16GccfBiwHbh/uAWt5LY8BfwecOsLj1lqSCcA2wMPARcABVfVQklcAU5pubwO+CfxofdUlSZLURhv1DH0/WwFPwKrZ71uSXEMTGJMcn2RBM6N+YZJNk7w9yaeb/ScneaDZfk2S20ZaQJKrkixO8sMkJ3S1v7n5BuGuJDckmQKcCHywqeeQJFOS3Jjk7qbPq5pj/y3JBUm+B3wqyX1Jtmv2bZLkp32v1+AS4NgkW4+g3uVJzm7av53kgGam/IEkRzV9Nm36LGxqntN16sOA+cCWdD5ULgOoqmer6t7mm4mjgLOb6981yZ1d4+/W/bqr/YgkdzTv5deSTBzs5yFJkrQx2NgD/YQmEP4YuBj4h659+wEnV9XuSfYEjgUOamb0VwKzgFuAQ5r+hwDLkuzYbN88xNhf6lpys03TNruqpgHTgfcn2aYJ258D/qyq9gHeXlX/BVwAnFtVU6vqFuBfgEuram/gS8B5XWPtBLyhqv4W+GJTO8Abgbuq6tFB6lxOJ9SfPMC+l9TbtG8B3FhVvwc8CZwBvAk4Gvg/TZ+/An5dVfsD+wPvSbJLs+9I4Lqqehy4BngwyeVJZiXZpKpub9pPaa7/fuDXSaY2x78b+Hx3oUm2BT4GvLGq9gMWAX/b/4KSnJBkUZJFPD3IuyJJktQSY2nJzQzgsiR7NfsWVNXPmu0/BKYBC5MATAAeqar/STIxyZbAzsCXgUPpBPp/H2LsgZbcvD/J0c32zsBuwHbAzX21NCF3IDOAP222vwB8qmvf16pqZbN9CXA18BlgNv2C7xqcByxJcs4w6l0GPAdc17QvBZ6tqueTLGX1kpkjgL277h+Y1Bz/M+Ag4EPN9f51kt+n8+HjQ3Q+GPzlADVeDLw7yd/S+fB1QL/9BwKvA25rfoa/A9zR/yRVdRGdZT5kcmrgt0OSJKk9NvZAv0pV3dHM4vYtP3mqa3fozH6fNsCht9OZEb6Xzoz9bDrh+u9GMn6Sw+iE1hlV9XSS+cD4kZxjEKuupap+nuThJIfTCb2z1nzYqmN+leTLwPuGWe/zVdUXhl8Anm3O80KSvr9TAU6qquu7x0ryGuDnVfVc1/hLgaVJvkAn8P/lAGVeCcwFbgQWV9WyfvsD/GdVHTfU9UqSJG1MNvYlN6sk2QPYlGa9dj83AMekedpLkq2TvLrZdwudmeObge8DM+nMSP96hCVMAp5owvEedGaUAb4LHNq3HKVrLfuTdNaY97kdeEez3bccaE0uprP0pnvmfiifBuaw+kPemuodruuB9yYZB5Bk9yRb0Cy3adomNh8c+kwFHmy2X3T9VbWiOef/y8DfOnwXOCjJ7zbn3iLJ7iOsWZIkqXU29kDft4Z+CXAF8K6BAm5V/YjO+ut5Se4G/hPYodl9C53lJjc3x/4cuHUtarkO2CzJPcBZdAIozfr2E4B/T3JXUyfAtcDRfTfFAifRWXJyN/BOBl7z3ucaYCLDW25DU8djwDeAVwxW7whcTOeG4zuT/AC4kM6HhTezerlOgA+necQn8AlWz85/BTglnUda7tq0fYnONwLzBqj/0ebYy5v36A5gjxHWLEmS1DpZvXJCG4t0nst+blUdMmTn9Sidx1LeVlVr9dz4JB8CJlXVx9dJPZNTzBm6n9RGNdd/2yVpY5Jk8Zoy1JhZQz9WJPkI8F6GsXZ+fauqZ+k8MWfEknwD2BU4fJ0WJUmS1HIG+pehCZm79Gs+tf+NoOtTVZ1FZ4nMKkk+Cry9X9evVdUn11thL1NVHT10L0mSpLHHQP8ytCVkNsG9NeFdkiRJw7ex3xQrSZIkbdQM9JIkSVKLGeglSZKkFjPQS5IkSS1moJckSZJazKfcaMyaNnkai+Yu6nUZkiRJL4sz9JIkSVKLGeglSZKkFjPQS5IkSS1moJckSZJazEAvSZIktZiBXpIkSWoxA70kSZLUYgZ6SZIkqcUM9JIkSVKLGeglSZKkFjPQS5IkSS1moJckSZJazEAvSZIktZiBXpIkSWoxA70kSZLUYgZ6SZIkqcUM9JIkSVKLGeglSZKkFjPQS5IkSS1moJckSZJazEAvSZIktZiBXpIkSWoxA70kSZLUYgZ6SZIkqcUM9JIkSVKLGeglSZKkFktV9boGqScyOcWcXlfRfjXXf0MkSRptSRZX1fSB9jlDL0mSJLWYgV6SJElqMQO9JEmS1GIGekmSJKnFDPSSJElSixnoJUmSpBYz0EuSJEktZqCXJEmSWsxArw1SkrclqSR7NK+nJHkmyZIkdyW5Pclrk2yeZFmSrfodf1WSY3tTvSRJ0vpjoNeG6jjg1ua/fe6vqqlVtQ9wKfD3VfU0cD1wdF+nJJOAg4Fr12O9kiRJPWGg1wYnyUQ6gfyvgHesodtWwBPN9uX9+h0NXN+EfUmSpI3aZr0uQBrAW4HrquonzXKaacAyYNckS4Atgc2B1zf9rwcuTrJNVS2jE+4/24vCJUmS1jdn6LUhOg74SrP9FVYvu+lbcrMr8AHgIoCqeg64BjgmybbAvnRC/kskOSHJoiSLcP5ekiRtBJyh1wYlydbA4cDvJylgU6CAf+3X9Rrg812vLwc+DgS4uqqeH+j8VXURzQeBTE6t2+olSZLWP2fotaE5BvhCVb26qqZU1c7Az4Cd+/U7GLi/6/V8YDfgfXTCvSRJ0pjgDL02NMcB/9iv7UrgNFavoQ/wHPDXfR2q6oUkXwf+HPjOeqpVkiSp51LlqgONTZmcYk6vq2i/muu/IZIkjbYki6tq+kD7XHIjSZIktZiBXpIkSWoxA70kSZLUYgZ6SZIkqcUM9JIkSVKLGeglSZKkFjPQS5IkSS1moJckSZJazEAvSZIktdhmvS5A6pVpk6exaO6iXpchSZL0sjhDL0mSJLWYgV6SJElqMQO9JEmS1GIGekmSJKnFDPSSJElSixnoJUmSpBYz0EuSJEktZqCXJEmSWsxAL0mSJLWYgV6SJElqMQO9JEmS1GIGekmSJKnFDPSSJElSixnoJUmSpBYz0EuSJEktZqCXJEmSWsxAL0mSJLWYgV6SJElqMQO9JEmS1GIGekmSJKnFDPSSJElSixnoJUmSpBYz0EuSJEktZqCXJEmSWsxAL0mSJLWYgV6SJElqsVRVr2uQeiKTU8zpdRXrV831f++SJLVRksVVNX2gfc7QS5IkSS1moJckSZJazEAvSZIktZiBXpIkSWoxA70kSZLUYgZ6SZIkqcUM9JIkSVKLGeglSZKkFjPQS5IkSS1moJckSZJazEA/hiVZmWRJkruS3JnkDaM83vwki7peT08yfzTHlCRJ2tgZ6Me2Z6pqalXtA5wGnLkuTppk00F2b5/kyHUxjiRJkgz0Wm0r4Im+F0lOSbIwyd1JPtHVfnySBc3M/oV94T3J8iT/lOQuYMYg45wNfLR/Y5IpSW5pvilY9W1BksOSfCfJ1UkeSHJWkllNDUuT7Nr02y7JlU3NC5MctG7eFkmSpA3bZr0uQD01IckSYDywA3A4QJIjgN2AA4AA1yQ5FHgUOBY4qKqeT3I+MAu4DNgC+F5V/d0QY94BHJ1kJvBkV/sjwJuqakWS3YDLgenNvn2APYHHgQeAi6vqgCQnAycBHwD+GTi3qm5N8irg+uaYF0lyAnACAJOG9yZJkiRtyAz0Y9szVTUVIMkM4LIkewFHNH++3/SbSCfg7w1MAxYmAZhAJ4gDrASuHOa4ZwAfA07tahsHfDbJ1OZcu3ftW1hVv2zqvB+Y17QvBWY2228EXtfUBbBVkolVtbx74Kq6CLgIIJNTw6xXkiRpg2WgFwBVdUeSbYHt6MzKn1lVF3b3SXIScGlVnTbAKVZU1cphjnVjkjOAA7uaPwg8TGc2fhNgRde+Z7u2X+h6/QKr/w5vAhxYVd3HSZIkbfRcQy8AkuwBbAoso7NcZXaSic2+HZNsD9wAHNNsk2TrJK9eyyHPAD7c9XoS8MuqegF4Z1PLSMyjs/ym73qmrmVdkiRJrTLiGfok/wvYuaruHoV6tH71raGHzqz8u5pZ9nlJ9gTuaJawLAeOr6ofJflYs38T4HngfcCDIx24qv4jyaNdTecDVyb5C+A64KkRnvL9wL8muZvO3+ubgRNHWpckSVLbpGroZcTNs8KPohOUFtNZN31bVf3tqFYnjaJMTjGn11WsXzXX2wYkSWqjJIuravpA+4a75GZSVf0G+FPgsqp6PZ2bECVJkiT10HCX3GyWZAfgzxngGeJStyTfAHbp13xqVV3fi3okSZI2ZsMN9P+Hzo2St1XVwiSvAe4bvbLUZlV1dK9rkCRJGiuGFeir6mvA1wz1vMoAAB6MSURBVLpePwD82WgVJUmSJGl4hrWGPsnuSW5I8oPm9d7N004kSZIk9dBwb4r9HHAanccU0jyy8h2jVZQkSZKk4RluoN+8qhb0a/vtui5GkiRJ0sgM96bYx5LsChRAkmOAX45aVdJ6MG3yNBbNXdTrMiRJkl6W4Qb69wEXAXsk+W/gZ8CsUatKkiRJ0rAMN9BXVb0xyRbAJlX1ZJL+zxmXJEmStJ4Ndw39lQBV9VRVPdm0fX10SpIkSZI0XIPO0CfZA/g9YFKSP+3atRUwfjQLkyRJkjS0oZbcvBZ4C/BK4E+62p8E3jNaRUmSJEkankEDfVVdDVydZEZV3bGeapIkSZI0TMO9KfaEJC+Zka+q2eu4HkmSJEkjMNxA/82u7fHA0cAv1n05kiRJkkZiWIG+qq7sfp3kcuDWUalIkiRJ0rAN97GV/e0GbL8uC5EkSZI0csOaoU/yJFBAmv/+D3DqKNYlSZIkaRiGu+Rmy9EuRJIkSdLIDfemWJpfLHUwnRn6W6rqqlGrSpIkSdKwDGsNfZLzgROBpcAPgBOT/OtoFiZJkiRpaMOdoT8c2LOqCiDJpcAPR60qSZIkScMy3Kfc/BR4VdfrnZs2SZIkST006Ax9kmvprJnfErgnyYLm9euBBaNfniRJkqTBDLXk5pz1UoUkSZKktTJooK+q76yvQiRJkiSN3FBLbm6tqoO7frHUql1AVdVWo1qdJEmSpEENNUN/cPNff7GUJEmStAEa8ik3STZN8uP1UYwkSZKkkRky0FfVSuDeJK8aqq8kSZKk9Wu4v1jqfwE/bB5b+VRfY1UdNSpVSZIkSRqW4Qb6j49qFZIkSZLWynAD/f9dVad2NyT5R8DHWkqSJEk9NOQa+sabBmg7cl0WIkmSJGnkhnoO/XuB/wfYNcndXbu2BG4fzcKk0bb4F4vJJ9LrMiRJUovV3Bq60ygbasnNl4FvAWcCH+lqf7KqHh+1qiRJkiQNy1C/WOrXwK+T/LaqHuzel+QLVfXOUa1OkiRJ0qCGu4b+97pfJNkMmLbuy5EkSZI0EoMG+iSnJXkS2DvJb/r+AA8DV6+XCiVJkiSt0VBLbs4EzkxyJvApYHdgfN/uUa5NkiRJ0hCG+xz6B4CbgZ2AJcCBwB3A4aNUlyRJkqRhGO4a+vcD+wMPVtVMYF/gV6NWlSRJkqRhGW6gX1FVKwCSvKKqfgy8dvTKkiRJkjQcw11y81CSVwJXAf+Z5AngwSGOkYYlyUeB/w2sBF4A5gAzgIuq6ukhjl1eVRNHv0pJkqQN07ACfVUd3WyenuQmYBJw3ahVpTEjyQzgLcB+VfVskm2B3wGuAL4IDBroJUmSxrrhLrlZpaq+U1XXVNVzo1GQxpwdgMeq6lmAqnoMOAaYDNyU5KYks5N8pu+AJO9Jcm7/EyU5JcnCJHcn+cT6ugBJkqReGnGgl9axecDOSX6S5Pwkf1BV5wG/AGY2N2F/FfiTJOOaY94NXNJ9kiRHALsBBwBTgWlJDl1vVyFJktQjBnr1VFUtp/Nbh08AHgWuSPKXA/S5EXhLkj2AcVW1tN+pjmj+fB+4E9iDTsB/kSQnJFmUZJGLeSRJ0sZguDfFSqOmqlYC84H5SZYC7xqg28XA3wM/Bj4/wP4AZ1bVhUOMdRFwEUAmx1+OJkmSWs8ZevVUktcm6Z5Jn0rnCUpPAlv2NVbV94Cd6TwN5/IBTnU9MDvJxOa8OybZftQKlyRJ2kA4Q69emwj8S/NY1N8CP6Wz/OY44Lokv2jW0UNnLf3Uqnqi/0mqal6SPYE7kgAsB44HHlkP1yBJktQzqXLVgdohyTeBc6vqhnVyvskp5qyLM0mSpLGq5q6fLJ1kcVVNH2ifS260wUvyyiQ/AZ5ZV2FekiRpY+GSG23wqupXwO69rkOSJGlD5Ay9JEmS1GIGekmSJKnFDPSSJElSixnoJUmSpBYz0EuSJEktZqCXJEmSWsxAL0mSJLWYz6HXmDVt8jQWzV3U6zIkSZJeFmfoJUmSpBYz0EuSJEktZqCXJEmSWsxAL0mSJLWYgV6SJElqMQO9JEmS1GIGekmSJKnFDPSSJElSixnoJUmSpBYz0EuSJEktZqCXJEmSWsxAL0mSJLWYgV6SJElqMQO9JEmS1GIGekmSJKnFDPSSJElSixnoJUmSpBYz0EuSJEktZqCXJEmSWsxAL0mSJLWYgV6SJElqMQO9JEmS1GIGekmSJKnFDPSSJElSixnoJUmSpBYz0EuSJEktlqrqdQ1ST2Ryijm9rkLqqLn+WyxJWrMki6tq+kD7nKGXJEmSWsxAL0mSJLWYgV6SJElqMQO9JEmS1GIGekmSJKnFDPSSJElSixnoJUmSpBYz0EuSJEktZqCXJEmSWsxAryEl+WiSHya5O8mSJK9P8oEkmw/j2OXDHOOCJAc125sleTTJWf36zE9yb1PDPUlOaNo/n2ROv75vS/Kt4V+lJElSOxnoNagkM4C3APtV1d7AG4GfAx8Ahgz0I3Ag8N1m+03AT4C3J0m/frOqaipwEPCPSX4HuBx4R79+72jaJUmSNmoGeg1lB+CxqnoWoKoeA44BJgM3Jbkpyewkn+k7IMl7kpzb/0RJTkmysJnp/0RX+57AT6pqZdN0HPDPwP8HzFhDXROBp4CVwA3AHkl2aM63BZ0PHle9nAuXJElqAwO9hjIP2DnJT5Kcn+QPquo84BfAzKqaCXwV+JMk45pj3g1c0n2SJEcAuwEHAFOBaUkObXYfCVzX9BtPJ4xfS2eG/bh+9Xwpyd3AvcA/VNXK5oPAlcCfN33+BJhfVb9ZN2+BJEnShstAr0FV1XJgGnAC8ChwRZK/HKDPjcBbkuwBjKuqpf1OdUTz5/vAncAedAI+wB/RBHo6y3tuqqpn6IT0tyXZtOs8s5qlP68CPpTk1U1797KbNS63SXJCkkVJFvH0MN8ESZKkDdhmvS5AG75mBnw+MD/JUuBdA3S7GPh74MfA5wfYH+DMqrrwRY2dG2tfWVW/aJqOAw5O8l/N622Aw4H/7FfTo0nuBF4PPAjcDuyQZB/gDbx0TX3fcRcBFwFkcmrNVy1JktQOztBrUElem2S3rqapdAL0k8CWfY1V9T1gZ+B/M/Ds+PXA7CQTm/PumGR7YCZwU9O2FXAI8KqqmlJVU4D38dJlN30fBPYF7m/GL+AK4FLgW1W14mVctiRJUms4Q6+hTAT+Jckrgd8CP6Wz/OY44Lokv2jW0UNnLf3Uqnqi/0mqal5z8+sdzYNrlgPH01k///Wm29HAjX034DauBj6V5BXN6y8leQZ4BfBvVbW4q+/lwIeBj7zci5YkSWqLdCY2pZcvyTeBc6vqhhEccyfw+qp6fvQqW8PYk1PMGbqftD7UXP8tliStWZLFVTV9oH0uudHLluSVSX4CPDOSMA9QVfv1IsxLkiRtLFxyo5etqn4F7N7rOiRJksYiZ+glSZKkFjPQS5IkSS1moJckSZJazEAvSZIktZiBXpIkSWoxA70kSZLUYj62UmPWtMnTWDR3Ua/LkCRJelmcoZckSZJazEAvSZIktZiBXpIkSWoxA70kSZLUYgZ6SZIkqcUM9JIkSVKLGeglSZKkFjPQS5IkSS1moJckSZJazEAvSZIktZiBXpIkSWoxA70kSZLUYgZ6SZIkqcUM9JIkSVKLGeglSZKkFjPQS5IkSS1moJckSZJazEAvSZIktZiBXpIkSWoxA70kSZLUYgZ6SZIkqcUM9JIkSVKLGeglSZKkFjPQS5IkSS1moJckSZJazEAvSZIktViqqtc1SD2RySnm9LoKqR1qrv9fIUm9lGRxVU0faJ8z9JIkSVKLGeglSZKkFjPQS5IkSS1moJckSZJazEAvSZIktZiBXpIkSWoxA70kSZLUYgZ6SZIkqcUM9JIkSVKLGeglSZKkFjPQb8CSrEyyJMldSe5M8oZRHm9+kkVdr6cnmT+aYzbjjEtyZ9frtyWpJHt0tU1J8kzX+3F7ktcm2TzJsiRb9TvnVUmOHe3aJUmSes1Av2F7pqqmVtU+wGnAmevipEk2HWT39kmOXBfjjMDBwG1dr48Dbm3+2+3+rvfjUuDvq+pp4Hrg6L5OSSY157x2VKuWJEnaABjo22Mr4Im+F0lOSbIwyd1JPtHVfnySBc1M9oV94T3J8iT/lOQuYMYg45wNfLR/YzNDfkvzTcGqbwuSHJbkO0muTvJAkrOSzGpqWJpk16bfdkmubGpemOSgrtO/GfhW028inTD+V8A7hvl+XN6v79HA9U3YlyRJ2qgZ6DdsE5pg/mPgYuAfAJIcAewGHABMBaYlOTTJnsCxwEFVNRVYCcxqzrUF8L2q2qeqbh1kzDuA55LM7Nf+CPCmqtqvGeO8rn37ACcCewLvBHavqgOamk9q+vwzcG5V7Q/8WbOvz0xgfrP9VuC6qvoJsCzJtK5+uzbvx/3A3wKfbtqvB/ZLsk3z+h10Qv5LJDkhyaIkizDuS5KkjcBmvS5Ag3qmCeYkmQFclmQv4Ijmz/ebfhPpBPy9gWnAwiQAE+gEceiE+yuHOe4ZwMeAU7vaxgGfTdL3QWH3rn0Lq+qXTZ33A/Oa9qV0wjrAG4HXNXUBbNXMxk8CHu+aTT+OTvgH+ErzenHz+v6u9+NY4CLgzVX1XJJrgGOSXAnsSyfkv0RVXdQcRyanhvd2SJIkbbgM9C1RVXck2RbYDghwZlVd2N0nyUnApVV12gCnWFFVK4c51o1JzgAO7Gr+IPAwndn4TYAVXfue7dp+oev1C6z+O7YJcGBVdR/XF8yvb7a3Bg4Hfj9JAZsCleSUAcq8Bvh81+vLgY/TeW+urqrnh3OtkiRJbeeSm5ZonviyKbCMTgCe3cxwk2THJNsDN9CZpd6+ad86yavXcsgzgA93vZ4E/LKqXqCzrGawG2sHMo/Vy29oZvqha/08cAzwhap6dVVNqaqdgZ8BhwxwvoOB+7tez6fzLcX7WMNyG0mSpI2RM/QbtglJljTbAd7VzLLPa9bL39EsYVkOHF9VP0rysWb/JsDzdALugyMduKr+I8mjXU3nA1cm+QvgOuCpEZ7y/cC/Jrmbzt+7m5O8D/jdqvpx0+c44B/7HXdlV/uuzfsR4Dngr7vqfSHJ14E/B74zwtokSZJaK1UuI1ZvJDmYzgeRE3sy/uQUc3oxstQ+Ndf/r5CkXkqyuKqmD7TPGXr1TPO0ncGeuCNJkqQhGOjHoCTfAHbp13xqVQ34ZBhJkiRtuAz0Y1BVHT10L0mSJLWBT7mRJEmSWsxAL0mSJLWYgV6SJElqMQO9JEmS1GLeFKsxa9rkaSyau6jXZUiSJL0sztBLkiRJLWaglyRJklrMQC9JkiS1mIFekiRJajEDvSRJktRiBnpJkiSpxQz0kiRJUosZ6CVJkqQWM9BLkiRJLWaglyRJklrMQC9JkiS1mIFekiRJajEDvSRJktRiBnpJkiSpxQz0kiRJUosZ6CVJkqQWM9BLkiRJLWaglyRJklrMQC9JkiS1mIFekiRJajEDvSRJktRiBnpJkiSpxQz0kiRJUosZ6CVJkqQWM9BLkiRJLWaglyRJklrMQC9JkiS1WKqq1zVIPZHJKeb0ugqpHWqu/18hSb2UZHFVTR9onzP0kiRJUosZ6CVJkqQWM9BLkiRJLWaglyRJklrMQC9JkiS1mIFekiRJajEDvSRJktRiBnpJkiSpxQz0kiRJUosZ6DcgSVYmWZLkriR3JnnDKI83P8mirtfTk8wfzTGbccYlubPZ/miSHya5u7n21zftH0iy+TDOtXy065UkSdqQGeg3LM9U1dSq2gc4DThzXZw0yaaD7N4+yZHrYpwROBi4LckM4C3AflW1N/BG4OdNnw8AQwZ6SZKksc5Av+HaCnii70WSU5IsbGayP9HVfnySBc3s9oV94T3J8iT/lOQuYMYg45wNfLR/Y5IpSW5pvilY9W1BksOSfCfJ1UkeSHJWkllNDUuT7Nr02y7JlU3NC5Mc1HX6NwPfAnYAHquqZwGq6rGq+kWS9wOTgZuS3JRkdpLPdNX2niTnDlDzgO+RJEnSxsxAv2GZ0ATzHwMXA/8AkOQIYDfgAGAqMC3JoUn2BI4FDqqqqcBKYFZzri2A71XVPlV16yBj3gE8l2Rmv/ZHgDdV1X7NGOd17dsHOBHYE3gnsHtVHdDUfFLT55+Bc6tqf+DPmn19ZgLzgXnAzkl+kuT8JH8AUFXnAb8AZlbVTOCrwJ8kGdcc/27gku5i1/Qe9b/YJCckWZRkEU8P8q5IkiS1xGa9LkAv8kwTzGmWo1yWZC/giObP95t+E+mE172BacDCJAAT6ARx6IT7K4c57hnAx4BTu9rGAZ9N0vdBYfeufQur6pdNnffTCeYAS+mEdegsn3ldUxfAVkkmApOAx6vq6eb4acAhzXFXJPlIVf1bd3FVtTzJjcBbktwDjKuqpf2uYU3v0c39znURcBFAJqeGeF8kSZI2eAb6DVRV3ZFkW2A7IMCZVXVhd58kJwGXVtVpA5xiRVWtHOZYNyY5Aziwq/mDwMN0ZuM3AVZ07Xu2a/uFrtcvsPrv1CbA/9/O/cfqWdZ3HH9/bJ1rgKENaChWwI2NLWRh0hFxwDqdzu2PaRcUHW6dLKEapmzZJs6ZlD80I4wxs2X+QIbWTdxmAK3LAnU4AopDWlYoFqsEMcgQBJRRNYrtd3/c96FPyzn93XM/1/O8X//0PveP5/6ec/XK+TzX+d7PS6tq9DqSnAPcMHLvbXSr9Tcl2QSsBD46S5lXAu8CvgJ8ZJbjs/6MJEmSJp0tN2MqyUnAAuAxugB8Xr/CTZJjkzwfuBE4u98myeIkx+3nLd8DvGPk6yOBh6pqO11bze4erJ3NOna039Cv9MOO/nmS/FySE0euOQX4Rr/9JHDEzIGqug1YCvwu8IlZ7jfXz0iSJGmiuUI/XhYl2dhvB1jZr2Cv6/vlv9i3sGwF3lRVm5O8uz/+LOAp4AJ2hOK9VlX/keTbI7veD1yT5PeB64Hv7eNLvh34hyR30f0/uznJBcDPVNVX+nMOB/4+yXOBHwP3Auf3x64Ark/yv30fPXS99KdU1dMPC4/UP+vPiB0tSJIkSRMpVbYRa34kOYPujchb9vP6f6d70PbGg1LPkhSrDsYrSZOvVvu7QpKGlGRDVS2b7ZgtN5o3VfX5/QnzSZ6b5Kt0Dw0flDAvSZI0KWy5mQJJrgNO2GX3RVV1w2znj5uq+i47f8qOJEmSegb6KVBVK4auQZIkSYeGLTeSJElSwwz0kiRJUsMM9JIkSVLDDPSSJElSwwz0kiRJUsP8lBtNrVOXnMr61euHLkOSJOmAuEIvSZIkNcxAL0mSJDXMQC9JkiQ1zEAvSZIkNcxAL0mSJDXMQC9JkiQ1zEAvSZIkNcxAL0mSJDXMQC9JkiQ1zEAvSZIkNcxAL0mSJDXMQC9JkiQ1zEAvSZIkNcxAL0mSJDXMQC9JkiQ1zEAvSZIkNcxAL0mSJDXMQC9JkiQ1zEAvSZIkNcxAL0mSJDXMQC9JkiQ1zEAvSZIkNcxAL0mSJDXMQC9JkiQ1zEAvSZIkNcxAL0mSJDUsVTV0DdIgsiTFqqGrkDRNarW/cyXtnyQbqmrZbMdcoZckSZIaZqCXJEmSGmaglyRJkhpmoJckSZIaZqCXJEmSGmaglyRJkhpmoJckSZIaZqCXJEmSGmaglyRJkhpmoJckSZIaZqA/iJJsS7IxyZ1J7kjysnm6701JtvT33pjk7P14jeWHut4kFyf5fpLnj+zbeijvKUmSNOkWDl3AhPlBVZ0CkOQ3gL8CfnX0hCQLq+rHh+De51bV+gO4fjmwFbh1by/Yz+/lUeBPgYv28TpJkiTNwhX6Q+engO/A06vftyRZC2zu970pyZf6FfUPJVmQ5HVJLu+PX5jkvn77xUm+sK8FJPlUkg1Jvpzk/JH9r+7/gnBnkhuTHA+8BfiTvp4zkxyf5HNJ7urPeVF/7UeTfDDJbcClSb6W5Oj+2LOS3Dvz9RyuAs5Jsngf6t2a5K/7/f+Z5LT+rxL3Jfnt/pwF/Tm39zWv2teflyRJUotcoT+4FiXZCPwkcAzw8pFjLwFOrqqvJ/l54BzgV6rqqSTvB84F1gHv6M8/E3gsybH99s17uPfHk/yg335FVT0GnFdVjydZBNye5Bq6N3EfBs7qa1ncn/NBYGtVXQaQ5DPAmqpak+Q84O+A1/av/0LgZVW1LckTfe3vA34duLOqvr2bOrfShfoLgdW7HHtGvf33cRjwuar68yTXAe8BXgn8ArAGWAv8IfBEVf1ykucAX0iyrqq+PnqD/o1C92bhyD38RCVJkhpgoD+4RltuTgc+luTk/tiXRsLlK4BT6UIrwCLgkar6VpLDkxwBLAWuBs6iC/TX7uHes7XcvD3Jin57KXAicDRw80wtVfX4HK93OvA7/fY/AZeOHPtkVW3rt68CPk0X6M8DPrKHOqF7c7AxyWV7Ue9jwI+A6/v9m4Af9m+ENgHH9/tfBfziyPMDR/bX7xToq+oK4AqALEntRa2SJEljzUB/iFTVF5McRRegAb43cjh0q99/McultwJvBrYAt9CF5NPp+s73WpLldCvmp1fV95PcRPeXg4Ph6e+lqh5I8nCSlwOn0a3W71ZVfTfJ1cAFe1nvU1U1E763Az/sX2d7kpn/wwHeVlU3HNB3JkmS1Bh76A+RJCcBC+hWmHd1I3D2zKe9JFmc5Lj+2C3An9G12PwP8Gt0K9JP7GMJRwLf6cPxScBL+/3/DZyV5ISZe/f7nwSOGLn+VuAN/fa5fV1zuRL4Z3Zeud+Ty4FV7HhTOVe9e+sG4K1Jng2Q5GeTHLaPryFJktQcA/3BtWjmoyOBfwVWzhZwq2oz8G5gXZK7gM/S9dxDF5yX0rXFbAMeAD6/H7VcDyxMcg9wCV2Qp+9vPx+4NsmdfZ0AnwFWzDwUC7wNeHNf3+/R9bzPZS1wOHvXbkNfx6PAdcBzdlfvPriS7oHjO5LcDXwI/wIlSZKmQHZ0Mkj7J8ky4G+r6syha9kXWZLCz8KRNI9qtb9zJe2fJBuqatlsx1zB1AFJ8k7grexF77wkSZIOPgN9Q/qPbDxhl90XDfkgaFVdQtci87Qkfwm8bpdTP1lV7523wiRJkqaEgb4hVbViz2cNrw/uhndJkqR54EOxkiRJUsMM9JIkSVLDDPSSJElSwwz0kiRJUsN8KFZT69Qlp7J+9fqhy5AkSTogrtBLkiRJDTPQS5IkSQ0z0EuSJEkNM9BLkiRJDTPQS5IkSQ0z0EuSJEkNM9BLkiRJDTPQS5IkSQ0z0EuSJEkNM9BLkiRJDTPQS5IkSQ0z0EuSJEkNM9BLkiRJDUtVDV2DNIgkTwJbhq5Dz3AU8OjQRegZHJfx5diMJ8dlPLU8LsdV1dGzHVg435VIY2RLVS0bugjtLMl6x2X8OC7jy7EZT47LeJrUcbHlRpIkSWqYgV6SJElqmIFe0+yKoQvQrByX8eS4jC/HZjw5LuNpIsfFh2IlSZKkhrlCL0mSJDXMQK+plOTVSbYkuTfJO4euR50k9yfZlGRjkvVD1zOtklyV5JEkd4/sW5zks0m+1v/7vCFrnEZzjMvFSR7s58zGJL81ZI3TKMnSJP+VZHOSLye5sN/vnBnQbsZlIueMLTeaOkkWAF8FXgl8E7gdeGNVbR60MJHkfmBZVbX6GcETIclZwFbgY1V1cr/vUuDxqrqkfxP8vKq6aMg6p80c43IxsLWqLhuytmmW5BjgmKq6I8kRwAbgtcAf4JwZzG7G5fVM4JxxhV7T6DTg3qq6r6p+BPwL8JqBa5LGRlXdDDy+y+7XAGv67TV0vxg1j+YYFw2sqh6qqjv67SeBe4Bjcc4MajfjMpEM9JpGxwIPjHz9TSZ4kjemgHVJNiQ5f+hitJMXVNVD/fa3gBcMWYx28kdJ7upbcmzrGFCS44FfAm7DOTM2dhkXmMA5Y6CXNE7OqKqXAL8JXNC3GGjMVNerab/mePgA8NPAKcBDwN8MW870SnI4cA3wx1X1f6PHnDPDmWVcJnLOGOg1jR4Elo58/cJ+nwZWVQ/2/z4CXEfXHqXx8HDfkzrTm/rIwPUIqKqHq2pbVW0HPoxzZhBJnk0XGj9eVdf2u50zA5ttXCZ1zhjoNY1uB05MckKSnwDeAKwduKapl+Sw/sElkhwGvAq4e/dXaR6tBVb22yuBTw9Yi3ozgbG3AufMvEsS4B+Be6rq8pFDzpkBzTUukzpn/JQbTaX+Y6reBywArqqq9w5c0tRL8mK6VXmAhcDVjsswknwCWA4cBTwMrAY+Bfwb8CLgG8Drq8oHNOfRHOOynK51oID7gVUjfduaB0nOAG4BNgHb+93vouvXds4MZDfj8kYmcM4Y6CVJkqSG2XIjSZIkNcxAL0mSJDXMQC9JkiQ1zEAvSZIkNcxAL0mSJDXMQC9JkiQ1zEAvSZIkNcxAL0mSJDXs/wFuchfx3As7zgAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 720x720 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt,posRanking0_rnn =aggregateRankings(pos_rankings0_rnn,positive=True,maxLenAttributes=2)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Mine patterns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-02-18T11:05:08.102145Z",
     "start_time": "2020-02-18T11:05:07.620632Z"
    }
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAABIEAAAFOCAYAAAARsA0HAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8li6FKAAAgAElEQVR4nO3debhld10m+vcbAkTGqKQFgRCIXGhkNkyCNoP0BYHggAMyKA1GERUnbBzRtK22jRPYgGGwmURAQZkRmlkQMhACYbgdcACae5lDGAQC3/vHWid16nCqzqnKqay1an0+z3OeOnvtfare5Nln77Xf9RuquwMAAADA0e2YqQMAAAAAcOQpgQAAAABWQAkEAAAAsAJKIAAAAIAVUAIBAAAArIASCAAAAGAFjp3qH77GNa7RJ5100lT/PAAAAMBR5+yzz/54d5+w3X2TlUAnnXRSzjrrrKn+eQAAAICjTlX9y4HuMx0MAAAAYAWUQAAAAAAroAQCAAAAWAElEAAAAMAKKIEAAAAAVkAJBAAAALACSiAAAACAFdixBKqq46rqbVX1jqo6v6p+e5vHXLGqnltVF1TVW6vqpCMRFgAAAIDDs5uRQF9MctfuvkWSWya5R1XdfstjHprkU939LUn+OMl/29uYAAAAAFwaO5ZAPfjsePPy41dvedh9kzx9/P6vk9ytqmrPUgIAAABwqexqTaCqulxVnZvko0le1d1v3fKQayf5YJJ098VJLkzyjXsZFAAAAIDDd+xuHtTdX0lyy6o6PskLq+qm3f2uQ/3Hquq0JKclyYknnnioPw4AAAAcJU569EunjnCJf/79e00d4TJxSLuDdfenk7w2yT223PXhJNdNkqo6NsnVk3xim58/o7tP6e5TTjjhhMNLDAAAAMAh283uYCeMI4BSVV+X5O5J3rvlYS9K8qPj9/dL8pru3rpuEAAAAAAT2c10sGsleXpVXS5DafS87n5JVZ2e5KzuflGSpyZ5ZlVdkOSTSX74iCUGAAAA4JDtWAJ193lJbrXN8d/c9P2/JfmBvY0GAAAAwF45pDWBAAAAAFgmJRAAAADACiiBAAAAAFZACQQAAACwAkogAAAAgBVQAgEAAACsgBIIAAAAYAWUQAAAAAAroAQCAAAAWAElEAAAAMAKKIEAAAAAVkAJBAAAALACSiAAAACAFVACAQAAAKyAEggAAABgBZRAAAAAACugBAIAAABYASUQAAAAwAoogQAAAABWQAkEAAAAsAJKIAAAAIAVUAIBAAAArIASCAAAAGAFlEAAAAAAK6AEAgAAAFgBJRAAAADACiiBAAAAAFZACQQAAACwAkogAAAAgBVQAgEAAACsgBIIAAAAYAWUQAAAAAAroAQCAAAAWAElEAAAAMAK7FgCVdV1q+q1VfXuqjq/qh65zWPuXFUXVtW549dvHpm4AAAAAByOY3fxmIuT/GJ3n1NVV01ydlW9qrvfveVxb+zue+99RAAAAAAurR1HAnX3R7r7nPH7i5K8J8m1j3QwAAAAAPbOIa0JVFUnJblVkrduc/cdquodVfXyqvrWPcgGAAAAwB7ZzXSwJElVXSXJ3yT5ue7+zJa7z0lyve7+bFV9d5K/TXLDbf6O05KcliQnnnjiYYcGAAAA4NDsaiRQVV0+QwH07O5+wdb7u/sz3f3Z8fuXJbl8VV1jm8ed0d2ndPcpJ5xwwqWMDgAAAMBu7WZ3sEry1CTv6e4/OsBjrjk+LlV12/Hv/cReBgUAAADg8O1mOtgdkzwoyTur6tzx2K8mOTFJuvtJSe6X5OFVdXGSLyT54e7uI5AXAAAAgMOwYwnU3W9KUjs85s+S/NlehQIAAABgbx3S7mAAAAAALJMSCAAAAGAFlEAAAAAAK6AEAgAAAFgBJRAAAADACiiBAAAAAFZACQQAAACwAkogAAAAgBVQAgEAAACsgBIIAAAAYAWUQAAAAAAroAQCAAAAWAElEAAAAMAKKIEAAAAAVkAJBAAAALACSiAAAACAFVACAQAAAKyAEggAAABgBZRAAAAAACugBAIAAABYASUQAAAAwAoogQAAAABWQAkEAAAAsAJKIAAAAIAVUAIBAAAArIASCAAAAGAFlEAAAAAAK6AEAgAAAFgBJRAAAADACiiBAAAAAFZACQQAAACwAkogAAAAgBVQAgEAAACsgBIIAAAAYAV2LIGq6rpV9dqqendVnV9Vj9zmMVVVj6uqC6rqvKq69ZGJCwAAAMDhOHYXj7k4yS929zlVddUkZ1fVq7r73Zsec88kNxy/bpfkieOfAAAAAMzAjiOBuvsj3X3O+P1FSd6T5NpbHnbfJM/owT8mOb6qrrXnaQEAAAA4LIe0JlBVnZTkVkneuuWuayf54KbbH8rXFkUAAAAATGQ308GSJFV1lSR/k+Tnuvszh/OPVdVpSU5LkhNPPPFw/goAAIA9ddKjXzp1hP388+/fa8fHzCnz0vImy8u8tLzJ7jJz2dvVSKCqunyGAujZ3f2CbR7y4STX3XT7OuOx/XT3Gd19SnefcsIJJxxOXgAAAAAOw252B6skT03ynu7+owM87EVJHjzuEnb7JBd290f2MCcAAAAAl8JupoPdMcmDkryzqs4dj/1qkhOTpLuflORlSb47yQVJPp/kIXsfFQAAAIDDtWMJ1N1vSlI7PKaTPGKvQgEAAACwtw5pdzAAAAAAlkkJBAAAALACSiAAAACAFVACAQAAAKyAEggAAABgBZRAAAAAACugBAIAAABYASUQAAAAwAoogQAAAABWQAkEAAAAsAJKIAAAAIAVUAIBAAAArIASCAAAAGAFlEAAAAAAK6AEAgAAAFgBJRAAAADACiiBAAAAAFZACQQAAACwAkogAAAAgBVQAgEAAACsgBIIAAAAYAWUQAAAAAAroAQCAAAAWAElEAAAAMAKKIEAAAAAVkAJBAAAALACSiAAAACAFVACAQAAAKyAEggAAABgBZRAAAAAACugBAIAAABYASUQAAAAwAoogQAAAABWQAkEAAAAsAJKIAAAAIAV2LEEqqqnVdVHq+pdB7j/zlV1YVWdO3795t7HBAAAAODSOHYXj/mfSf4syTMO8pg3dve99yQRAAAAAHtux5FA3f2GJJ+8DLIAAAAAcITs1ZpAd6iqd1TVy6vqWw/0oKo6rarOqqqzPvaxj+3RPw0AAADATvaiBDonyfW6+xZJHp/kbw/0wO4+o7tP6e5TTjjhhD34pwEAAADYjUtdAnX3Z7r7s+P3L0ty+aq6xqVOBgAAAMCeudQlUFVds6pq/P6249/5iUv79wIAAACwd3bcHayqnpPkzkmuUVUfSvKYJJdPku5+UpL7JXl4VV2c5AtJfri7+4glBgAAAOCQ7VgCdff9d7j/zzJsIQ8AAADATO3V7mAAAAAAzJgSCAAAAGAFlEAAAAAAK6AEAgAAAFgBJRAAAADACiiBAAAAAFZACQQAAACwAkogAAAAgBVQAgEAAACsgBIIAAAAYAWUQAAAAAAroAQCAAAAWAElEAAAAMAKKIEAAAAAVkAJBAAAALACSiAAAACAFVACAQAAAKyAEggAAABgBZRAAAAAACugBAIAAABYASUQAAAAwAoogQAAAABWQAkEAAAAsAJKIAAAAIAVUAIBAAAArIASCAAAAGAFlEAAAAAAK6AEAgAAAFgBJRAAAADACiiBAAAAAFZACQQAAACwAkogAAAAgBVQAgEAAACsgBIIAAAAYAV2LIGq6mlV9dGqetcB7q+qelxVXVBV51XVrfc+JgAAAACXxm5GAv3PJPc4yP33THLD8eu0JE+89LEAAAAA2Es7lkDd/YYknzzIQ+6b5Bk9+Mckx1fVtfYqIAAAAACX3rF78HdcO8kHN93+0HjsI1sfWFWnZRgtlBNPPHEP/ul5OOnRL506wn7++ffvddD7l5Y3WV7mpeVNlpd5aXmT5WVeWt5keZmXljdZXual5U3mlXlpeZPlZV5a3mR5mXeTF2AtLtOFobv7jO4+pbtPOeGEEy7LfxoAAABg1faiBPpwkutuun2d8RgAAAAAM7EXJdCLkjx43CXs9kku7O6vmQoGAAAAwHR2XBOoqp6T5M5JrlFVH0rymCSXT5LuflKSlyX57iQXJPl8koccqbAAAAAAHJ4dS6Duvv8O93eSR+xZIgAAAAD23GW6MDQAAAAA01ACAQAAAKyAEggAAABgBZRAAAAAACugBAIAAABYASUQAAAAwAoogQAAAABWQAkEAAAAsAJKIAAAAIAVUAIBAAAArIASCAAAAGAFlEAAAAAAK6AEAgAAAFgBJRAAAADACiiBAAAAAFZACQQAAACwAkogAAAAgBVQAgEAAACsgBIIAAAAYAWUQAAAAAAroAQCAAAAWAElEAAAAMAKKIEAAAAAVkAJBAAAALACSiAAAACAFVACAQAAAKyAEggAAABgBZRAAAAAACugBAIAAABYASUQAAAAwAoogQAAAABWQAkEAAAAsAJKIAAAAIAVUAIBAAAArMCuSqCqukdVva+qLqiqR29z/49V1ceq6tzx62F7HxUAAACAw3XsTg+oqssl+R9J7p7kQ0nOrKoXdfe7tzz0ud3900cgIwAAAACX0m5GAt02yQXd/YHu/lKSv0py3yMbCwAAAIC9tJsS6NpJPrjp9ofGY1t9f1WdV1V/XVXX3e4vqqrTquqsqjrrYx/72GHEBQAAAOBw7NXC0C9OclJ33zzJq5I8fbsHdfcZ3X1Kd59ywgkn7NE/DQAAAMBOdlMCfTjJ5pE91xmPXaK7P9HdXxxvPiXJt+1NPAAAAAD2wm5KoDOT3LCqrl9VV0jyw0letPkBVXWtTTdPTfKevYsIAAAAwKW14+5g3X1xVf10klcmuVySp3X3+VV1epKzuvtFSX62qk5NcnGSTyb5sSOYGQAAAIBDtGMJlCTd/bIkL9ty7Dc3ff8rSX5lb6MBAAAAsFf2amFoAAAAAGZMCQQAAACwAkogAAAAgBVQAgEAAACsgBIIAAAAYAWUQAAAAAAroAQCAAAAWAElEAAAAMAKKIEAAAAAVkAJBAAAALACSiAAAACAFVACAQAAAKyAEggAAABgBZRAAAAAACugBAIAAABYASUQAAAAwAoogQAAAABWQAkEAAAAsAJKIAAAAIAVUAIBAAAArIASCAAAAGAFlEAAAAAAK6AEAgAAAFgBJRAAAADACiiBAAAAAFZACQQAAACwAkogAAAAgBVQAgEAAACsgBIIAAAAYAWUQAAAAAAroAQCAAAAWAElEAAAAMAKKIEAAAAAVkAJBAAAALACuyqBquoeVfW+qrqgqh69zf1XrKrnjve/tapO2uugAAAAABy+HUugqrpckv+R5J5JbpLk/lV1ky0Pe2iST3X3tyT54yT/ba+DAgAAAHD4djMS6LZJLujuD3T3l5L8VZL7bnnMfZM8ffz+r5Pcrapq72ICAAAAcGlUdx/8AVX3S3KP7n7YePtBSW7X3T+96THvGh/zofH2+8fHfHzL33VaktPGmzdK8r69+g85Slwjycd3fNR8LC1vsrzMS8ubLC/z0vImy8u8tLzJ8jIvLW+yvMxLy5ssL/PS8ibLy7y0vMnyMi8tb7K8zEvLmywv89LyJsvMfCRdr7tP2O6OYy/LFN19RpIzLst/c0mq6qzuPmXqHLu1tLzJ8jIvLW+yvMxLy5ssL/PS8ibLy7y0vMnyMi8tb7K8zEvLmywv89LyJsvLvLS8yfIyLy1vsrzMS8ubLDPzVHYzHezDSa676fZ1xmPbPqaqjk1y9SSf2IuAAAAAAFx6uymBzkxyw6q6flVdIckPJ3nRlse8KMmPjt/fL8lreqd5ZgAAAABcZnacDtbdF1fVTyd5ZZLLJXlad59fVacnOau7X5TkqUmeWVUXJPlkhqKIQ7e0qXJLy5ssL/PS8ibLy7y0vMnyMi8tb7K8zEvLmywv89LyJsvLvLS8yfIyLy1vsrzMS8ubLC/z0vImy8u8tLzJMjNPYseFoQEAAABYvt1MBwMAAABg4ZRAAAAAACugBAIAAABYASUQACSpquvv5thcLC0vMI2qOrWqHjt+3WfqPHBpVdU1p84AS2Zh6IlV1cnd/f6pc+xWVV05yRe6+6tV9X8luXGSl3f3lyeO9jWq6nJJXt3dd5k6y+Gqqq9Pct3uPm/qLDupqmt29/97oNtzU1XXTnK9bNolsbvfMF2iA6uqF2TYhfHl3f3VqfMcTFU9PskB31i6+2cvwziHpKrO6e5bbzl2dnd/21SZDmaBeSvJA5LcoLtPr6oTk1yzu982cbRtje8h53f3jafOwnxU1Q909/N3OjYXVfV7SW6b5NnjofsnObO7f3W6VLtTVddLcsPufnVVfV2SY7v7oqlzHQ2W9nq81Xbvf3NUVVdM8v1JTsr+55unT5XpaFBV33Cw+7v7k5dVlqXacYt4jrinVdV1kpyZ5I1J3tDd75w408G8Icl3jOXE32fI/UMZ3khmpbu/UlVfraqrd/eFU+fZrap6XZJTM/x+np3ko1X1D939C5MG29lTk9zrILdno6r+W4bn7buTfGU83Bme33P0hCQPSfK4qnp+kr/o7vdNnOlAzhr/vGOSmyR57nj7BzL8/56dqrpxkm9NcvWq+r5Nd10tyXHTpDqwpeXd5AlJvprkrklOT3JRkr9JcpspQx3I+B7yvqo6sbv/deo8O6mqi7J9AVtJuruvdhlH2tF4MemJSb6pu29aVTdPcmp3/87E0Q7mV5JsLXy2OzYX90pyy40LCFX19CRvTzLrEqiqfjzJaUm+IcnJSa6T5ElJ7jZlrq2q6sU5+IWPUy/DOIdiUa/H26ipA+zS3yW5MMP5/BcnznJAC3wen50hbyU5Mcmnxu+PT/KvSYyK3oESaGLd/R+q6goZXnTvnOSlVXWV7j5owzmh6u7PV9VDkzyhu/+gqs6dOtRBfDbJO6vqVUk+t3FwzqMRkly9uz9TVQ9L8ozufkxVzX4kUHff62C3Z+Z7ktyou2f7hrxZd786yaur6uoZruK+uqo+mOTJSZ41p5F43f30JKmqhye5U3dfPN5+Uoaie45ulOTeGU4eNk+VuCjJj0+S6OCWlnfD7br71lX19iTp7k+N739z9vVJzq+qt2X/95C5nRCnu686dYbD8OQkj0ry50nS3edV1V8mmV0JVFX3TPLdSa5dVY/bdNfVklw8TapdOz7JxpXxq08Z5BA8IsMIprcmSXf/76r6d9NG2tZjxz+/L8k1kzxrvH3/JP/fJIl2Z4mvx5s9eeoAu3Sd7r7H1CF24bE7P2Q+uvv6SVJVT07ywu5+2Xj7nhnO8dmBEmhiVXWnJN8xfh2f5CWZ7welZBhBeocMI38eOh673IR5dvKC8WtJjq2qayX5wSS/NnWYo9QHklw+M74qs1VVfWOSByZ5UIaruM9OcqckP5qhQJ6br8/w4Wjjg8dVxmOz091/l+TvquoO3f2WqfPsZGl5N/nyOMWqk6SqTshwJXrOfmPqAIdr/MB8yciwmY5mulJ3v22YmXKJuRYq/yfDSMdTM1yF3nBRkp+fJNHu/F6St1fVazNcKf/OJI+eNtKufLG7v7Tx3KiqY3OQkQpT6e7XJ0lV/WF3n7LprhdX1VkH+LE5WOLr8SW6+wlTZ9ilN1fVzWY+y+OS53GSjFMvT5zxiPPNbt/dl1z86u6XV9UfTBloKZRA03tdhpOJ30vysu7+0rRxdvRzGYY9v7C7z6+qGyR57cSZDmhjVMLCnJ7klUn+obvPHP8f/++JMx1tPp/k3Kr6X9lUBM11hFhVvTDD6I9nJrlPd39kvOu5Mz7J/P187QeP35o00c6+t6rOT/KFJK9IcvMkP9/dzzr4j03mgqr61XztWgP/abJEB/e4JC9M8u+q6r8muV+SX5820sF19+u3rEtypcz7wkeq6tQkf5jkm5N8NMPaZ+/JMIVwbj5eVSdn3wfR+yX5yMF/ZBrd/Y4k76iqv5zT6MuddPdzxmnmt8nw//k/z3m9vk1eP76+fV1V3T3JTyV58cSZDubKVXWD7v5Acski/VeeONPBLO71eKHulOTHquqfMpxvbkzPvfm0sbY3Lhz/2CRXSHL9qrplktPnOPp19H+q6tezbwTeAzIU9uzAwtATq6rjM6yd8Z0Z3qC/muQt3b3Yq49zMr7ofs2TvLtvMEEcZqKqfnS743MtDavquzeGum46dsW5T2cbd++4XYbfwbfN/YNHVZ3b3besqu/NMN3qFzKs03aLiaNtq6renGHk6NnZt7ZVuvtvJgu1g3E9o7tlOBH+X939nokjHdTmdUm6++SqumGSJ3X3rNYl2ayq3pFhnY9Xd/etquouSR7Y3Q/d4Ucvc+NFjjOSfHuGNR3+KckDuvtfJg12EFV1xwyF9sbGAhsf6mZ7XjGuHXanDK/Fb+ruF04caUdVdUyGEef/McP/41cmeUrP9INLVd0jw3P5AxnyXi/JT3T3KycNdhBLez1eovEiwteY62tcVZ2d4f3jdd19q/HYO7v7ZtMm2964QPRjMnyO3ljb83QLQ+/MSKCJdfenq+oDSa6bYdG7b88wTWWWxqv625Uqd50gzm5sHpp7XIbFaee63lKSxS6UuShzLXsO4neSvGzLsbckmfvOGLfNMNU1GV435nwVN9n32nuvJM/v7gu3TFOZmyt193+eOsRujbvPfD6bngcLWHR5KeuSbPbl7v5EVR1TVcd092ur6k+mDrXV+CH/lO7+rhp2Hj1mITs/PTXD9K/9yte5qqonJPmWJM8ZD/1EVX1Xdz9iwli78T0Z1kVcxNov3f2KsSTe2E3wvXO8UFP776r00ex7XqSqvsGH573V3f9SVbfIvnOhN46jCufqy9uc+8yyeE0u2QXskVV15e7+3I4/wCWUQBMbC6D3JnlThg/+D5n5lLBf2vT9cRm2PZzr/P109ye2HPqTseX+zSny7NJiFspcmqp6Xnf/YFW9M9uXmbManjuOpLl2huHwt8q+3TCuluRKkwXbhar6/QyjGze2Jf7ZcQ2bOe9I8+Kqem+G6WAPH9dI+LeJMx3MS7YbJTZjL82+3TyOy7B7x/syz2lKGxaxLskWn66qq2S4IvrsqvpoNi1qPRfd/dWq+uUkz1vYyfuF3f3yqUMcgrsm+fcbI2hq2B3s/Gkj7cp9kvxxVb0hwy6Tr9jYaGCOxqmiv5Dket3941V1w6q6UXe/ZOpsW2zeVSnZ93pW4/ezHdG2RFX1yAwbNmysT/qsqjqjux8/YayDOb+qfiTJ5cZS82eTvHniTAdUVd+e5CkZ1p08cSzcfqK7f2raZPNnOtjExqt0i1mIbTtV9bbuvu3UObZTVZtHShyTYWTQw+c6vSNJqurM7r5NVb1901DMc7v7llNnW7qqulZ3f2Qpw3PHaWs/luF5e2b2nbR9JsnTu3u2i57XsKPd5m2JL5fk7XMr2rYar5Je2MP24FdKcrW5TmOrYUvwK2dYZ+DLmfFW4NsZX59/qrsfNnWWAxkXmPx0kgcn+ZkM65K8u7tnu2j/OKrm3zI8Hx6QYTeoZ29zUWRyY1n88Qwf8jfvvja70Qibzid+MMO6UC/I/mvKnTNFrp1U1UuSPGLj/W18//uz7r7PwX9yelV1+ST3TPJDGaazvWqurxdV9dwMBcuDx1HcV0ry5jmfu43vdzfM/gvIv/7AP8GhGs+F7rBRdI+vz2+Z67nQ+Lz9tQzTMJNhGubvdPcsL4hV1VszrGf1ok2fmd7V3TedNtn8GQk0vW+pqsVM/dkyjHSjVJnzdqN/uOn7i5P8c4YTuDlbzEKZS7OxoPLcyp4D6e6nV9Uzk9y/u5+94w/MzxK3Jf7mJN9VVcdtOvaMqcIcTC9zS/BLdPc5VXW7qXPs4NEZ1iV5Z5KfyDAt8ymTJtrBllE1c5/6+kPjn5unJs11NMIfbrm9ebp5ZxhxMxtV9eIMua6a5D1V9bbx9u2SvG3KbLvV3V+uqpdnyP11GaaIzbIESnJyd/9QVd0/Sbr78zXj+cRV9bAkj8ywFMW5SW6fYcTHbNc7W6jK/tNGv5J9F/RmZbxYd3p3/1IWtDtxd39wy6/a7KfpzoESaHpLm/qzMYw02VeqzG6xyQ3dfZepMxyGR2RYXPDGVfXhDAtlPnDaSEeHceTEdsMfZzuCYpwy8fPZN61qKRa3LXFVPSbJnZPcJMOH/XtmmKo7qxKoqm7c3e/dMtLxEjMekfALm24ek2FNq1nv4jGOZHvy+DVrVfWm7r7TNq9zc359u/7UGXZrgecTj506wKVRVRsjgO6cYSfdp2TeF/G+VMPW2hsX8E7OppFiM/TIDFO2/7G77zIuEv27E2c6Gv1FkrfWsMtrJblvhnXFZmccAX2nqXMcog+OU8J6HDn4yAy7YbID08EmtrSpP+Mb3E9l3y4Tb0zyxBkPE7x69q0anySvz9ByXzhdqt1Z2EKZHEFLmjKxWVVdK8NJZrKM3cHemeQWGaat3aKqvinJs7r77hNH28+4nsBpY8G2Vc91of6xZNuwcRHhb+b4/nGgdcM2zHUo/9JU1YO3O97dsypeN9tSZm64MMnZ3X3uZZ1nt6rqatl08XcB7x/PyfCe9/I5LrC82Tji50EZLoreJMnfZ9j598e6+3UTRjugTZ8/zk1yu+7+YlWd391zXqNtkcYLNpt353v7xJEOaJydcu0kz8/+55uzXH6gqq6R5E+TfFeGku3vkzxyjtOf58ZIoOktberP0zOsR/K48faPJHlmhl235uhpSd6VfVePHpShlf++yRIdQFU9sLuftfUEc2OIY3f/0STBjiJbpjN+jRmfFC9pysRmt8m+AnYJu4N9YRx5dfH4gemjGXZunJXuPm38c1EjE7r7t6fOcAjuPXWAwzEO5z+/u2+844Pn4Tabvj8uw1SUczKz0XdbnDJ+bbye3TvJeUl+sqqe391/MFmybVTVaUlOz7BO1FezkAWAu/v+YxF/9/E86G3d/dGJY22ru7uqHpVh1NLtM/w/fmR3f3zSYAf3oao6PsnfJnlVVX0qySKmyi/Uxu/dLKeCbXJckk9k/+mtnX0LW8/K+Dv2gKlzLJESaHrbTf2Z85P5pt19k023X1tV754szc5O7u7v33T7t8erHnN05fHPRa/zMaq24/EAABBDSURBVHNbd8XYbLYnxUuaMrGhlrk72FnjSfGTMzxXPpvkLdNGOrBx6PPDs69oe12SP+/uL08W6iA2rVGy2YVJzsqQezYjgjYtonvPrTtBVdVPJnnSJMF2MA7nf19Vndjd/zp1np10989svj3+/v3VRHF26zpJbt3dn00uGeH20gy/h2cnmVUJlGHJgZvOvJD4GlX1AxmmtL0uw3v246vqUd3915MGO7Bzktygu186dZDd6O7vHb/9rXFU6dWTvGLCSEelqvrNDBfK/ybD8/gvxrJ4dst+jBcRzuvuP546y25V1eO2OXxhkrO6++8u6zxLYjrYhKrqmCT36+7nLWXqT1U9K8OuEv843r5dhl0nth3SPbWqekuSR3X3m8bbd0zy2O6+w7TJtje+AP/skl6AOXKq6q7d/Zqq2nbk2lyH5ybL3R1sQ1WdlGFnsPMmjnJAVfWUJJfPvsV/H5TkKzPePedPk5yQ5DnjoR/KMLK0M/y/ftBU2Q6kqt6c5Ne7+zXj7V9Ocpfuvue0yQ6shi21b5Vh8d/Nw/lPnSzULo3F5ru6+0ZTZzmQqnpvkpttlK1VdcUk7+juG2+e2j8XVfWKJN/X3Z+fOsuhqKp3JLn7xuifqjohyat7pru7js+Lb8kwmuZz2bcW1yLe8zgyqup9SW6xcZFjXFbj3Lm+xtWMd3zeTlWdkeTGGaavJcn3ZxhQ8Y1JPtDdPzdVtrkzEmhC47SDX07yvC27eczOpvURLp/kzVX1r+Pt6yV575TZdvCTSZ4xrg2UJJ9K8qMT5jmo8Sru/ZMogY6wqjo1m0ZQdPdLpsxzAN+Z5DVJ7pN9I5g2/znbEmi0iN3BDrTA8sZ9c11oOclttnwges34wWmuvr27N0//efGmdSnOnyzVwZ2a5CXjVI97ZDjZvO+0kXb0G1MH2K0to8OOybCeyvOmS7Qrz86w0OvGVeb7JPnL8WLeHEdG/0qG87a3Zv8t7X92uki7csyW6V+fyPAcmav/e+oAzNL/yTDFamOk6xWTfHi6ODv6h6r6s3ztGpRzPQ+6eZI7dvdXkkvWNHpjhjWY3jllsLlTAk3v1VX1S5n/gq+LXB8hyWfGBV6vliTd/ZmqmvvUmqW9AC/ONlOVHllV3z7DqUoXjWtEvSv7T2NbwhDO301yTlW9LvPfHWzr1s+bzW7r502+UlUnd/f7k6SqbpB5b416lc3TlKrqxCRXGe/70nSxDqy7Pz4Wxq/OMNXnfj3zIdTd/fqqul6SG3b3q6vqSkkuN3WuA9i8g9XFSf6luz80VZjd6O7/UsO25XccD/1kd581fj/H6fx/nuFiwjszrAm0FK+oqldm/5GDL5swz0FtTCGFJKmqx2c4f7gwyflV9arx9t0zjNKcq42NiU7fdGzO50Ffn+E8YmPDnysn+YbxovqsF5SfmulgE6uqf8o2H+i6e5ZrkyxNVZ3T3bfecuzs7v62qTLtZGk7/izRUqYqbdpN6UYZSqu/y1Co3CfDIpkPnCrbTsapo/9PhtF3/5zkzLnvDrY0VXW3DAvdf2A8dFKSh3T3dq8hk6uq786wls77MzyPr59ht8nXJfnx7v6T6dLtr/Zts74x6u4KGUqKzky3W99QVT+e5LQMJ8InV9UNkzypu+82cbRFq6qrjReStt1gYIYX75Ikc5yitltV9f3ZV7a9sbtfOGUe2K2qOuisg+5++sHuZ3eq6qFJfj371g77zgwXIZ+T5Le6+1HTpZs3JdDEavst15/U3V+YNNjCVdWNk3xrhgUaN78AXC3DGkG2wFyxsQS688ZJ+3hS/7q5lUAbxjU+7rWxZlhVXTXJS7v7Ow/+k9Opqrsk+Y7x6+Qkb0/yhu7+00mDbeNAay5tmOvaS1V1XJJfzLCj0qeTnJnkj+e0wPJW4/opGztXvW/OWZdq3PzgtkneuvHhv6re2d03mzbZ19pUtm22sVj4L3b3B772p6ZRVS/p7ntvc/FuY+2XWV68q6rfzVDEvzj7TwebZWkFR4vxAuMzunuOIwQPqKruleEz1HEbx7r79AP/xLSq6pszrIn4ngyjgj7U3W+YNtX8mQ42ve22XH969m1pzuG5UYYpbMdnGDWx4aIkPz5Jol2qYUvU303yzd19z6q6SZI7dPdTJ452VKiqyjAF4e3jqKu5T1VKkm/K/tNlvjQem63ufu1YXt0myV0yrM/1rUlmVwJl/9eIrea89tIzMrx//Jfx9o8keWaGnUjm6tsyjFg6NsktqirdPdvtwKvqe5O8prsvHG8fn6FA/ttpkx3UF7v7S8NLXVJVx2a+U0j/JMmHkvxlhtfiH85QGp+T5GkZttyehe7emBZ/coZpX9fv7tPHaY3Xmi7Zju4//vkrm47NdjfMAxSDyb6ybbaj8GCzcUrS9arqCt09yynPW1XVk5JcKcN521OS3C8znr5WVQ9L8sgMuzaem+T2GXZ1NXtiB0YCTayq3t37b7m+7TEOTw1bUs92i+ftjGsN/EWSXxvXMzo2w1Sl2V3FXapxofP/mKGgSIapVbOdqlRVv5ahGN4YCv89SZ7b3b83XaqDq6r/lWFu9lsyjHB805ZFPrmUlvb+UVXPzPAB+tzsW7uo57xAbVWd29233HJs1tNrquoPMowMe3CSn8kw2vjd3f1rkwbbRlW9Y+tuTxv/z7e7bw7GhUe/muSu3f3vq+rrk/z9lkXPAVJVz0jy75O8KPuv8/lHk4U6iKo6r7tvvunPqyR5eXd/x9TZtjOez98myT+O7xs3TvK73X3QEd4YCTQH51TV7Xv/LdfP2uFn2L1PjB9Gv6m7b1pVN09yanf/ztTBDuIa3f28qvqVJOnui6tqzou9LtE5Sa7T3S+aOshudPd/HcvBjTfhh3T326fMtAvnZRj1cdMM0zs+XVVvmftU14UNg17a+8cpSW4y94WVt9huN6K5nzs9OslDMywE/BMZFtN9yqSJDuzzVfWDSf56vH2/7NtFZ67Pk9t1962r6u1J0t2fqqorTB3qQKrqwdsdn/MIPDiKvH/8OibJVSfOshsb52ifH6dZfSLzHun4b939b1WVqrpid7+3qm40daglmPuJzFGrlrvl+tI8OcOaQH+eJN19XlX9ZZI5l0Cfq6pvzHgCXFW3z75V79kbt0vygKr6lwxXZjaGmc9yTaDkkt3hFrNDXHf/fHLJ+kU/lmF02zUzbI86S0sbBp2hZNt4/0iSE5O8b+P9ZYbP53dleA58ZOogh+CsqvqjJP9jvP2IDLuEzVZ3f7Wqnp7krRneR9434+LtARmmiD4hQ9Z/TPLAcb3En54y2EF8eVzrY+M9+oTMe9etzSOUjsuwhtg5GaaTAkdQd//21BkO0UvGac9/kH3vdXO9iJAkHxrz/m2SV1XVp5LYqW8XTAebyLh96wHZanJvVNWZ3X2bzcP3txvePydVdeskj88wguJdSU7IsC3xeZMGO4oc6PfP793eqaqfzjBy6dsyLEr6xgy7u7xmylwHs8Bh0It6HxnX4LplhmJt8wK1p04WagdVdeUkv5Hku8ZDr0ryO939uQP/1LTG0Wxbd2H7ie5++aTBjhJV9YAM25XfOsMajvdL8uvd/fxJg+3S+IHpr7r7HlNngaPd+L633S7Qs1yzZizgH57h/G1jw6InLmETh6r6D0munuQVS1mDaUpGAk1kbifnR7GPV9XJ2XfF7n6Z/1Xok5PcM8l1k3x/hlErflf3kN+/y8RxSf4oydndffHUYXZpUcOgF/g8/q2pAxyqseyZ86Lx2/nDJHfp7guSZHwPfGmS2ZRAVfXL3f0HVfX4bP8BabbrRHX3s6vq7AwjairJ93T3eyaOdSg+l6EYBI68X9r0/XEZzuvnfE709Ayb6GzesOgZWcCGRd39+qkzLIkPlhztHpHkjCQ3rqoPJ/mnDMPP5+w3uvv542KTd8mwk9UTM5RBsAjd/dipMxyGpQ2DXpTufv24++HmBdlnvVj40q7iji7aKIBGH8hwUj8n/znD79n7k3xq4iyHrLvfm4VM3a+qF2ffc/hySW6S5HnTJYL16O6t04f/oarmPM38pls2l3htVb17sjQcMUogjkpV9Qubbr4syWszLMr2uQwt/CxX5R9tLAJ9ryRP7u6XVtWc1zCCo8Vjs28Y9MauZk+cNNFRZFwA+L8neV2GERSPr6pHdfdfH/QHp7W0q7jJsI7RyzJ80O8kP5DkzKr6viTp7hdMGW70/42j7R6SYRv4mjbOUe2x2VcCXZzkX7r7wxPmgdWoqm/YdPOYDFPkrz5RnN1Y2oYTHCZrAnFUqqrHjN/eKMNV57/LcJJ5nwxXnx84VbadVNVLknw4yd0zrDnwhQyZZ7dVLhxNqup5GUZMPGs89CNJrt7dsx8GvQRV9Y4kd98Y/TMuqPvqpb22VdXbuvu2U+c4kKr6i4Pc3d39ny6zMAdQVRtb198gw/vdJXdlyHiDSYIdRarqTd19p6q6KEMJtFG09fj1yST/vbufMFVGONpV1T9l3+/fxRlmJJze3W+aNNgWWzYsulGS/TYs2jI6iKOAEoijWlW9Icm9uvui8fZVk7y0u79z2mQHVlVXSnKPJO/s7v9dVddKcrPu/vuJo8FRrarevfVEZ7tjHJ6qemd332zT7WOSvGPzsbnZ5iruKUn+tLttQbsHquqJ3f3wqXOs0bgL6Zs9l4GlbTTBpWc6GEe7b0qyeYX4L43HZqu7P5/kBZtufyTzX8wajgaGQR9ZL6+qVyZ5znj7hzJM152zs7P/VJp/TvLQydLsQlVdP8nPJDkpm87z5rgLmwJoOt39iaq689Q54GhWVZfPMM184+Lz65L8eXd/ebJQ21DyrI8SiKPdM5K8rapeON7+niT/c7o4wNxsGQb95qrabxj0lNmOMp3kz5Pcabx9RpLbTxdnV26SYdrSnbJvu9y5F4N/m+SpSV6c5KsTZ2HGxotMwJHzxAznFhvTLh80HnvYZIkgpoOxAlV16wwLvSbJG7r77VPmAebFMOjLRlWd09233nLsvO6++VSZdjKuE/WZJM8eD/1IkuO7+wemS3VwVfXW7rabJMDEquodW9e92+4YXNaMBOKo193nJDln6hzAPCl5jqyqenjGRYCr6rxNd101yT9Mk2rXlrhd7p+OmyP8fZIvbhwc3wsBuOx8papO7u73J0lV3SD7dgGGySiBAIAj6S+TvDzJ7yV59KbjF3X3J6eJtGtLXCfqZhmmHNw1+6aD9XgbgMvOozJcPPjAePukJA+ZLg4MTAcDANhkydvlVtUFSW7S3V/a8cEAHDFVdVySX0xytySfTnJmkj/u7n+bNBirZyQQAMD+7j11gEvhXUmOT/LRqYMArNwzMqwr91/G2z+S5JlJZruuHOugBAIA2GTh60Qdn+S9VXVm9l8TaHZbxAMc5Za4rhwroAQCADh6PGbqAAAkWea6cqyANYEAAABgDyx5XTnWQQkEALBwVfWm7r5TVV2U4cPGJXcl6e6+2kTRAFalqq53sPsXPuWYo4ASCAAAAGAFjpk6AAAAAABHnhIIAAAAYAWUQAAAAAAroAQCAAAAWAElEAAAAMAK/P9mhk+YV1dXgQAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 1440x360 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "import numpy as np\n",
    "df = flippedPredictions_pos0_rnn[flippedPredictions_pos0_rnn.alteredAttribute==('Beer_Name',)]\n",
    "D = countTokens(df.rtable_Beer_Name.values)\n",
    "fig, ax = plt.subplots(figsize=(20,5))\n",
    "x = list(D.keys())\n",
    "y = list(D.values())\n",
    "ax.bar(x,y,align='center')\n",
    "plt.xticks(np.arange(len(x)),x, rotation='vertical')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Retrain"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-02-09T17:11:56.818441Z",
     "start_time": "2020-02-09T17:11:54.501094Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING:deepmatcher.data.dataset:Rebuilding data cache because: ['One or more data files have been modified.', 'Data file list has changed.']\n",
      "\n",
      "Reading and processing data from \"datasets/Structured/itunes-amazon/improvedTrain1.csv\"\n",
      "0% [############################# ] 100% | ETA: 00:00:00\n",
      "Reading and processing data from \"datasets/Structured/itunes-amazon/valid_withid.csv\"\n",
      "0% [############################# ] 100% | ETA: 00:00:00\n",
      "Reading and processing data from \"datasets/Structured/itunes-amazon/test_withid.csv\"\n",
      "0% [############################# ] 100% | ETA: 00:00:00\n",
      "Building vocabulary\n",
      "0% [#] 100% | ETA: 00:00:00\n",
      "Total time elapsed: 00:00:00\n",
      "\n",
      "Computing principal components\n",
      "0% [#] 100% | ETA: 00:00:00\n",
      "Total time elapsed: 00:00:00\n"
     ]
    }
   ],
   "source": [
    "train,validation,test = dm.data.process(ITUNES_DIR,train='improvedTrain1.csv',validation='valid_withid.csv',\n",
    "                                       test='test_withid.csv',left_prefix='ltable_',right_prefix='rtable_',\n",
    "                                        cache='impr.pth',ignore_columns=['ltable_id','rtable_id'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-02-09T17:18:26.304899Z",
     "start_time": "2020-02-09T17:12:00.034643Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "* Number of trainable parameters: 17757810\n",
      "===>  TRAIN Epoch 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/nvidia/anaconda3/envs/torchenv/lib/python3.6/site-packages/torch/nn/functional.py:1932: UserWarning: reduction: 'mean' divides the total loss by both the batch size and the support size.'batchmean' divides only by the batch size, and aligns with the KL div math definition.'mean' will be changed to behave the same as 'batchmean' in the next major release.\n",
      "  warnings.warn(\"reduction: 'mean' divides the total loss by both the batch size and the support size.\"\n",
      "0% [█████] 100% | ETA: 00:00:00\n",
      "Total time elapsed: 00:00:12\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Finished Epoch 1 || Run Time:   11.0 | Load Time:    2.2 || F1:  44.71 | Prec:  29.97 | Rec:  87.96 || Ex/s:  30.40\n",
      "\n",
      "===>  EVAL Epoch 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "0% [█] 100% | ETA: 00:00:00\n",
      "Total time elapsed: 00:00:01\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Finished Epoch 1 || Run Time:    1.0 | Load Time:    0.5 || F1:  54.74 | Prec:  38.24 | Rec:  96.30 || Ex/s:  73.57\n",
      "\n",
      "* Best F1: tensor(54.7368, device='cuda:0')\n",
      "Saving best model...\n",
      "Done.\n",
      "---------------------\n",
      "\n",
      "===>  TRAIN Epoch 2\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "0% [█████] 100% | ETA: 00:00:00\n",
      "Total time elapsed: 00:00:13\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Finished Epoch 2 || Run Time:   11.7 | Load Time:    2.4 || F1:  54.79 | Prec:  38.91 | Rec:  92.59 || Ex/s:  28.56\n",
      "\n",
      "===>  EVAL Epoch 2\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "0% [█] 100% | ETA: 00:00:00\n",
      "Total time elapsed: 00:00:01\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Finished Epoch 2 || Run Time:    1.0 | Load Time:    0.5 || F1:  65.00 | Prec:  49.06 | Rec:  96.30 || Ex/s:  72.26\n",
      "\n",
      "* Best F1: tensor(65., device='cuda:0')\n",
      "Saving best model...\n",
      "Done.\n",
      "---------------------\n",
      "\n",
      "===>  TRAIN Epoch 3\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "0% [█████] 100% | ETA: 00:00:00\n",
      "Total time elapsed: 00:00:12\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Finished Epoch 3 || Run Time:   11.3 | Load Time:    2.3 || F1:  66.46 | Prec:  50.24 | Rec:  98.15 || Ex/s:  29.40\n",
      "\n",
      "===>  EVAL Epoch 3\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "0% [█] 100% | ETA: 00:00:00\n",
      "Total time elapsed: 00:00:01\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Finished Epoch 3 || Run Time:    1.0 | Load Time:    0.5 || F1:  72.22 | Prec:  57.78 | Rec:  96.30 || Ex/s:  71.88\n",
      "\n",
      "* Best F1: tensor(72.2222, device='cuda:0')\n",
      "Saving best model...\n",
      "Done.\n",
      "---------------------\n",
      "\n",
      "===>  TRAIN Epoch 4\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "0% [█████] 100% | ETA: 00:00:00\n",
      "Total time elapsed: 00:00:13\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Finished Epoch 4 || Run Time:   11.5 | Load Time:    2.4 || F1:  81.82 | Prec:  69.23 | Rec: 100.00 || Ex/s:  28.85\n",
      "\n",
      "===>  EVAL Epoch 4\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "0% [█] 100% | ETA: 00:00:00\n",
      "Total time elapsed: 00:00:01\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Finished Epoch 4 || Run Time:    1.0 | Load Time:    0.5 || F1:  70.27 | Prec:  55.32 | Rec:  96.30 || Ex/s:  72.68\n",
      "\n",
      "---------------------\n",
      "\n",
      "===>  TRAIN Epoch 5\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "0% [█████] 100% | ETA: 00:00:00\n",
      "Total time elapsed: 00:00:12\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Finished Epoch 5 || Run Time:   11.2 | Load Time:    2.3 || F1:  95.15 | Prec:  90.76 | Rec: 100.00 || Ex/s:  29.67\n",
      "\n",
      "===>  EVAL Epoch 5\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "0% [█] 100% | ETA: 00:00:00\n",
      "Total time elapsed: 00:00:01\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Finished Epoch 5 || Run Time:    1.0 | Load Time:    0.5 || F1:  76.19 | Prec:  66.67 | Rec:  88.89 || Ex/s:  72.76\n",
      "\n",
      "* Best F1: tensor(76.1905, device='cuda:0')\n",
      "Saving best model...\n",
      "Done.\n",
      "---------------------\n",
      "\n",
      "===>  TRAIN Epoch 6\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "0% [█████] 100% | ETA: 00:00:00\n",
      "Total time elapsed: 00:00:12\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Finished Epoch 6 || Run Time:   11.3 | Load Time:    2.3 || F1:  95.58 | Prec:  91.53 | Rec: 100.00 || Ex/s:  29.45\n",
      "\n",
      "===>  EVAL Epoch 6\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "0% [█] 100% | ETA: 00:00:00\n",
      "Total time elapsed: 00:00:01\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Finished Epoch 6 || Run Time:    1.0 | Load Time:    0.5 || F1:  72.00 | Prec:  56.25 | Rec: 100.00 || Ex/s:  71.66\n",
      "\n",
      "---------------------\n",
      "\n",
      "===>  TRAIN Epoch 7\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "0% [█████] 100% | ETA: 00:00:00\n",
      "Total time elapsed: 00:00:12\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Finished Epoch 7 || Run Time:   11.0 | Load Time:    2.2 || F1:  99.08 | Prec:  98.18 | Rec: 100.00 || Ex/s:  30.41\n",
      "\n",
      "===>  EVAL Epoch 7\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "0% [█] 100% | ETA: 00:00:00\n",
      "Total time elapsed: 00:00:01\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Finished Epoch 7 || Run Time:    1.0 | Load Time:    0.5 || F1:  84.75 | Prec:  78.12 | Rec:  92.59 || Ex/s:  72.26\n",
      "\n",
      "* Best F1: tensor(84.7458, device='cuda:0')\n",
      "Saving best model...\n",
      "Done.\n",
      "---------------------\n",
      "\n",
      "===>  TRAIN Epoch 8\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "0% [█████] 100% | ETA: 00:00:00\n",
      "Total time elapsed: 00:00:13\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Finished Epoch 8 || Run Time:   11.5 | Load Time:    2.4 || F1: 100.00 | Prec: 100.00 | Rec: 100.00 || Ex/s:  28.86\n",
      "\n",
      "===>  EVAL Epoch 8\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "0% [█] 100% | ETA: 00:00:00\n",
      "Total time elapsed: 00:00:01\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Finished Epoch 8 || Run Time:    1.0 | Load Time:    0.5 || F1:  85.25 | Prec:  76.47 | Rec:  96.30 || Ex/s:  72.01\n",
      "\n",
      "* Best F1: tensor(85.2459, device='cuda:0')\n",
      "Saving best model...\n",
      "Done.\n",
      "---------------------\n",
      "\n",
      "===>  TRAIN Epoch 9\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "0% [█████] 100% | ETA: 00:00:00\n",
      "Total time elapsed: 00:00:12\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Finished Epoch 9 || Run Time:   11.3 | Load Time:    2.3 || F1: 100.00 | Prec: 100.00 | Rec: 100.00 || Ex/s:  29.50\n",
      "\n",
      "===>  EVAL Epoch 9\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "0% [█] 100% | ETA: 00:00:00\n",
      "Total time elapsed: 00:00:01\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Finished Epoch 9 || Run Time:    1.0 | Load Time:    0.5 || F1:  80.00 | Prec:  68.42 | Rec:  96.30 || Ex/s:  74.11\n",
      "\n",
      "---------------------\n",
      "\n",
      "===>  TRAIN Epoch 10\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "0% [█████] 100% | ETA: 00:00:00\n",
      "Total time elapsed: 00:00:12\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Finished Epoch 10 || Run Time:   11.3 | Load Time:    2.3 || F1: 100.00 | Prec: 100.00 | Rec: 100.00 || Ex/s:  29.56\n",
      "\n",
      "===>  EVAL Epoch 10\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "0% [█] 100% | ETA: 00:00:00\n",
      "Total time elapsed: 00:00:01\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Finished Epoch 10 || Run Time:    1.0 | Load Time:    0.5 || F1:  81.25 | Prec:  70.27 | Rec:  96.30 || Ex/s:  73.15\n",
      "\n",
      "---------------------\n",
      "\n",
      "===>  TRAIN Epoch 11\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "0% [█████] 100% | ETA: 00:00:00\n",
      "Total time elapsed: 00:00:12\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Finished Epoch 11 || Run Time:   10.8 | Load Time:    2.1 || F1: 100.00 | Prec: 100.00 | Rec: 100.00 || Ex/s:  30.87\n",
      "\n",
      "===>  EVAL Epoch 11\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "0% [█] 100% | ETA: 00:00:00\n",
      "Total time elapsed: 00:00:01\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Finished Epoch 11 || Run Time:    1.0 | Load Time:    0.5 || F1:  81.25 | Prec:  70.27 | Rec:  96.30 || Ex/s:  74.13\n",
      "\n",
      "---------------------\n",
      "\n",
      "===>  TRAIN Epoch 12\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "0% [█████] 100% | ETA: 00:00:00\n",
      "Total time elapsed: 00:00:12\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Finished Epoch 12 || Run Time:   11.3 | Load Time:    2.3 || F1: 100.00 | Prec: 100.00 | Rec: 100.00 || Ex/s:  29.56\n",
      "\n",
      "===>  EVAL Epoch 12\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "0% [█] 100% | ETA: 00:00:00\n",
      "Total time elapsed: 00:00:01\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Finished Epoch 12 || Run Time:    1.0 | Load Time:    0.5 || F1:  81.25 | Prec:  70.27 | Rec:  96.30 || Ex/s:  72.86\n",
      "\n",
      "---------------------\n",
      "\n",
      "===>  TRAIN Epoch 13\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "0% [█████] 100% | ETA: 00:00:00\n",
      "Total time elapsed: 00:00:12\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Finished Epoch 13 || Run Time:   11.3 | Load Time:    2.3 || F1: 100.00 | Prec: 100.00 | Rec: 100.00 || Ex/s:  29.48\n",
      "\n",
      "===>  EVAL Epoch 13\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "0% [█] 100% | ETA: 00:00:00\n",
      "Total time elapsed: 00:00:01\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Finished Epoch 13 || Run Time:    1.0 | Load Time:    0.5 || F1:  81.25 | Prec:  70.27 | Rec:  96.30 || Ex/s:  71.34\n",
      "\n",
      "---------------------\n",
      "\n",
      "===>  TRAIN Epoch 14\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "0% [█████] 100% | ETA: 00:00:00\n",
      "Total time elapsed: 00:00:13\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Finished Epoch 14 || Run Time:   11.6 | Load Time:    2.4 || F1: 100.00 | Prec: 100.00 | Rec: 100.00 || Ex/s:  28.70\n",
      "\n",
      "===>  EVAL Epoch 14\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "0% [█] 100% | ETA: 00:00:00\n",
      "Total time elapsed: 00:00:01\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Finished Epoch 14 || Run Time:    1.0 | Load Time:    0.5 || F1:  81.25 | Prec:  70.27 | Rec:  96.30 || Ex/s:  70.41\n",
      "\n",
      "---------------------\n",
      "\n",
      "===>  TRAIN Epoch 15\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "0% [█████] 100% | ETA: 00:00:00\n",
      "Total time elapsed: 00:00:13\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Finished Epoch 15 || Run Time:   11.6 | Load Time:    2.4 || F1: 100.00 | Prec: 100.00 | Rec: 100.00 || Ex/s:  28.82\n",
      "\n",
      "===>  EVAL Epoch 15\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "0% [█] 100% | ETA: 00:00:00\n",
      "Total time elapsed: 00:00:01\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Finished Epoch 15 || Run Time:    1.0 | Load Time:    0.5 || F1:  81.25 | Prec:  70.27 | Rec:  96.30 || Ex/s:  70.96\n",
      "\n",
      "---------------------\n",
      "\n",
      "===>  TRAIN Epoch 16\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "0% [█████] 100% | ETA: 00:00:00\n",
      "Total time elapsed: 00:00:13\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Finished Epoch 16 || Run Time:   11.6 | Load Time:    2.3 || F1: 100.00 | Prec: 100.00 | Rec: 100.00 || Ex/s:  28.71\n",
      "\n",
      "===>  EVAL Epoch 16\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "0% [█] 100% | ETA: 00:00:00\n",
      "Total time elapsed: 00:00:01\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Finished Epoch 16 || Run Time:    1.0 | Load Time:    0.5 || F1:  81.25 | Prec:  70.27 | Rec:  96.30 || Ex/s:  71.28\n",
      "\n",
      "---------------------\n",
      "\n",
      "===>  TRAIN Epoch 17\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "0% [█████] 100% | ETA: 00:00:00\n",
      "Total time elapsed: 00:00:13\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Finished Epoch 17 || Run Time:   11.6 | Load Time:    2.4 || F1: 100.00 | Prec: 100.00 | Rec: 100.00 || Ex/s:  28.83\n",
      "\n",
      "===>  EVAL Epoch 17\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "0% [█] 100% | ETA: 00:00:00\n",
      "Total time elapsed: 00:00:01\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Finished Epoch 17 || Run Time:    1.0 | Load Time:    0.5 || F1:  81.25 | Prec:  70.27 | Rec:  96.30 || Ex/s:  71.14\n",
      "\n",
      "---------------------\n",
      "\n",
      "===>  TRAIN Epoch 18\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "0% [█████] 100% | ETA: 00:00:00\n",
      "Total time elapsed: 00:00:13\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Finished Epoch 18 || Run Time:   11.6 | Load Time:    2.4 || F1: 100.00 | Prec: 100.00 | Rec: 100.00 || Ex/s:  28.65\n",
      "\n",
      "===>  EVAL Epoch 18\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "0% [█] 100% | ETA: 00:00:00\n",
      "Total time elapsed: 00:00:01\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Finished Epoch 18 || Run Time:    1.0 | Load Time:    0.5 || F1:  81.25 | Prec:  70.27 | Rec:  96.30 || Ex/s:  71.48\n",
      "\n",
      "---------------------\n",
      "\n",
      "===>  TRAIN Epoch 19\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "0% [█████] 100% | ETA: 00:00:00\n",
      "Total time elapsed: 00:00:13\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Finished Epoch 19 || Run Time:   11.5 | Load Time:    2.3 || F1: 100.00 | Prec: 100.00 | Rec: 100.00 || Ex/s:  28.98\n",
      "\n",
      "===>  EVAL Epoch 19\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "0% [█] 100% | ETA: 00:00:00\n",
      "Total time elapsed: 00:00:01\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Finished Epoch 19 || Run Time:    1.0 | Load Time:    0.5 || F1:  81.25 | Prec:  70.27 | Rec:  96.30 || Ex/s:  70.35\n",
      "\n",
      "---------------------\n",
      "\n",
      "===>  TRAIN Epoch 20\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "0% [█████] 100% | ETA: 00:00:00\n",
      "Total time elapsed: 00:00:12\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Finished Epoch 20 || Run Time:   11.3 | Load Time:    2.3 || F1: 100.00 | Prec: 100.00 | Rec: 100.00 || Ex/s:  29.47\n",
      "\n",
      "===>  EVAL Epoch 20\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "0% [█] 100% | ETA: 00:00:00\n",
      "Total time elapsed: 00:00:01\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Finished Epoch 20 || Run Time:    1.0 | Load Time:    0.5 || F1:  81.25 | Prec:  70.27 | Rec:  96.30 || Ex/s:  71.38\n",
      "\n",
      "---------------------\n",
      "\n",
      "===>  TRAIN Epoch 21\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "0% [█████] 100% | ETA: 00:00:00\n",
      "Total time elapsed: 00:00:12\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Finished Epoch 21 || Run Time:   11.3 | Load Time:    2.3 || F1: 100.00 | Prec: 100.00 | Rec: 100.00 || Ex/s:  29.43\n",
      "\n",
      "===>  EVAL Epoch 21\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "0% [█] 100% | ETA: 00:00:00\n",
      "Total time elapsed: 00:00:01\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Finished Epoch 21 || Run Time:    1.0 | Load Time:    0.5 || F1:  81.25 | Prec:  70.27 | Rec:  96.30 || Ex/s:  70.69\n",
      "\n",
      "---------------------\n",
      "\n",
      "===>  TRAIN Epoch 22\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "0% [█████] 100% | ETA: 00:00:00\n",
      "Total time elapsed: 00:00:13\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Finished Epoch 22 || Run Time:   11.6 | Load Time:    2.4 || F1: 100.00 | Prec: 100.00 | Rec: 100.00 || Ex/s:  28.74\n",
      "\n",
      "===>  EVAL Epoch 22\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "0% [█] 100% | ETA: 00:00:00\n",
      "Total time elapsed: 00:00:01\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Finished Epoch 22 || Run Time:    1.0 | Load Time:    0.5 || F1:  81.25 | Prec:  70.27 | Rec:  96.30 || Ex/s:  71.17\n",
      "\n",
      "---------------------\n",
      "\n",
      "===>  TRAIN Epoch 23\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "0% [█████] 100% | ETA: 00:00:00\n",
      "Total time elapsed: 00:00:13\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Finished Epoch 23 || Run Time:   11.6 | Load Time:    2.3 || F1: 100.00 | Prec: 100.00 | Rec: 100.00 || Ex/s:  28.74\n",
      "\n",
      "===>  EVAL Epoch 23\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "0% [█] 100% | ETA: 00:00:00\n",
      "Total time elapsed: 00:00:01\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Finished Epoch 23 || Run Time:    1.0 | Load Time:    0.5 || F1:  81.25 | Prec:  70.27 | Rec:  96.30 || Ex/s:  71.59\n",
      "\n",
      "---------------------\n",
      "\n",
      "===>  TRAIN Epoch 24\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "0% [█████] 100% | ETA: 00:00:00\n",
      "Total time elapsed: 00:00:13\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Finished Epoch 24 || Run Time:   11.7 | Load Time:    2.4 || F1: 100.00 | Prec: 100.00 | Rec: 100.00 || Ex/s:  28.62\n",
      "\n",
      "===>  EVAL Epoch 24\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "0% [█] 100% | ETA: 00:00:00\n",
      "Total time elapsed: 00:00:01\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Finished Epoch 24 || Run Time:    1.0 | Load Time:    0.5 || F1:  81.25 | Prec:  70.27 | Rec:  96.30 || Ex/s:  71.42\n",
      "\n",
      "---------------------\n",
      "\n",
      "===>  TRAIN Epoch 25\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "0% [█████] 100% | ETA: 00:00:00\n",
      "Total time elapsed: 00:00:12\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Finished Epoch 25 || Run Time:   11.4 | Load Time:    2.3 || F1: 100.00 | Prec: 100.00 | Rec: 100.00 || Ex/s:  29.13\n",
      "\n",
      "===>  EVAL Epoch 25\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "0% [█] 100% | ETA: 00:00:00\n",
      "Total time elapsed: 00:00:01\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Finished Epoch 25 || Run Time:    1.0 | Load Time:    0.5 || F1:  81.25 | Prec:  70.27 | Rec:  96.30 || Ex/s:  70.78\n",
      "\n",
      "---------------------\n",
      "\n",
      "Loading best model...\n",
      "Training done.\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "tensor(85.2459, device='cuda:0')"
      ]
     },
     "execution_count": 75,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "hybrid_model = dm.MatchingModel(attr_summarizer='hybrid')\n",
    "hybrid_model.run_train(train,validation,pos_neg_ratio=5,\n",
    "                      batch_size=16,epochs=25,best_save_path='itunesamazon_impr.pth')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-02-09T17:19:30.136901Z",
     "start_time": "2020-02-09T17:19:28.886499Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "===>  EVAL Epoch 8\n",
      "Finished Epoch 8 || Run Time:    0.6 | Load Time:    0.6 || F1:  86.21 | Prec:  80.65 | Rec:  92.59 || Ex/s:  91.97\n",
      "\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "tensor(86.2069, device='cuda:0')"
      ]
     },
     "execution_count": 76,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "hybrid_model.run_eval(test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-02-09T17:19:34.853185Z",
     "start_time": "2020-02-09T17:19:33.248785Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(0.7903955578804016, 0.011770060641655073)"
      ]
     },
     "execution_count": 77,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "standard_test = pd.read_csv(ITUNES_DIR+'merged_test.csv')\n",
    "getMeanConfidenceAndVariance(hybrid_model,standard_test)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Evaluate first retrain"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-02-25T13:16:22.505092Z",
     "start_time": "2020-02-25T13:16:21.969408Z"
    }
   },
   "outputs": [],
   "source": [
    "retrained1 = dm.MatchingModel(attr_summarizer='hybrid')\n",
    "retrained1.load_state('models/beer_improved_hybrid.pth')\n",
    "retrained1 = retrained1.to('cuda')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-02-25T09:27:33.534333Z",
     "start_time": "2020-02-25T09:27:33.489613Z"
    }
   },
   "outputs": [],
   "source": [
    "augmented_test = pd.read_csv(DATASET_DIR+'augmented_test.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-02-25T09:27:34.472083Z",
     "start_time": "2020-02-25T09:27:33.537010Z"
    }
   },
   "outputs": [],
   "source": [
    "predictions1 = wrapDm(augmented_test,retrained1,ignore_columns=['ltable_id','rtable_id','id','label'])\n",
    "tp_group = augmented_test[(predictions1[:,1]>=0.5)& (augmented_test['label'] == 1)]\n",
    "tn_group = augmented_test[(predictions1[:,0] >=0.5)& (augmented_test['label']==0)]\n",
    "correctPredictions1 = pd.concat([tp_group,tn_group])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-02-25T09:28:30.780006Z",
     "start_time": "2020-02-25T09:27:34.473465Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 46/46 [00:56<00:00,  1.22s/it]\n"
     ]
    }
   ],
   "source": [
    "neg_rankings1,neg_triangles1,flipped_neg1 = explainSamples(\n",
    "    correctPredictions1,[tableA,tableB],\n",
    "    retrained1,originalClass=0,maxLenAttributeSet=4)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-02-25T09:28:30.944720Z",
     "start_time": "2020-02-25T09:28:30.781837Z"
    },
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAATAAAADCCAYAAADO1p5VAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8li6FKAAASqUlEQVR4nO3df5RU5X3H8fdHpFkUJOFXGgFd4oGI8SA/jSSEQGo0JC0Ga6LUGE9si6TGmJzYKolW2sZDEqmxnsSDP0qjsVFriCCeBghGglQNLGZFFClKjse1RmSDsqBEkG//mGdgWPfH3WVnZ+/yeZ0zZ+997p253zvLfnjuMzPPKCIwM8ujoypdgJlZeznAzCy3HGBmllsOMDPLLQeYmeWWA8zMcuvoShdgXcOAAQOiurq60mWYvcv69eu3R8TAprY5wAyA6upqampqKl2G2btIerG5bb6ENLPccoCZWW45wMwstzwGZlYBe/fupa6ujj179lS6lC6jqqqKIUOG0LNnz8z3cYCZVUBdXR19+vShuroaSZUup+Iigvr6eurq6hg2bFjm+/kS0qwC9uzZQ//+/R1eiST69+/f5h6pA8ysQhxeh2rP8+EAMztC3XzzzYwcOZLBgwfz1a9+FYAFCxZw1113lfW4c+fOZf78+R3yWB4DM+sKOro3lmGi0ltuuYWVK1eycuXKA29inj17dsfWUWbugZkdgWbPns3WrVuZNm0aO3bsONBe2juaMmUKV1xxBaNHj+bUU09l7dq1B/a56KKLmDhxIsOHD+f2228/cP8bbriBCRMmMGrUKK677roD7ddffz0jRoxg0qRJbN68ucPOwz0wsyPQggULWLZsGY888ggPPfRQs/u9+eab1NbWsnr1ai655BI2btwIwIYNG3jiiSfYvXs3Y8aM4bOf/SwbN25ky5YtrF27lohg+vTprF69mmOPPZZ7772X2tpa9u3bx9ixYxk3blyHnIcDzArWr2/6MsbfmXBEmzlzJgCTJ09m586dvP766wCcc8459OrVi169ejF16lTWrl3LmjVrWLFiBWPGjAFg165dbNmyhYaGBmbMmMExxxwDwPTp0zusPgeYmTWr8SuDxfWm2iOCOXPmcOmllx6y7aabbipbfR4DM7Nm3XfffQCsWbOGvn370rdvXwCWLFnCnj17qK+vZ9WqVUyYMIGzzz6bhQsXsmvXLgBefvlltm3bxuTJk1m8eDFvvfUWDQ0NLF26tMPqcw/MzJpVVVXFmDFj2Lt3LwsXLjzQPmrUKKZOncr27du59tprOf744zn++OPZtGkTEydOBKB3797cfffdjB07lvPPP5/TTjuNQYMGMWHChA6rT/5eSAMYL0WTs4H530dZbNq0iZEjR1a6jBZNmTKF+fPnM378+EPa586dS+/evbnyyis7/JhNPS+S1kfE+Kb29yWkmeWWLyHNrEmrVq1qsn3u3LmdWkdL3AMzs9xygJlViMefD9We58MBZlYBVVVV1NfXO8SS4nxgVVVVbbqfx8ByQNLngAeAkRHxnKRqYBOwGRCwG/gy8FK6DYuInSX3XwzcExH3dXLp1owhQ4ZQV1fHa6+9VulSuozijKxt4QDLh5nAmvSz+AnZFyJiNICkS4FvRcTFkpYDM4A707a+wCTgrzq9amtWz5492zTzqDXNl5BdnKTeFALor4ELmtntOKA4pcA9jfabASyPiDfLVqRZhbgH1vWdAyyLiP+VVC9pHFAPnCSpFugDHAN8JO2/HLhDUv+IqKcQZj+sROFm5eYeWNc3E7g3Ld+b1iFdQkbEScDXgdsAIuJt4EHgPEkDgDEUQu1dJM2SVCOpxiMxlkf+KFEXJqkfUAe8BgTQI/38BLA0Ik5N+/UC6iPimLT+SeBa4D5gbETMau1Y/iiRdVX+KFF+nQf8JCJOjIjqiBgK/A4Y2mi/ScALJeurgOHAZRTGxMy6JY+BdW0zge81alsEzOHgGJiAt4G/Ke4QEfsl/Qz4AvDrTqrVrNP5EtIAX0Ja1+VLSDPrlhxgZpZbDjAzyy0HmJnllgPMzHLLAWZmueUAM7PccoCZWW45wKxg3LjCm1Yb38y6MAeYmeWWA8zMcssBZma55QAzs9xygJlZbjnAzCy3HGBmllsOMDPLLQeYmeWWA8zMcssBZma55QAzs9xygJlZbjnAzCy3HGBmllsOMDPLLQeYmeWWA8zMcqvNASbpfZJGlaMYM7O2yBRgklZJOk5SP+BJ4HZJN5a3NDOzlmXtgfWNiJ3AucBdEfER4MzylWVm1rqsAXa0pA8AXwAeKmM9ZmaZZQ2wfwaWAy9ExDpJHwS2lK8sM7PWHZ1lp4i4H7i/ZH0r8JflKsrMLIusg/gjJD0saWNaHyXpmvKWZmbWsqyXkLcDc4C9ABGxAbigXEWZmWWRNcCOiYi1jdr2dXQxZmZtkWkMDNgu6SQgACSdB7xStqqs861fD1Klq+geIipdwREja4BdBtwGnCzpZeB3wIVlq8rMLIOsARYRcaakY4GjIqJB0rByFmZm1pqsY2CLACJid0Q0pLaflackM7NsWuyBSToZ+DDQV9K5JZuOA6rKWZiZWWtau4T8EPDnwHuBvyhpbwD+tlxFmZll0WKARcQSYImkiRHxeCfVZGaWSdZB/FmS3tXjiohLOrgeM7PMsgZY6QwUVcAM4P86vhwzs+yyfph7Uem6pHuANWWpyMwso/bOiT8cGNSRhZiZtVXW2SgaJO0s/gSWAleVt7TOJekdSbWSnpL0pKSPlvl4qyTVlKyPl7SqnMc0626yXkL2KXchXcBbETEaQNLZwDzgE4f7oJJ6RMQ7zWweJGlaRPzicI9jdiTKfAkp6VxJN0r6V0mfK2dRXcBxwI7iiqS/l7RO0gZJ/1TS/kVJa1PP7VZJPVL7rvQ8PQVMbOE4NwDfbtwoqVrSo6kneKA3KGmKpF9LWiJpq6TvSrow1fB0+sA9kgZKWpRqXifpY00dXNIsSTWSal5rz7NkVmFZLyFvAWYDTwMbgdmSflTOwiqgVwqi54A7gH8BkHQWhTG/04HRwDhJkyWNBM4HPpZ6bu9w8APuxwK/iYjTIqKlFzseB96WNLVR+zbgUxExNh3j5pJtp1H4XYwELgJGRMTpqebL0z7/BvwgIiZQmDn3jqYOHhG3RcT4iBg/sMWnxqxryvo2ik8CIyOiOJ3OncAzZauqMkovIScCd0k6FTgr3X6b9utNIdBGAeOAdSpMQ9OLQvBAIcwOeeW2Bd8BruHQMcWewA8lFYNxRMm2dRHxSqrzBWBFan8aKAbhmcApOjg9znGSekfErow1meVC1gB7HjgBeDGtD01t3VJEPC5pADAQEDAvIm4t3UfS5cCdETGniYfY08K4V+Nj/UrSd4AzSpq/AbxKobd1FLCnZNsfS5b3l6zv5+Dv8yjgjIgovZ9Zt9PiJaSkpZIeBPoAm9IrZ48Am1Jbt5Q+xN4DqKfwbUyXSOqdtg2WNAh4GDgvLSOpn6QT23nI7wD/ULLeF3glIvZTuEzs0cbHW8HBy0lST86s22mtBza/U6roGnpJqk3LAi5OvagVabzr8XRJtgv4YkQ8m77YZIWkoyh8X8BlHOylZhYR/y2pdBz9FmCRpC8By4DdbXzIrwE/krSBwu94NYVxM7NuReHpbw0YL0VN67tZFv6b6lCS1kfE+Ka2tTYf2JqImCSpgTQffnEThVlaj+vAOs3M2qS16XQmpZ/ddryr3CQ9ADSefvuqiFheiXrMupNWX4VMb858JiJO7oR6up2ImFHpGsy6q1bfyJoGsjdLOqET6jEzyyzr+8DeBzwjaS0lr4hFxPSyVGVmlkHWALu2rFWYmbVD1gD7TEQcMn2OpO8Bv+74kszMssk6G8Wnmmib1pGFWIWNG1d4/5Jvh3+zTtPa+8C+AvwdcFJ6V3dRH+CxchZmZtaa1i4hfwr8gsLkfleXtDdExB/KVpWZWQatvZH1DeANSfsi4pDP+En6SURcVNbqzMxakHUM7MOlK5KOpjAXlplZxbQ2nc6c9DnIUelLPXamL/V4FVjSKRWamTWjtUvIecA8SfOA71OYGbSquLnMtZmZtSjr+8C2UphTaghQS2H20McpTDVtZlYRWcfAvgZMAF6MiKnAGOD1slVlZpZB1gDbU5xfXdJ7IuI54EPlK8vMrHVZLyHrJL0XWAz8UtIO2jF1splZR8r6zdzFOa3mpi/16EthrnYzs4rJ2gM7ICL8AW4z6xKyjoGZmXU5DjAzyy0HmJnllgPMzHLLAWZmueUAM7PccoCZWW45wMwstxxgZpZbDjAzyy0HmJnllgPMzHLLAWZmueUAM7PcavN0OtZNrV8PUqWrsCNVtO87gtwDM7PccoCZWW45wMwstxxgZpZbDjAzyy0HmJnllgPMzHLLAWZmueUAM7PccoCZWW6VLcAkvSOpVtJTkp6U9NFyHavRcVdJ2pyOXSvpvHY8xpRy1ytprqQ3JQ0qadtVzmOadTfl/CzkWxExGkDS2cA84BOlO0g6OiL2leHYF0ZEzWHcfwqwC3gs6x3aeS7bgW8CV7XxfmZG511CHgfsgAO9m0clPQg8m9q+KGlt6jHdKqmHpM9LujFtv0LS1rT8QUn/09YCJC2WtF7SM5JmlbR/OvUQn5L0sKRqYDbwjVTPxyVVS/qVpA1pnxPSfX8saYGk3wDfl7RF0sC07ShJzxfXm7EQOF9SvzbUu0vSDal9paTTU69zq6TpaZ8eaZ91qeZL2/p8meVCRJTlBrwD1ALPAW8A41L7FGA3MCytjwSWAj3T+i3Al4A/Bdaltp8B64DBwMXAvBaOuwrYnI5dC/RP7f3Sz17ARqA/MBB4qaSW4j5zgStLHnMpcHFavgRYnJZ/DDwE9Ejr1wFfT8tnAYtaqHMucCXwj8A/pbZdJdvfVW9aD2BaWn4AWAH0BE4DalP7LOCatPweoKZ4jo1qmJW21ZxQmA/AN98qc2sBUNPc31FnXUJOBO6SdGratjYifpeW/wwYB6xTYTqXXsC2iPi9pN6S+gBDgZ8Ck4GPAz9v5dhNXUJ+TdKMtDwUGE4hwFYXa4mIPzTzeBOBc9PyT4Dvl2y7PyLeScsLgSXATRSC7j9aqRPgZqBW0vwM9dYDbwPLUvvTwB8jYq+kp4Hq1H4WMKpk/K9vun/xOQcgIm4DbgMYL0WGWs26lE6ZDywiHpc0gEJgQKEHViTgzoiY08RdHwO+TKFH9SiFUJhIYdwoM0lTgDOBiRHxpqRVQFVbHqMFB84lIl6S9KqkTwKnAxe2dueIeF3ST4HLMta7N/2vBLAf+GN6nP2Sir9PAZdHxPLDOjOzLq5TxsAknQz0oNCDaOxh4Lziq3GS+kk6MW17lMJl1mrgt8BUCj2ON9pYQl9gRwqDk4EzUvsTwGRJw4rHTu0NQJ+S+z8GXJCWL0x1NecO4G4O7Zm15kbgUg7+h9JcvVktB74iqSeApBGSjm3jY5h1eeUMsF7FtzIA91EYQ3rXH3REPAtcA6yQtAH4JfCBtPlRCpdPq9N9XwLWtKOWZcDRkjYB36UQXETEaxTGgX4u6alUJxTGvGYUB/GBy4Evp/ouAq5o4VgPAr3JdvlIqmM7hfGs97RUbxvcQeEFkiclbQRuxbPvWjekg1cj1hEkjQd+EBEfr3QtbTFeOqz3nZgdlhZySNL6iBjf1Db/r9yBJF0NfIUMY19mdvhy2wOT9AAwrFHzVV1t4FrSt4HPN2q+PyKur0Q9zXEPzCqqnT2w3AaYdSwHmFVUOwPMH+Y2s9xygJlZbjnAzCy3HGBmllt+G4UVjBsHNR7Gt3xxD8zMcssBZma55QAzs9xygJlZbjnAzCy3HGBmllsOMDPLLQeYmeWWA8zMcssBZma55QAzs9xygJlZbjnAzCy3PKW0ASCpgcIXCB+pBgDbK11EBXXl8z8xIgY2tcHT6VjR5ubmHT8SSKrx+efv/H0JaWa55QAzs9xygFnRbZUuoMJ8/jnkQXwzyy33wMwstxxghqRPS9os6XlJV1e6nnKTtFDSNkkbS9r6SfqlpC3p5/sqWWO5SBoq6RFJz0p6RtIVqT2X5+8AO8JJ6gH8CJgGnALMlHRKZasqux8Dn27UdjXwcEQMBx5O693RPuCbEXEKcAZwWfp95/L8HWB2OvB8RGyNiLeBe4FzKlxTWUXEauAPjZrPAe5My3cCn+vUojpJRLwSEU+m5QZgEzCYnJ6/A8wGAy+VrNeltiPN+yPilbT8e+D9lSymM0iqBsYAvyGn5+8AM2skCi/Nd+uX5yX1BhYBX4+InaXb8nT+DjB7GRhasj4ktR1pXpX0AYD0c1uF6ykbST0phNd/RsTPU3Muz98BZuuA4ZKGSfoT4ALgwQrXVAkPAhen5YuBJRWspWwkCfh3YFNE3FiyKZfn7zeyGpI+A9wE9AAWRsT1FS6prCTdA0yhMAPDq8B1wGLgv4ATgBeBL0RE44H+3JM0CXgUeBrYn5q/RWEcLHfn7wAzs9zyJaSZ5ZYDzMxyywFmZrnlADOz3HKAmVluOcDMLLccYGaWWw4wM8ut/wcI6S7/qkXsqwAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 216x216 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "ax,negRank1  = aggregateRankings(neg_rankings1,positive=False,maxLenAttributes=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-02-25T09:29:27.989615Z",
     "start_time": "2020-02-25T09:28:30.946416Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 46/46 [00:56<00:00,  1.24s/it]\n"
     ]
    }
   ],
   "source": [
    "pos_rankings1,pos_triangles1,flipped_pos1 = explainSamples(\n",
    "    correctPredictions1,[tableA,tableB]\n",
    "    ,retrained1,originalClass=1,maxLenAttributeSet=4)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-02-25T09:29:28.178445Z",
     "start_time": "2020-02-25T09:29:27.991318Z"
    }
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAWgAAAD4CAYAAADB9HwiAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8li6FKAAAWxUlEQVR4nO3dfbRVdZ3H8fdHpABBzKeWCIa5MCkHgQsmRQQ9aPaA0ZjGmFo0AY1j1mSZpQualctSSscxUzQazFIrUtA1I6RJiFpwsSughBjlCjURfICrYAjf+WPvA4fb5Z5z8Zyzf8jntRaLc357n72/ZwMffvd79tlbEYGZmaVnn6ILMDOz9jmgzcwS5YA2M0uUA9rMLFEOaDOzRO1bdAGWhoMPPjj69+9fdBlme50lS5asi4hD2lvmgDYA+vfvT3Nzc9FlmO11JD2xq2VucZiZJcoBbWaWKAe0mVmi3IM2s5rbsmULa9asYfPmzUWXkoxu3brRt29funbtWvVrHNBmVnNr1qyhV69e9O/fH0lFl1O4iGD9+vWsWbOGI488surXucVhZjW3efNmDjroIIdzThIHHXRQp3+icECbWV04nHe2O8fDAW1mlij3oM2s7vSt2s6mY0rl69hfddVV/PCHP2TDhg2MGzeOq6++mmuvvZYePXpw1lln1bSeclOnTqVnz56cf/75r3lbDmgze1265ppruPvuu7n77ru3f0t28uTJBVfVOW5xmNnrzuTJk1m9ejUnn3wyzz///PbxqVOnMm3aNABGjx7Neeedx+DBgzn22GNZtGjR9nXOPPNMRowYwYABA7j++uu3v/7yyy9n+PDhDBo0iClTpmwfv+SSSzj66KMZOXIkK1eurNn78AzazF53rr32Wu666y7uvfde7rzzzl2u9/LLL9PS0sKCBQuYMGECy5cvB2Dp0qX87ne/46WXXmLIkCF85CMfYfny5axatYpFixYREYwdO5YFCxaw3377ccstt9DS0sKrr77K0KFDaWpqqsn7cEAbAEueWlLzPmFJNf1CsyKMHz8egFGjRrFhwwZeeOEFAE455RS6d+9O9+7dGTNmDIsWLWLhwoXMmzePIUOGANDa2sqqVavYuHEj48aNo0ePHgCMHTu2ZvU5oM1sr9X21LfS8/bGI4ILL7yQSZMm7bTsyiuvrFt97kGb2V7r1ltvBWDhwoX07t2b3r17AzB79mw2b97M+vXrmT9/PsOHD+ekk05ixowZtLa2AvDkk0+ydu1aRo0axe23386mTZvYuHEjd9xxR83q8wzazOou1TZXt27dGDJkCFu2bGHGjBnbxwcNGsSYMWNYt24dF198MX369KFPnz6sWLGCESNGANCzZ09uuukmhg4dyumnn85xxx3HoYceyvDhw2tWnyLSPHDWWOqjYFLl9XZHqv84rX5WrFjBwIEDiy6jQ6NHj2batGkMGzZsp/FansfcVnvHRdKSiBjW3vpucZiZJcotDjPbK82fP7/d8alTpza0jo54Bm1mdeH26c5253g4oM2s5rp168b69esd0rnS9aC7devWqde5xWFmNde3b1/WrFnDs88+W3QpySjdUaUzHNCJkfRN4F+ArcA2YBIwApgeES9XeG1rRPSsf5VmHevatWun7hxi7XNAJ0TSCOCjwNCIeEXSwcAbgFuBm4AOA9rMXl/cg07LYcC6iHgFICLWAacCfYB7Jd0raYKk7d8tlfR5SVe03ZCkr0paLGmppG816g2YWe04oNMyD+gn6TFJ10h6b0RcBTwFjImIMcDPgY9JKt0a+LPAjPKNSDoRGAAcDwwGmiSNati7MLOacEAnJCJagSZgIvAscKukz7Szzm+Aj0o6BugaEcvabOrE/NcfgIeAY8gCeyeSJkpqltTs5olZetyDTkxEbAXmA/MlLQPObme1G4BvAH8EftzOcgGXRsR1FfY1HZgO+Ve9zSwpnkEnRNLbJJXPdAcDTwAbgV6lwYj4PdCP7GyPm9vZ1FxggqSe+XYPl3Ro3Qo3s7rwDDotPYH/lnQA8CrwOFm7Yzxwl6Sn8j40ZL3owRHxfNuNRMQ8SQOBB/Pr2rYCnwbWNuA9mFmN+Gp2eyhJdwJXRMQ9Ndmer2ZnVghfze51RNIBkh4DNtUqnM0sTW5x7GEi4gXg6KLrMLP68wzazCxRDmgzs0Q5oM3MEuWANjNLlAPazCxRDmgzs0T5NDsDoKlPE81Tmosuw8zKeAZtZpYoB7SZWaIc0GZmiXJAm5klygFtZpYoB7SZWaIc0GZmiXJAm5klygFtZpYoB7SZWaIc0GZmiXJAm5klygFtZpYoB7SZWaIc0GZmiXJAm5klygFtZpYoB7SZWaIc0GZmiXJAm5klygFtZpYoB7SZWaIc0GZmiXJAm5klygFtZpYoB7SZWaIUEUXXYAlQHwWTiq7C7PUjplSXrZKWRMSw9pZ5Bm1mligHtJlZohzQZmaJckCbmSXKAW1mligHtJlZohzQZmaJckCbmSXKAW1mligHtJlZohzQZmaJqltAS9oqqUXSw5IekvSueu2rzX7nS1qZ77tF0qm7sY3R9a5X0lRJL0s6tGystZ77NLM9y7513PamiBgMIOkk4FLgveUrSNo3Il6tw77PiIjm1/D60UAr8EC1L9jN97IO+ApwQSdfZ2Z7gUa1OPYHnofts9P7JM0BHs3HPi1pUT7jvU5SF0mflPT9fPl5klbnj98q6f7OFiDpdklLJD0iaWLZ+IfyGf7Dku6R1B+YDHw5r+c9kvpL+o2kpfk6R+Sv/R9J10r6PXCZpFWSDsmX7SPp8dLzXZgBnC7pwE7U2yrp8nz8bknH5z81rJY0Nl+nS77O4rzmdq9TJ2mipGZJzbzc2SNqZvVWzxl0d0ktQDfgMOB9ZcuGAsdGxJ8lDQROB94dEVskXQOcAcwDvpav/x5gvaTD88cLKuz7p5I25Y/fHxHrgQkR8Zyk7sBiSbPI/oO6HhiV13Jgvs61QGtETAOQdAcwMyJmSpoAXAV8PN9+X+BdEbFV0ot57VcCHwAejohnO6izlSykzwOmtFn2D/Xm72M/4DcR8VVJtwHfBj4IvB2YCcwBPge8GBHDJb0RuF/SvIj4c/kOImI6MB3yy42aWVIa1eIYAdwo6dh82aKysHg/0EQWQgDdgbUR8TdJPSX1AvoBPwNGkQX0ryrsu70Wxxcljcsf9wMGAIcAC0q1RMRzu9jeCOAT+eOfAJeVLftFRGzNH88AZpMF9ATgxxXqhCzsWyRNq6Le9cDfgbvy8WXAK/l/bMuA/vn4icCgsv577/z1OwW0maWtngG9XUQ8KOlgskAEeKlsschmpxe289IHgM8CK4H7yEJvBFnftmqSRpPNaEdExMuS5pPN7Gth+3uJiL9KekbS+4DjyWbTHYqIFyT9DDinynq3xI67LGwDXsm3s01S6c9TwLkRMfc1vTMzK1RDetCSjgG6kM0A27oHOLV0NoOkAyW9JV92H3A+WUvjD8AYshnji50soTfwfB52xwAn5OO/A0ZJOrK073x8I9Cr7PUPAJ/KH5+R17UrNwA3sfPMupLvA5PY8R/mruqt1lzgC5K6Akg6WtJ+ndyGmRWs0wEt6U2SBlWxavfSqW7ArcDZ7QVWRDwKXATMk7QU+DVZzxqyIOxH1obYCvwVWNjZmslaAvtKWgF8hyyYyfvDE4FfSXo4rxPgDmBc6UNC4Fzgs3l9Z5L1jHdlDtCT6tob5HWsA24D3thRvZ1wA9kHsA9JWg5cR4N+WjKz2qnqnoT5j9hjyf6RLwHWAvdHxH/Utbo9kKRhwBUR8Z6ia+kM35PQrLYaeU/C3hGxgeyDshsj4p1kPVIrI+nrwCygvX66mVmnVPtj776SDgNOA75Zx3qqlp9idmSb4QuK/GAsIr5D1pLYTtI3gU+2WfUXEXFJwwozsz1StQH9n2QfPN0fEYslvRVYVb+yKouIcZXXKl4exA5jM+u0qgI6In4B/KLs+Wrgn+tVlJmZVdmDzk/Tuic/IwBJgyRdVN/SzMz2btV+SHg92QdfWwAiYik7zgs2M7M6qLYH3SMiFuVfxS6px1XorCBNfZponvJaLgBoZrVW7Qx6naSjgADIr/HwdN2qMjOzqmfQ55Bd9ewYSU+SXXSn4nUmzMxs91Ub0BERH8iv57BPRGwsXb/CzMzqo9oWxyyAiHgpIjbmY7+sT0lmZgYVZtD5ldTeAfSW9ImyRftTu8t1mplZOyq1ON4GfBQ4APhY2fhG4PP1KsrMzCoEdETMBmZLGhERDzaoJjMzo/oPCSdK+ocZc0RMqHE9ZmaWqzag7yx73A0YBzxV+3LMzKyk2oslzSp/Lulmdu/OJmZmVqXdvSfhAODQWhZiZmY7q2oGLWkj2de8lf/+N+CCOtZlZrbXq7bF0avyWmZmVktV3+k5/6LKSLIZ9H0RcXvdqjIzs6ov2H8NMBlYBiwHJkv6QT0LMzPb21U7g34fMDAiSpcbnQk8UreqzMys6rM4HgeOKHveLx8zM7M6qXSxpDvIes69gBWSFuXP3wksqn95ZmZ7r0otjmkNqcLMzP5BpYsl/bZRhZiZ2c4qtTgWRsTIsi+qbF9EdpeV/etanZnZXqzSDHpk/ru/qGJm1mAVz+KQ1EXSHxtRjJmZ7VAxoCNiK7BS0hGV1jUzs9qp9osqbwIeyU+ze6k0GBFj61KVmZlVHdAX17UKMzP7B9UG9IcjYqfLi0r6LuDT8MzM6kT55TU6Xkl6KCKGthlbGhGD6laZNZT6KJhUdBWNFVMq/903qzdJSyJiWHvLKp0H/QXg34CjJC0tW9QLeKB2JZqZWVuVWhw/A/4PuBT4etn4xoh4rm5VmZlZxS+qvAi8KOnViHiifJmkn0TEmXWtzsxsL1bt5UbfUf5E0r5AU+3LMTOzkg4DWtKF+XU4BknaUPoFPAPMbkiFZmZ7qUotjkuBSyVdClwGHA10Ky2uc21mZnu1as+DXg0sAPoCLcAJwINkt8IyM7M6qLYH/UVgOPBERIwBhgAv1K0qMzOrOqA3R8RmAElvjIg/Am+rX1lmZlZtQK+RdABwO/BrSbOBJyq8xmpE0sclhaRj8uf9JW2S1CLpYUkPSHqbpB6S1kvav83rb5d0ejHVm9nuqiqgI2JcRLwQEVPJLpz0I+Dj9SzMdjIeWJj/XvKniBgcEccBM4FvRMTLwFxgXGklSb2BkcAdDazXzGqg2hn0dhHx24iYExF/r0dBtjNJPckC9nPAp3ax2v7A8/njm9usNw6Ym4e3me1Bqj2Lw4pzCnBXRDyWty+agPVk10dpIbsuSg/gnfn6c4EbJB0UEevJwvrq9jYsaSIwEYDe9X0TZtZ5nZ5BW8ONB27JH9/CjjZHqcVxFPAlYDpA/pPNHOBUSQeTnXEzt70NR8T0iBgWEcPoUc+3YGa7wzPohEk6kOxc83+SFEAXsi8I/aDNqnOAH5c9v5nsswIBsyNiSwPKNbMa8ww6bacCP4mIt0RE/4joB/wZ6NdmvZHAn8qezwcGAOeQhbWZ7YE8g07beOC7bcZmAReyowct4O/Av5ZWiIhtkn4JnIbvemO2x6rqjir2+uc7qpgVo6M7qrjFYWaWKAe0mVmiHNBmZolyQJuZJcoBbWaWKAe0mVmiHNBmZolyQJuZJcrfJDQAmvo00TyluegyzKyMZ9BmZolyQJuZJcoBbWaWKAe0mVmiHNBmZolyQJuZJcoBbWaWKAe0mVmiHNBmZolyQJuZJcoBbWaWKAe0mVmiHNBmZolyQJuZJcoBbWaWKAe0mVmiHNBmZolyQJuZJcoBbWaWKAe0mVmiHNBmZolyQJuZJcoBbWaWKAe0mVmiHNBmZolyQJuZJUoRUXQNlgD1UTCp6CpsTxVTnCO7S9KSiBjW3jLPoM3MEuWANjNLlAPazCxRDmgzs0Q5oM3MEuWANjNLlAPazCxRDmgzs0Q5oM3MEuWANjNLlAM6J2mrpBZJD0t6SNK76ry/+ZKay54PkzS/nvs0sz2LA3qHTRExOCKOAy4ELq3FRiV16WDxoZJOrsV+zOz1xwHdvv2B50tPJH1V0mJJSyV9q2z805IW5TPv60phLKlV0vckPQyM6GA/lwPfbDsoqb+k+/KZ/PbZvKTRkn4rabak1ZK+I+mMvIZlko7K1ztE0qy85sWS3l2bw2JmjbRv0QUkpLukFqAbcBjwPgBJJwIDgOMBAXMkjQKeBU4H3h0RWyRdA5wB3AjsB/w+Ir5SYZ8PAuMkjQE2lo2vBT4YEZslDQBuBkpXuzoOGAg8B6wGboiI4yWdB5wLfAn4L+CKiFgo6Qhgbv6anUiaCEwEoHd1B8nMGscBvcOmiBgMIGkEcKOkY4ET819/yNfrSRbYg4AmYLEkgO5kwQqwFZhV5X6/DVwEXFA21hW4WtLgfFtHly1bHBFP53X+CZiXjy8DxuSPPwC8Pa8LYH9JPSOitXzHETEdmA755UbNLCkO6HZExIOSDgYOIZs1XxoR15WvI+lcYGZEXNjOJjZHxNYq9/UbSd8GTigb/jLwDNlseR9gc9myV8oebyt7vo0df577ACdERPnrzGwP4x50OyQdA3QB1pO1ByZI6pkvO1zSocA9wKn5YyQdKOktu7nLbwNfK3veG3g6IrYBZ+a1dMY8snZH6f0M3s26zKxAnkHvUOpBQzZrPjufBc+TNBB4MG8ZtAKfjohHJV2UL98H2AKcAzzR2R1HxP9KerZs6BpglqSzgLuAlzq5yS8CP5C0lOzPeAEwubN1mVmxfMsrA3zLK3ttfMur3edbXpmZ7YHc4qgzSbcBR7YZviAi5hZRj5ntORzQdRYR44quwcz2TG5xmJklygFtZpYoB7SZWaIc0GZmifKHhAZAU58mmqc0V17RzBrGM2gzs0Q5oM3MEuWANjNLlAPazCxRDmgzs0Q5oM3MEuWANjNLlAPazCxRDmgzs0Q5oM3MEuWANjNLlAPazCxRDmgzs0T5rt4GgKSNwMqi62jHwcC6ootoh+vqHNe1a2+JiEPaW+DLjVrJyl3d+r1IkppdV/VcV+ekWleJWxxmZolyQJuZJcoBbSXTiy5gF1xX57iuzkm1LsAfEpqZJcszaDOzRDmgzcwS5YA2JH1I0kpJj0v6etH1lEj6i6RlklokFXbLcUkzJK2VtLxs7EBJv5a0Kv/9TYnUNVXSk/kxa5H04QbX1E/SvZIelfSIpPPy8UKPVwd1FXq8KnEPei8nqQvwGPBBYA2wGBgfEY8WWhhZQAPDIqLQLxJIGgW0AjdGxLH52GXAcxHxnfw/tTdFxAUJ1DUVaI2IaY2spaymw4DDIuIhSb2AJcDHgc9Q4PHqoK7TKPB4VeIZtB0PPB4RqyPi78AtwCkF15SUiFgAPNdm+BRgZv54Jtk/9obaRV2FioinI+Kh/PFGYAVwOAUfrw7qSpoD2g4H/lr2fA3p/MUNYJ6kJZImFl1MG2+OiKfzx38D3lxkMW38u6SleQuk4a2XEkn9gSHA70noeLWpCxI5Xu1xQFvKRkbEUOBk4Jz8R/rkRNYnTKVX+EPgKGAw8DTwvSKKkNQTmAV8KSI2lC8r8ni1U1cSx2tXHND2JNCv7HnffKxwEfFk/vta4Daydkwqnsn7mqX+5tqC6wEgIp6JiK0RsQ24ngKOmaSuZCH404j4VT5c+PFqr64UjldHHNC2GBgg6UhJbwA+BcwpuCYk7Zd/mIOk/YATgeUdv6qh5gBn54/PBmYXWMt2pRDMjaPBx0ySgB8BKyLi+2WLCj1eu6qr6ONVic/iMPJTi64EugAzIuKSgktC0lvJZs2QXXXxZ0XVJelmYDTZpSmfAaYAtwM/B44AngBOi4iGfmC3i7pGk/24HsBfgEllvd9G1DQSuA9YBmzLh79B1u8t7Hh1UNd4CjxelTigzcwS5RaHmVmiHNBmZolyQJuZJcoBbWaWKAe0mVmiHNBmZolyQJuZJer/AYYVBU6Ir6nCAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 288x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "ax,posRank1 = aggregateRankings(pos_rankings1,positive=True,maxLenAttributes=1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Analyze patterns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-02-25T09:52:14.879636Z",
     "start_time": "2020-02-25T09:52:14.186593Z"
    }
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAABLMAAAK3CAYAAAB+ww1OAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAAWJQAAFiUBSVIk8AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8li6FKAAAgAElEQVR4nOzdf7TldV3v8ddbhgQpEb0VptkIC5i5ZhhooNwQmCWpLBEL01sSsrKyLBOldKXiYD8WldeflD8SnRzuvei1q9XCXy0FBkRvq+EalxX+xEEpfyEBImiBn/vH/h4bD+fM7DOz53z3Z3g81jrre/b3x+f72X5VZp5893dXay0AAAAA0IP7jD0BAAAAAJiWmAUAAABAN8QsAAAAALohZgEAAADQDTELAAAAgG6IWQAAAAB0Q8wCAAAAoBtiFgAAAADdELMAAAAA6IaYBQAAAEA3xCwAAAAAuiFmAQAAANCNNWNPYN5V1eeT3D/JtpGnAgAAALC3WJvkttbaw1d6oJi1c/fff//9H7h+/foHjj0RAAAAgL3BddddlzvvvHOXjhWzdm7b+vXrH7h169ax5wEAAACwVzj66KNz9dVXb9uVYz0zCwAAAIBuiFkAAAAAdEPMAgAAAKAbYhYAAAAA3RCzAAAAAOiGmAUAAABAN8QsAAAAALohZgEAAADQDTELAAAAgG6IWQAAAAB0Q8wCAAAAoBtiFgAAAADdELMAAAAA6IaYBQAAAEA3xCwAAAAAuiFmAQAAANANMQsAAACAbohZAAAAAHRDzAIAAACgG2IWAAAAAN0QswAAAADohpgFAAAAQDfELAAAAAC6IWYBAAAA0I01Y0+A1bX2JZeMPYW9xrbzTxl7CgAAAHCv484sAAAAALohZgEAAADQDTELAAAAgG6IWQAAAAB0Q8wCAAAAoBtiFgAAAADdELMAAAAA6MZMYlZVnV5Vb6iqK6rqtqpqVXXRMvtuGrbv6OfDU5537U7GuXgW7w8AAACA+bBmRuO8LMmRSW5PcmOSdTvY971Jti2z7YwkhyR5/wrP/4/DuItdu8JxAAAAAJhjs4pZZ2cSsT6b5PFJLl1ux9bae7NEeKqqByT53ST/lmTTCs//idbaxhUeAwAAAEBnZhKzWmvfjVdVtavDnJFk/yQXt9ZumsW8AAAAANi7zOrOrFn4lWH5ll049keq6teSPCjJ15N8rLV2zcxmBgAAAMBcmIuYVVWPTfLIJJ/e/i6vFXjC8LP9mJclObO19oUp57B1mU07ev4XAAAAAKtoJt9mOAO/Oiz/YoXH3ZHk95McneSg4WfhmV0nJPlwVR0wozkCAAAAMLLR78yqqgOT/Hx24cHvrbWvJjl30eotVXVykiuTHJPkOUleN8VYRy8zv61JjlrJvAAAAADYM+bhzqxnJblfkv89qwe/t9buSvLW4eXxsxgTAAAAgPHNQ8xaePD7m2c87teGpY8ZAgAAAOwlRo1ZVXVMkiMzefD7ZTMe/thhef2MxwUAAABgJGPfmbXw4Pe37GinqjqwqtZV1YMXrT+qqu7xHqpqQ5Kzh5cXzWSmAAAAAIxuJg+Ar6rTkpw2vDx4WD62qjYNv9/UWjtn0TH3T/KMJN9O8pc7OcXTkrx92O/Z261/dZLDquqqJDcO634iyUnD7y9vrV21ojcDAAAAwNya1bcZPirJmYvWHTL8JMkNSc5ZtP0XM3me1cW78eD3zZmErsckeVKSfZN8Jcm7klzQWrtiF8cFAAAAYA7NJGa11jYm2bjCY96Y5I1T7rspyaYl1l+Y5MKVnBcAAACAfo39zCwAAAAAmJqYBQAAAEA3xCwAAAAAuiFmAQAAANANMQsAAACAbohZAAAAAHRDzAIAAACgG2IWAAAAAN0QswAAAADohpgFAAAAQDfELAAAAAC6IWYBAAAA0A0xCwAAAIBuiFkAAAAAdEPMAgAAAKAbYhYAAAAA3RCzAAAAAOiGmAUAAABAN8QsAAAAALohZgEAAADQDTELAAAAgG6IWQAAAAB0Q8wCAAAAoBtiFgAAAADdELMAAAAA6IaYBQAAAEA3xCwAAAAAuiFmAQAAANANMQsAAACAbohZAAAAAHRDzAIAAACgG2IWAAAAAN0QswAAAADohpgFAAAAQDfWjD0BYGLtSy4Zewp7jW3nnzL2FAAAANhD3JkFAAAAQDfELAAAAAC6IWYBAAAA0A0xCwAAAIBuiFkAAAAAdEPMAgAAAKAbYhYAAAAA3RCzAAAAAOiGmAUAAABAN8QsAAAAALohZgEAAADQDTELAAAAgG6IWQAAAAB0Q8wCAAAAoBtiFgAAAADdELMAAAAA6IaYBQAAAEA3xCwAAAAAuiFmAQAAANANMQsAAACAbohZAAAAAHRDzAIAAACgGzOJWVV1elW9oaquqKrbqqpV1UXL7Lt22L7cz8W7cP7HVdX7qurmqrqzqq6pqhdU1T67/+4AAAAAmBdrZjTOy5IcmeT2JDcmWTfFMf+Y5L1LrL92JSeuqqcm+ask30ryziQ3J3lKktckOS7J01cyHgAAAADza1Yx6+xMItZnkzw+yaVTHPOJ1trG3TlpVd0/yV8kuTvJCa21fxjWvzzJR5KcXlXPbK2t+G4vAAAAAObPTD5m2Fq7tLX2mdZam8V4K3B6kh9McvFCyBrm861M7hZLkl9f5TkBAAAAsIfM6s6sXfEjVfVrSR6U5OtJPtZau2aFY5w0LD+wxLYtSe5I8riqum9r7du7PlUAAAAA5sGYMesJw893VdVlSc5srX1hyjGOGJafXryhtXZXVX0+ySOSHJLkuh0NVFVbl9k0zfO/AAAAAFgFM/mY4QrdkeT3kxyd5KDhZ+E5Wyck+XBVHTDlWAcOy1uX2b6w/gG7NFMAAAAA5sqq35nVWvtqknMXrd5SVScnuTLJMUmek+R1qzyvo5daP9yxddRqzgUAAACApY1xZ9aSWmt3JXnr8PL4KQ9buPPqwGW2L6y/ZVfnBQAAAMD8mJuYNfjasJz2Y4afGpaHL95QVWuSPDzJXUmu3/2pAQAAADC2eYtZxw7LaePTR4blE5fYdnyS+yW5yjcZAgAAAOwdVj1mVdVRVXWP81bVhiRnDy8vWrTtwKpaV1UPXnTYu5PclOSZVfXo7fbfL8kfDC/fOLPJAwAAADCqmTwAvqpOS3La8PLgYfnYqto0/H5Ta+2c4fdXJzmsqq5KcuOw7ieSnDT8/vLW2lWLTvG0JG9P8pdJnr2wsrV2W1X9SiZR67KqujjJzUlOTXLEsP6du/0GAQAAAJgLs/o2w0clOXPRukOGnyS5IclCzNqcSZx6TJInJdk3yVeSvCvJBa21K1Zy4tbae6vq8UlemuTnkuyX5LNJXpjk9a21tuJ3AwAAAMBcmknMaq1tTLJxyn0vTHLhCsfflGTTDrZ/NMmTVzImAAAAAP2ZtwfAAwAAAMCyxCwAAAAAuiFmAQAAANANMQsAAACAbohZAAAAAHRDzAIAAACgG2IWAAAAAN0QswAAAADohpgFAAAAQDfELAAAAAC6IWYBAAAA0A0xCwAAAIBuiFkAAAAAdEPMAgAAAKAbYhYAAAAA3RCzAAAAAOiGmAUAAABAN8QsAAAAALohZgEAAADQDTELAAAAgG6IWQAAAAB0Q8wCAAAAoBtiFgAAAADdELMAAAAA6IaYBQAAAEA3xCwAAAAAuiFmAQAAANANMQsAAACAbohZAAAAAHRDzAIAAACgG2IWAAAAAN0QswAAAADohpgFAAAAQDfELAAAAAC6IWYBAAAA0A0xCwAAAIBuiFkAAAAAdEPMAgAAAKAbYhYAAAAA3RCzAAAAAOiGmAUAAABAN8QsAAAAALohZgEAAADQDTELAAAAgG6IWQAAAAB0Q8wCAAAAoBtiFgAAAADdELMAAAAA6IaYBQAAAEA3xCwAAAAAuiFmAQAAANANMQsAAACAbohZAAAAAHRDzAIAAACgG2IWAAAAAN0QswAAAADohpgFAAAAQDdmErOq6vSqekNVXVFVt1VVq6qLltn3sKp6cVV9pKq+WFX/VlVfqaq/rqoTV3jetcO5lvu5eBbvDwAAAID5sGZG47wsyZFJbk9yY5J1O9j395M8I8k/JXlfkpuTHJHk1CSnVtVvt9Zev8Lz/2OS9y6x/toVjgMAAADAHJtVzDo7k4j12SSPT3LpDvb9QJI/bq393+1XVtXjk/xdkj+tqv/VWvvSCs7/idbaxpVNGQAAAIDezORjhq21S1trn2mttSn23bQ4ZA3rL09yWZLvS/K4WcwLAAAAgL3LrO7MmpV/H5Z3rfC4H6mqX0vyoCRfT/Kx1to1M50ZAAAAAKObm5hVVT+WZEOSO5JsWeHhTxh+th/vsiRntta+MOX5ty6zaUfP/wIAAABgFc3kY4a7q6rum+S/J7lvko2ttX+d8tA7Mnmg/NFJDhp+Fp7ZdUKSD1fVATOfMAAAAACjGP3OrKraJ8nmJMcleWeSV017bGvtq0nOXbR6S1WdnOTKJMckeU6S100x1tHLzG9rkqOmnRMAAAAAe86od2YNIeuiJE9P8q4kz5rmIfI701q7K8lbh5fH7+54AAAAAMyH0WJWVe2b5H8meWaS/5HkF4YINStfG5Y+ZggAAACwlxjlY4ZV9X2Z3In11CTvSHJWa+07Mz7NscPy+hmPCwAAAMBIVv3OrOFh7+/JJGRdmClCVlUdWFXrqurBi9YfVVX3eA9VtSHJ2cPLi2YzcwAAAADGNpM7s6rqtCSnDS8PHpaPrapNw+83tdbOGX5/U5InJ7kpyT8nObeqFg95WWvtsu1ePy3J25P8ZZJnb7f+1UkOq6qrktw4rPuJJCcNv7+8tXbVrr0rAAAAAObNrD5m+KgkZy5ad8jwkyQ3JFmIWQ8flv8p9/wmwu1dNsV5N2cSuh6T5ElJ9k3ylUw+wnhBa+2KKcYAAAAAoBMziVmttY1JNk657wm7MP6mJJuWWH9hJh9VBAAAAOBeYLRvMwQAAACAlRKzAAAAAOiGmAUAAABAN8QsAAAAALohZgEAAADQDTELAAAAgG6IWQAAAAB0Q8wCAAAAoBtiFgAAAADdELMAAAAA6IaYBQAAAEA3xCwAAAAAuiFmAQAAANANMQsAAACAbohZAAAAAHRDzAIAAACgG2IWAAAAAN0QswAAAADohpgFAAAAQDfELAAAAAC6IWYBAAAA0A0xCwAAAIBuiFkAAAAAdEPMAgAAAKAbYhYAAAAA3RCzAAAAAOiGmAUAAABAN8QsAAAAALohZgEAAADQDTELAAAAgG6IWQAAAAB0Q8wCAAAAoBtiFgAAAADdELMAAAAA6IaYBQAAAEA3xCwAAAAAuiFmAQAAANANMQsAAACAbohZAAAAAHRDzAIAAACgG2IWAAAAAN0QswAAAADohpgFAAAAQDfELAAAAAC6IWYBAAAA0A0xCwAAAIBuiFkAAAAAdEPMAgAAAKAbYhYAAAAA3RCzAAAAAOiGmAUAAABAN8QsAAAAALohZgEAAADQDTELAAAAgG6IWQAAAAB0Q8wCAAAAoBtiFgAAAADdmEnMqqrTq+oNVXVFVd1WVa2qLtrJMY+rqvdV1c1VdWdVXVNVL6iqfXbh/P+5qt5VVV+tqm9V1aeq6ryq2n/X3xUAAAAA82bNjMZ5WZIjk9ye5MYk63a0c1U9NclfJflWkncmuTnJU5K8JslxSZ4+7Ymr6pgkH0myb5J3J/likpOSnJtkQ1VtaK19e4XvBwAAAIA5NKuPGZ6d5PAk90/y6zvasarun+Qvktyd5ITW2i+31n4nyaOSfCzJ6VX1zGlOOtzF9fYk90tyemvtF1prL05yTCax7LhhbgAAAADsBWYSs1prl7bWPtNaa1PsfnqSH0xycWvtH7Yb41uZ3OGV7CSIbefxSdYn2dJa+5vtxvpOkt8dXj63qmrK8QAAAACYY2M8AP6kYfmBJbZtSXJHksdV1X13Z6zW2vVJPp3kx5IcsgvzBAAAAGDOzOqZWStxxLD89OINrbW7qurzSR6RSYC6blfHGnwmk48/Hp7kczsaqKq2LrNph8//AgAAAGD1jHFn1oHD8tZlti+sf8AqjwUAAADAnBvjzqy51Fo7eqn1wx1bR63ydAAAAABYwhh3Zi3cLXXgMtsX1t+yymMBAAAAMOfGiFmfGpaHL95QVWuSPDzJXUmu352xBocNy+WeqQUAAABAR8aIWR8Zlk9cYtvxSe6X5KrW2rd3Z6yqOiSTyHVDpgtjAAAAAMy5MWLWu5PclOSZVfXohZVVtV+SPxhevnH7A6rqflW1rqoetmisyzP5xsPjq+rU7fa/T5I/Hl6+qbXWZvweAAAAABjBTB4AX1WnJTlteHnwsHxsVW0afr+ptXZOkrTWbquqX8kkal1WVRcnuTnJqUmOGNa/c9EpfirJpZnEqxMWVrbW7q6qszK5Q+vdVfXuJF9IsiHJo5N8NMlrZvEeAQAAABjfrL7N8FFJzly07pDhJ5l81O+chQ2ttfdW1eOTvDTJzyXZL8lnk7wwyetXcidVa+3/VNVjkpyX5OQkPzCc75VJzp/y44oAAAAAdGAmMau1tjHJxhUe89EkT55y38uS1A62/1OSp6/k/AAAAAD0Z4xnZgEAAADALhGzAAAAAOiGmAUAAABAN8QsAAAAALohZgEAAADQDTELAAAAgG6IWQAAAAB0Q8wCAAAAoBtiFgAAAADdELMAAAAA6IaYBQAAAEA3xCwAAAAAuiFmAQAAANANMQsAAACAbohZAAAAAHRDzAIAAACgG2IWAAAAAN0QswAAAADohpgFAAAAQDfELAAAAAC6IWYBAAAA0A0xCwAAAIBuiFkAAAAAdEPMAgAAAKAbYhYAAAAA3RCzAAAAAOiGmAUAAABAN8QsAAAAALohZgEAAADQDTELAAAAgG6IWQAAAAB0Q8wCAAAAoBtiFgAAAADdELMAAAAA6IaYBQAAAEA3xCwAAAAAuiFmAQAAANANMQsAAACAbohZAAAAAHRDzAIAAACgG2IWAAAAAN0QswAAAADohpgFAAAAQDfELAAAAAC6IWYBAAAA0A0xCwAAAIBuiFkAAAAAdEPMAgAAAKAbYhYAAAAA3RCzAAAAAOiGmAUAAABAN8QsAAAAALohZgEAAADQDTELAAAAgG6IWQAAAAB0Q8wCAAAAoBtiFgAAAADdGCVmVdWzq6rt5OfuKcfatoMxvryn3wsAAAAAq2fNSOf9RJLzltn200lOSvL+FYx3a5LXLrH+9hXOCwAAAIA5NkrMaq19IpOgdQ9V9bHh17esYMhbWmsbd3deAAAAAMy3uXpmVlU9MsmxSf45ySUjTwcAAACAOTPWxwyX86vD8sLW2lTPzBrct6qeleRhSb6Z5JokW1Y4BgAAAABzbm5iVlXtn+RZSe5O8tYVHn5wks2L1n2+qs5qrV0+5fm3LrNp3QrnAgAAAMAeMk8fM/z5JA9I8oHW2hdXcNzbk2zIJGgdkOSRSd6cZG2S91fVkTOeJwAAAAAjmZs7s/IfHzF880oOaq0t/lbEa5M8t6puT/KiJBuTPG2KcY5eav1wx9ZRK5kTAAAAAHvGXNyZVVWPSPK4JDcmed+Mhn3TsDx+RuMBAAAAMLK5iFnZ9Qe/78jXhuUBMxoPAAAAgJGNHrOqar8kZ2Ty4PcLZzj0scPy+hmOCQAAAMCIRo9ZSZ6e5KAk71/uwe9VtW9VrauqQxetX19V97jzqqrWJrlgeHnRbKcLAAAAwFjm4QHwCx8xfMsO9nlIkuuS3JDJtxQueEaSF1XVlmHbN5IcmuSUJPtl8vytV814vgAAAACMZNSYVVXrk/yX7PqD3y9NckSSn0xyXCbPx7olyZVJNifZ3Fprs5ktAAAAAGMbNWa11q5LUlPst22p/Vprlye5fPYzAwAAAGAezcMzswAAAABgKmIWAAAAAN0QswAAAADohpgFAAAAQDfELAAAAAC6Meq3GQL0Yu1LLhl7CnuNbeefMvYUAACAjrkzCwAAAIBuiFkAAAAAdEPMAgAAAKAbYhYAAAAA3RCzAAAAAOiGmAUAAABAN8QsAAAAALohZgEAAADQDTELAAAAgG6IWQAAAAB0Q8wCAAAAoBtiFgAAAADdELMAAAAA6IaYBQAAAEA3xCwAAAAAuiFmAQAAANANMQsAAACAbohZAAAAAHRDzAIAAACgG2IWAAAAAN0QswAAAADohpgFAAAAQDfELAAAAAC6IWYBAAAA0A0xCwAAAIBuiFkAAAAAdEPMAgAAAKAbYhYAAAAA3RCzAAAAAOiGmAUAAABAN8QsAAAAALohZgEAAADQDTELAAAAgG6IWQAAAAB0Q8wCAAAAoBtiFgAAAADdELMAAAAA6IaYBQAAAEA3xCwAAAAAuiFmAQAAANANMQsAAACAbohZAAAAAHRDzAIAAACgG2IWAAAAAN0QswAAAADohpgFAAAAQDfELAAAAAC6IWYBAAAA0A0xCwAAAIBuiFkAAAAAdEPMAgAAAKAbo8WsqtpWVW2Zny+vcKyHVtXbqupfqurbw9ivraqD9tT8AQAAAFh9a0Y+/61JXrvE+tunHaCqDk1yVZIfSvLXST6Z5KeS/HaSJ1bVca21r89grgAAAACMbOyYdUtrbeNujvHnmYSs57fW3rCwsqpeneTsJH+Y5Lm7eQ4AAAAA5kDXz8wa7so6Ocm2JH+2aPMrknwzyRlVdcAqTw0AAACAPWDsO7PuW1XPSvKwTMLTNUm2tNbunvL4E4flh1pr39l+Q2vtG1X10Uxi17FJPjyjOQMAAAAwkrFj1sFJNi9a9/mqOqu1dvkUxx8xLD+9zPbPZBKzDs9OYlZVbV1m07op5gEAAADAKhjzY4ZvT7Ihk6B1QJJHJnlzkrVJ3l9VR04xxoHD8tZlti+sf8CuTxMAAACAeTHanVmttfMWrbo2yXOr6vYkL0qyMcnTVnE+Ry+1frhj66jVmgcAAAAAy5vHB8C/aVgeP8W+C3deHbjM9oX1t+zWjAAAAACYC/MYs742LKf5BsJPDcvDl9l+2LBc7plaAAAAAHRkHmPWscPy+in2vXRYnlxV3/NequoHkhyX5I4kH5/d9AAAAAAYyygxq6rWV9U97ryqqrVJLhheXrTd+n2ral1VHbr9/q21zyX5UCYPjX/eouHOy+Turs2ttW/ObPIAAAAAjGasB8A/I8mLqmpLkhuSfCPJoUlOSbJfkvcledV2+z8kyXXDvmsXjfUbSa5K8vqq2jDsd0ySEzP5eOFL99i7AAAAAGBVjRWzLk1yRJKfzOSjgAdk8pD2K5NszuRuqjbNQK21z1XVo5O8MskTkzw5yZeSvC7Jea21f5399AEAAAAYwygxq7V2eZLLV7D/tiS1g+1fTHLW7s8MAAAAgHk2jw+ABwAAAIAliVkAAAAAdEPMAgAAAKAbYhYAAAAA3RCzAAAAAOiGmAUAAABAN8QsAAAAALohZgEAAADQDTELAAAAgG6IWQAAAAB0Q8wCAAAAoBtiFgAAAADdELMAAAAA6IaYBQAAAEA3xCwAAAAAuiFmAQAAANANMQsAAACAbohZAAAAAHRDzAIAAACgG2IWAAAAAN1YM/YEAGB3rH3JJWNPYa+x7fxTZj6m6zM7e+L6AAD0yJ1ZAAAAAHRDzAIAAACgG2IWAAAAAN0QswAAAADohpgFAAAAQDfELAAAAAC6IWYBAAAA0A0xCwAAAIBuiFkAAAAAdEPMAgAAAKAbYhYAAAAA3RCzAAAAAOiGmAUAAABAN8QsAAAAALohZgEAAADQDTELAAAAgG6IWQAAAAB0Q8wCAAAAoBtiFgAAAADdELMAAAAA6IaYBQAAAEA3xCwAAAAAuiFmAQAAANANMQsAAACAbohZAAAAAHRDzAIAAACgG2IWAAAAAN1YM/YEAAAYx9qXXDL2FPYK284/ZeZjujaz4/rMrz1xbYB7B3dmAQAAANANMQsAAACAbohZAAAAAHRDzAIAAACgG2IWAAAAAN0QswAAAADohpgFAAAAQDfELAAAAAC6MUrMqqoHVdVzquo9VfXZqrqzqm6tqiur6peraup5VdW2qmrL/Hx5T74PAAAAAFbXmpHO+/Qkb0zypSSXJvlCkh9O8rNJ3prkSVX19NZam3K8W5O8don1t89grgAAAADMibFi1qeTnJrkktbadxZWVtXvJfn7JD+XSdj6qynHu6W1tnHWkwQAAABgvozyMcPW2kdaa3+7fcga1n85yZuGlyes+sQAAAAAmGtj3Zm1I/8+LO9awTH3rapnJXlYkm8muSbJltba3bOeHAAAAADjmauYVVVrkvzS8PIDKzj04CSbF637fFWd1Vq7fMpzb11m07oVzAMAAACAPWiUjxnuwPlJfjzJ+1prH5zymLcn2ZBJ0DogySOTvDnJ2iTvr6oj98A8AQAAABjB3NyZVVXPT/KiJJ9Mcsa0x7XWzlu06tokz62q24fxNiZ52hTjHL3MvLYmOWra+QAAAACw58zFnVlV9ZtJXpfkn5Kc2Fq7eQbDLjxI/vgZjAUAAADAHBg9ZlXVC5K8IZM7qk4cvtFwFr42LA+Y0XgAAAAAjGzUmFVVL07ymiSfyCRkfXWGwx87LK+f4ZgAAAAAjGi0mFVVL8/kge9bk2xord20g333rap1VXXoovXrq+oed15V1dokFwwvL5rZpAEAAAAY1SgPgK+qM5O8MsndSa5I8vyqWrzbttbapuH3hyS5LskNmXxL4YJnJHlRVW0Ztn0jyaFJTkmyX5L3JXnVHnkTAAAAAKy6sb7N8OHDcp8kL1hmn8uTbNrJOJcmOSLJTyY5LpPnY92S5Mokm5Nsbq213Z0sAAAAAPNhlJjVWtuYZOMK9t+W5B63brXWLs8kegEAAABwLzD6txkCAAAAwLTELAAAAAC6IWYBAAAA0A0xCwAAAIBuiFkAAAAAdGOUbzMEAACAWVv7kkvGnsJeY9v5p8x8TNdnNvbEtemNO7MAAAAA6IaYBQAAAEA3xCwAAAAAuiFmAQAAANANMQsAAACAbohZAAAAAHRDzAIAAACgG2IWAAAAAN0QswAAAADohpgFAAAAQDfELAAAAAC6IWYBAAAA0A0xCwAAAIBuiFkAAAAAdEPMAgAAAKAbYhYAAAAA3ao1O+kAACAASURBVBCzAAAAAOiGmAUAAABAN8QsAAAAALohZgEAAADQDTELAAAAgG6IWQAAAAB0Q8wCAAAAoBtiFgAAAADdELMAAAAA6IaYBQAAAEA3xCwAAAAAuiFmAQAAANANMQsAAACAbohZAAAAAHRDzAIAAACgG2IWAAAAAN0QswAAAADohpgFAAAAQDfELAAAAAC6IWYBAAAA0A0xCwAAAIBuiFkAAAAAdEPMAgAAAKAbYhYAAAAA3RCzAAAAAOiGmAUAAABAN8QsAAAAALohZgEAAADQDTELAAAAgG6IWQAAAAB0Q8wCAAAAoBtiFgAAAADdELMAAAAA6IaYBQAAAEA3xCwAAAAAujFqzKqqh1bV26rqX6rq21W1rapeW1UHrXCcBw7HbRvG+Zdh3IfuqbkDAAAAsPrWjHXiqjo0yVVJfijJXyf5ZJKfSvLbSZ5YVce11r4+xTgPGsY5PMlHklycZF2Ss5KcUlWPba1dv2feBQAAAACracw7s/48k5D1/Nbaaa21l7TWTkrymiRHJPnDKcf5o0xC1qtbaxuGcU7LJIr90HAeAAAAAPYCo8Ss4a6sk5NsS/Jniza/Isk3k5xRVQfsZJzvT3LGsP/GRZsvSHJDkp+pqkN2f9YAAAAAjG2sO7NOHJYfaq19Z/sNrbVvJPlokvslOXYn4xybZP8kHx2O236c7yT54KLzAQAAANCxaq2t/kmr/jTJOUnOaa39tyW2X5DkeUl+o7X2xh2M87xM7sC6oLX2W0tsPyfJnyb5k9bai3cyp63LbDpy//3332f9+vU7Orwb1/7zrWNPYa/x4w85cKbjuTazM+trk7g+s+R/O/PL/3bmm+szv1yb+eb6zC/XZr65PvNrT1ybMVx33XW58847b26tPWilx471APiF/+SX+2/ywvoHrNI4O3L3nXfeeevVV1+9bTfGYHrrhuUnR53FFK7+ytgzGEUX18e1mW+uz/xybeab6zO/XJv55vrML9dmvrk+82svujZrk9y2KweO9m2G86a1dvTYc+A/7pBzPeaT6zO/XJv55vrML9dmvrk+88u1mW+uz/xybeab69OPsZ6ZtXDH1HL3xi2sv2WVxgEAAACgA2PFrE8Ny8OX2X7YsPz0Ko0DAAAAQAfGilmXDsuTq+p75lBVP5DkuCR3JPn4Tsb5eJI7kxw3HLf9OPdJcvKi8wEAAADQsVFiVmvtc0k+lMnDvp63aPN5SQ5Isrm19s2FlVW1rqrWbb9ja+32JJuH/TcuGuc3h/E/2Fq7fobTBwAAAGAkYz4A/jeSXJXk9VW1Icl1SY5JcmImHwt86aL9rxuWtWj97yU5IckLq+pRSf4+yfokT03y1dwzlgEAAADQqWqtjXfyqh9N8sokT0zyoCRfSvKeJOe11v510b4tSVpri2NWquqBSV6R5LQkD07y9STvT3Jua+3GPfkeAAAAAFg9o8YsAAAAAFiJsR4ADwAAAAArJmYBAAAA0A0xCwAAAIBuiFkAAAAAdEPMAgAAAKAbYhYAAAAA3RCzAAAAAOiGmAUAwNypqodV1fFjzwMAmD9iFnOlqg6qqh8dex4A7J2q6peq6mfGngdTOSvJpWNPAmB3VNW5OwvzVfXTVXXuas2JpVXVQ6vqKVV1RlWdWlUPHXtOLG/N2BOAqvr+JOcl+cUkP5ikZfjvZlUdk+QVSV7WWrt6tEnCHKiqh+3qsa21L8xyLnyvqvqlXT22tfaOWc6FnXpbkjck+eDYE4GeVdX9khyUZJ+ltvvnDnzXxuFnyw72OT6Tv/O8chXmwyJV9WNJ3pzkCUts+7skz22tbVvtebFjYhajqqoDk1yZ5BFJPpHkpiTrt9vl/yX56ST/NYmYtYdV1dt28dDWWvvlmU6GpWzLJPau1HcDMXvMpnzvtans/Fot7CNmra4vx53psMuq6owkL873/nltMf/c2cOq6vpdPLS11g6d6WSYhX2TfGfsSdwbVdXBmfx99CGZ/Fl7S5IvJXlwJn8PPTnJlVX16Nbal8eaJ/fkHzKM7aWZhKxnt9beUVWvSPLdW2xba3dU1eVJNow1wXuZZy+zvmXyF+/l1rckYtae947sWsxizztriXU/m+QpSS5PclkmEeXgJCdm8m9g/ybJe1ZpfvyHDyQ5saru01rzFwdYgap6diZ3N96d5IokX0xy15hzuhe7T3btzwRL/XmO8R2Vyb/UZ/W9PJOQ9eIkr26t3b2woar2SXJ2kj9J8rIkvznKDFlStebvRYynqj6b5DOttScNr1+R5NzW2j7b7fNnSU5vrf3wSNO81xhusd3efZK8JpN/K/H63PMv5L+Vyb+9eGFr7fOrN1OYb1X15ExC1emttb9dYvtTk7wryVNbax9Y7fndm1XVDyf5eCb/f/Y7rTV/eZhTVbUxkz8TuJNuTlTVtf+/vTsPt6sszz/+vZktWIJQAWWKOE+VIIooEIqIWCcsWJyQScC21lpEwShhqCj6c6D+oIgKAYcwKUgZDGqEIAjKJNAqIGGSUQiEUYXk7h/vOmTnZJ99TpC91l7Z9+e6zpWz9nrD9VzZnH3WetbzPg+lWuENtn/TdDwRg0rS7I7DqZSKn5u7LF0eWB/YEJhp+/39ji0WJ+lm4Le239xjzY+AF9veqK64YnypzIqmrQd8f5w1DwOr1xDL0LN9S+expI9REllTRp27DrhA0gnA5cA7gK/WFmjE4JsGnN4tkQVg+4eSzqA8DUwyq14zgfnArsAu1UXsXSxZ4WDbqQpukO2DKX1mYnA8H5iRRFbEuKZ2fG9go+prtIXAfcDJlAqgqN86wHfHWXM5i7+nMQCSzIqmPQQ8e5w1k0nZbVP2Bk4ZneQaYfsmSadW65LMiljkbxl/CtvvgLfUEEssbmrH9ysDL6q+RkvpesSS5gF/ajqIGJ+kNYDVbN/WdCzDqLOiVNJC4GDbae4+mOZTKuN62aBaFwMkyaxo2q+At0p6pu2HRp+UtC7lZu+s2iMLKE+QHhhnzf10f9IUNZG0GbA9Zb//yl2WpEF//f5MSWj18rfA4zXEEh2yZS3iL3IWMFWSnF4lAycTwgfW7sCVTQcRY/o5sJOko21fPPpk9bOzM3B27ZFFT0lmRdOOBM4FzpG0d+cJSS8BvgGsQunXFPW7l5IkObDbSUmqzt9XZ1BRVP/+M4D3s6gRf2dj1zTob85PgXdJ+hfgqM6bvup9+xdgB8bfZh0RMUgOBC4CjpG0n+2Hmw4oikwIH2hbUx7+Xj3WAklvBd5le4/aoooRnwX+ntJC5SRKZf2dlO2HUyk/MwuBw5sKMLpLA/hoXNX0fTrlhvtxymja+4E1KDfin7T9xeYiHF6Svgz8G3Aa5X24qePcZOAI4B+Ar9j+eDNRDi9JH6EkhE+kJHwvo2z3PIXyy/cA4BzgwLG2ikZ/SNoYuJTyOXYT5QbjbmBt4A2U7dPzgNfafqrj1SMialU1tZ5EqSx9FLiB7hXc6TlXM0lfAD7OqAnho4YqnQU8x/aUpuIcRhPZZihpGnBo5/sV9amSiSdQrts6EySiXK/tYfvMJmKLsaUyKxpn+xBJc4B/BTYH1qR8iJxDSZLM7vX3o68Ootx47wTsKOl2Ft2QP5cygeVXpEFvUz4IXGd7N4BS8MMDti8BLpE0izK17cfA8U0FOYxs3yhpc+Bo4I3A80Yt+THwz0lkNUPScsA/U7bivARY1fbIVpxNgA8BX7V9fXNRRgykqR3frwq8aox1eVpev3cBs2yf2GPNLcBmNcUTS2dlYEHTQQwr22dJ2oAy1GoKZfjYfMr20DNsP9JkfNFdklkxEGz/jPGbJUfNbD8s6Q2UJ327AxtTGiBCaV59PPAl239uKMRh92LKU6ROT36u276yegr7TySZVTvbvwPeJOm5wCZ0XBjZvr3R4IaYpJUo29unUp62PgSs1rHkJmAP4A+UquGIqKTn3EDLhPDBNmaCV9LKwFaUybrRkCph9b3qK1ogyayI6KlKVB0OHF41Fl0dmJ8+GQOjc7LKI8CzRp2/AXhTfeHEaFXiKsmrwbE/sA2lovQ/KBWonxk5afuBqlp4e5LMioj2yITwASJpdOX1xyTt3mXp8pRm/SsDx/Q9sBhXJoG2R56uRMSE2X7Y9u1JZA2M2ynbPUfMBTYdteYFlCRXNETSiyXtKOkDTccSQNlaeJHtQ20vpPvT8ptYVIUaEdEGT04I73ayY0L4z2uNangtR+m31Dmgp9vX45Tm/EdQHrZEAyStJulLku6iJHw7+wS/VtI5ktJrbsCkMitqVTVAfCp9FDzSzyTqJ+lvKI3eR3rL7NXx+mTgGtuPNRjisPoliyevzgX2l/QZ4AeUbVTvoIxSj5pJehXwTcoWwxHfrs5tTXm//tH2fzcQ3jCbzPjjteexZJVjRFQk7QLsxaIt1A8ClwPfsn1Sk7ENsUwIHyC2Nxr5vrr/+UqvBvDRnEwCba8kB6Juc0hT0FaRtCflwmcVFj1d2qs6vTbwC2Bv4FuNBDjcvg+8WtLkatLkF4B3A4dQtlCNTGA5oLEIh5SkFwLnU7YPHAm8ENihY8kcynuzE5BkVr3+SJnG1ssGdJ/QFjHUVCaNnAi8l/I7ZgGlv9xawLbA30l6m+33NRflcLI9S9IhlO3R11IqfpB0L4tPCL+4uSiH1jbAzU0HEWOaRklkLTYJdOSk7UclXUD5jIsBkmRW1Mr21KZjiImTtB1wLHA15eJoe2DfkfO2r5X0P8A7STKrdrbPAM7oOJ7XMYltY8qF04m272wmwqE2HVgJeLXt/60ujJ5MZtm2pF+QqVJNuIrSmH+lbsMrqie02wO54YtY0j6UrbqXA58ELrC9QNLywNbA54FdJF1oO/1/apYJ4YPJ9gVNxxA9ZRJoSyWZFRG9fBK4E9ja9oNVomS0q4HX1RtWjMX2fOD/NR1HsC3wA9v/22PNbcB2NcUTixwLfBf4blV5+iRJkyiTP9cgjXgjutmD8qBkq872ArYXALOrLdTXAnuSn6FGZEL44JG060TXjpNQif7IJNCWSjIrBkLVlHJbSjPrlbssse3D6o0qgFcDJ9l+sMea3wPr1BRPRFusQfnZ6EWU6q2oke2ZVdXpbsDbgfsBJF1G2WawMnCU7XMaCzJicL0U+PpYfTJtPybpDEoFV9RI0mzKcIvPjLs46jaD8dusjLTySDKrfpkE2lJJZkXjqv39B7D4/48jH+id3yeZVb+VGH8S3iRKz4zoM0lbVd/+0vYfO47HZXtOn8KK7u4Gnj/OmpdRqrOiZrb3qLbifBR4JeX3zBTgf4Av2z6+yfgiBtjIVLZexjsf/bE5cEnTQURXu4/x+iTK1rVdKJVB4w0nif54chKo7YdGn+yYBJqBSgMmyaxolKT3AZ8BZgNHUT7IZwDnUSax7QmcCny9mQiH3s0sPi2vm9cC1/U/lKA0FDdlwsr1HccTsXx/QooxzAbeI+lFtpf4+ZC0GaUa9ajaIwsAbM8AZkh6BqWSbr7t8ZL3EcPuN8C7JE3rVp1V/Ty9E+i1xTr64wZg/aaDiCXZPqHXeUnHUxJZmTTZjEwCbakks6JpH6ZsxXmz7SfKkBxursY6nyTpdMqH+8wGYxxmPwQ+IWln26eOPilpd0pVw7TaIxtOh1KSV/eOOo7B8zlgZ2COpIOB5wBIehmwFaVB/EOkv1njqhvyrlumImIJxwFHUz7bDqA0gH+iagC/FeWzb0PKdN2o1zeBQyRtYPvWpoOJibP9U0k/olzX/V3T8QybTAJtL9m5D4rmSJoPzLS9b3W8EDjM9vSONWcDz7Q94S1V8fSQtAZwBeVJ3/cpjQ+3o2zN2ZIy/eNGYNNUNEQsTtKbKYn4vx55iUVbdB4AdspkqeZIWg3YEdiE8tk2H7gSON32w03GFjGoVJ46fgd4D+XzbCEwD3gWsBzl8+0U27s0FuSQkrQRpXJkE+AIytapu+jy0CvJrsEj6QvAvrb/etzF0ReStmHxSaDzKVt3Mwl0QCWZFY2S9CjlA2JadfwI8A3b/9ax5gjKh3smSDRA0gaUZpTdkokXAu+1fXu9UQWApIOAm2x/u+lYortqOt4HWfLC6Hjb85qMbZhJ2pkyaW0Si/f3MSXRuI/t05qILaINJL2HMtlwdDL4ONuppm9A9UB45IFJrxs8287unAEj6Sxgy9zvRExcPsiiaXcC63Yc30rZttbpOcATtUUUT6oajD9oe6qkVwKvo+OG3PbljQYYnwa+2nQQMTbbD1B6MRzZdCxRVJMMZ1IqSk6k9J67izKVdRvgvcBMSQ/Y/klTcUYMsiphlaTVYDmRtB5oFUnLUXY/fAjYgdK3KWomaVfgKttX91jzCmAT25k2OUBSmRWNknQasIHt11THRwF7U572/YDSBP40yqjhNzYV57CStIAygvufmo4lliTpJmDWyDbdiBifpAspkwu3tH1Fl/OvBuYAl2V7e0RE/KU6qubGXALcB2xl+zf1RBUjqvfnYNuH9lgzDTjUdgYqDZBUZkXTzgKOljTZ9k3A54F/pEw0nFGteZxSgRL1u5c0Rh5kpwNvl/SMblOlojlVVeN4FgIPAjfk/avVJsDJ3RJZALYvk3QKsFO9YUW0R9Wf6QMsuc3wO9X1XNSsqi652/aspmOJJcyhezJrIXA/8EtK+4E/1BpVLI3lSeXjwEkyKxo1Mhq94/i2amT9fsDGwM3A0bavaSK+4Hxgi6aDiDFNpzTiP0PSfravbTqgeNL5TPyiZ4GkWcDHbV/Xv5Ci8ifKFvde7qjWRcQokvYDPgusyOI9594JfFrSgba/3Ehww+044GtAklkDxvbUpmOIv9gLKYnHGCDZZhgRY5L0AuBS4ChKae3jDYcUHSTNBVZiUd+5PwL3sGQSxbY3rjO2YSfpYGAzSg+M64GLgbuBtSkJ4hcC5wA3Uba8vY4yEWyzVDX0l6QzgEm9bi4kXQDMs71jbYFFtEDV+P27lJu6/2TJnnP/SqnUeq/tkxsKcyhJ+j3wfdsfbTqWiEEn6biOw92Aq6qv0ZYHNqA8PD7b9jv6H11MVJJZETGm6oP++cDrKTfiv6b7mGfb3rPm8IaepJuZYPWP7cn9jSY6Sdoc+BnwUcqEVnecE7AP8GVgG9uXStqN8lT9W7Y/1EDIQ0PSi4BfAMcCh9l+pOPcqpSKxz2BLVIpF7E4SZcBk4Eptm/pcn4ycDlwo+3N6o5vmEn6JvAa4FW2FzYdT3QnaX1Gbc+1fVuzUQ2fqk/WiJEpoGMx5eH++23P7WtgsVSSzIqIMY36oO/FaYgYsYikHwOP9nqCJ+lMYGXb21fHcygDMTaqJ8rhMOrp64jnUZ6yzgeuYFHV3BTKDcYcYG6S9BGLk/QocILtD/dY83XgA7b/qr7IQtLawCWUarn9bd/bbETRqdrtcDTwd11Ozwb+2fb19UY1vCRtOPItMJcyHbzb5OkFwP2dD75icKRnVkT0kmqeiKfmNZTeJb1cDXyk4/jK6u/F02u3Hucm0f3GYmtgK0qFVkQs8hDwwDhr7qcMt4h6zaQk6HcFdqmqt8eqpt+25tiGmqTnU9oNrAncCPycRdtz3wBsC/xc0ha2f9dYoEOks7JU0iHAz7pVm8ZgSzIrIsaUD/V2kbQGsFrK1QeCKNU/vYzuY/YEaTreD0nKRzx9zgO2Bw7sdrLaRv2mal3Ua2rH9ysDL6q+Rsu2nPp9jpLI+ihwVOc2UEnLUR5sfQU4HHh3IxEOMduHNB1DPDXZZhgR0WKSVgMOAd4H/A3liesK1bnXUvr/fNr2Fc1FOXwknUep7nmb7SVu6iS9GTiT8iRwZJvhD4EX2+528xER0ThJ61IqTC4BDhhV3bABcATwWkrPubuaiTJisEiaB1xs+6091pwDbG77WfVFFqNJWh5Yi5IQXoLtW+uNKHpJZVZEREtJWp1Sqv4yygSWe4GXdCy5htIX6D2UvkBRn2mUvkvnSpoNXMSivkxvoEz9+hPwaXjyvdwO+E4j0UZEdFF9fo32AKV65B8k3cqiz7YNKJO/rqZMPMxWtohiJbpPyut0JeWaLRog6RXA5ynXZ10TWZSqxuRPBkjejIiI9ppGSWTtZvtESdOBg0ZO2n5U0gXkhqJ2tn8laXvgW5R//21ZfFrOjcBetn9VHf+ZMt3o7rpjHVbV1o7nAusBK3ZbY3tOrUFFDJ6pPc6tQNlOPXpL9d+SrWwRnX5NmQ7ey/MpieComaSXUCpOAX4MvI3ynt1NGQyzFmVCdaqyBkySWRER7fUuYJbtE3usuQXIePQG2J4j6YXAFiwaw/0g5enrRe7Y52/7MeC6RgIdQpL2Bz5OuUDtJVNaY6jZXq7pGGLpSVqPkqwfa6tUEvX1Ohw4XdIOts8dfVLS3wM7Au+sPbKAUiW/IrCZ7Wuqae6n2z5U0qrAfwJvofdAmWhAklkREe21HvD9cdY8TEmiRAOqhNVF1VcMAEkHUyoY7wNOAG6nNN+PiGg1SW+iNBJ/8ThLk6jvI0m7dnn5XOAsST+ltCEY2Z67NWWq7n8z/gOW6I+pwFm2r+l4TQC2H5G0D6Vq7jCS0BooSWZFRLTXQ8Czx1kzmdJLKxpQNUvelrGfkNv2YfVGNfT2BOYCm9qe33QwEW0iaQFwku33NR1LLE7S5sBZwB+A/0+ZkHcBpep3S0pPzTMp1cHRXzNYcqvtSJuBN1Zfo72dsr2tV7V99MdawA0dx08AfzVyYPsJST+jVM/FAEkyKyKivX4FvFXSM20/NPpklUh5C+XiNmom6RDgABb/XSsWXeCOfJ9kVr3WBI5JIiviKXmI9I0ZVAcCf6RslbpD0kcoE3MPlSTK5ON/p/TbjP7avekAYqnMA1brOL6XMtCi05/JToeBk2RWRER7HUkpWz9H0t6dJ6pmlt8AVqHs9Y8aSXof8BlgNnAUZTvoDOA8Sjn7nsCpwNebiXCo/Q5Yo+kgIlrqSuClTQcRXb0OONP2HR2vLQdPbnk/SNIOlKTWTg3ENzRsn9B0DLFUbgQ26ji+HNhO0rNt31P1zXoHcFMTwcXY0tQxIqKlbM+iXJS+HriW8lQWSfdWx1sAB9q+eMz/SPTLh4HfA2+2fXr12s22T7K9L/BWymj7v24qwCF2NKWicZ2mA4looSOAt0jarulAYgmrs3jV3J+BVUetuQjYqraIItrhPGCbKmkFcAzwLOBKSacC1wAbAt9sKL4YQyqzIiJazPYhkuYA/wpsTtlCZeAc4Cu2ZzcZ3xB7BTDTdmdj8Scb7tqeJWkWsD+l6WvUxPYx1ZTJiyQdClwBdN1yaDvbqSIW92zgR8C5ks6gbHe/iyX7AzHOpN14+t3D4lWn9wAbj1qzIvCM2iKKxUj6G+AfKP3LVrW9V8frk4FrqunGUa9vUHrLPQN4xPbZkj4GTKe8X49SEvnZ6TBg1DEZPCIiIp4Gkh6lJBOnVcePAN+w/W8da44A9rWdHgw1k/RBysSvXv/2tp2HfhEdqpH1ZlEz6xGdNxSi/PxkYl6Nqil5C2y/qTo+BdiBMuzi+qoa9SrgDttTGgx1KEnak5IMWYVRPyOSXg78Gtjb9reaizI6SVqe0hz+HidpMpBykRYREfH0uxNYt+P4VuCVo9Y8hzIxJ2okaS9Kr7IngPOBO8j7EDFRaWw9uH4E/IekZ9meR+mr+S7KVqn/BV4APBP4RIMxDqVqW+6xwNWUap/tgX1Hztu+VtL/AO8EkswaELYXAHc3HUeMLcmsiIiWk7QR8AFgE0qlyXxKk97v2E6zymZcCby843g2sLekDwA/oDSB34nSvyTq9XHK9pst8vMRsXTS2HqgfR2YAzwOYPsiSTtTJua+HLgZ+ES2fzbik5SHXFvbflDSJl3WXE1p4h8NkLQe8DHgVcB6lC25o9n26K270aBsM4yIaDFJ+wGfpfzSHb3t43FKA/gv1x7YkJO0G6XR+Mts3yRpfUqCq7OfyePAVNuXNBDi0JL0GPBN2x9pOpaIiFj2SXoAGBkAg6TpwEGdW3ElfR74iO3RTfujzyRNpfSaXYVSqX03Y1Rs255cX2QxnlRmRUS0lKT3AF8E7qf0YTif0oh3HWAbSlP4L0q63fbJTcU5jGzPAGZ0HN8maTNgP0pD3puBo21f00R8Q+52uj9xjYhoPUmrATuyeLX2FcAZth9uMrYhthLwyDhrJgELaogllvQFypCeXYHv2V7YcDwxQanMiohoKUmXUabfTLF9S5fzk4HLgRttb1Z3fBGDSNJBwF6UqrmHmo4nok0kzZ3g0mzHaUC1rfAYSmKks1rbwAPAPrZPayK2YVb1LLvH9tTquFtl1pXAE7leq19VsX2y7d2ajiWWznJNBxAREU/ZS4FTuiWyAKp+QKcCL6s1qojBdjjwS+AnkqZKembTAUW0yHKUJMnorzWAjaqvlcg9Ru2qJuMzKU3eT6Q069+h+vPb1eszJb2xsSCH1w+BLatk4xIk7U4ZEvP9WqOKEfcD85oOIpZeKrMiIlpK0t3AcbYP7LHm88ButtepL7KIwSVpZBuHKNUKY7HttGOImCBJz6dseV8V2N72HxsOaahIuhCYAmxp+4ou519NaRB/me2t6o5vmElag7LVc31Kwmp1YDvgo8CWlKmTNwKb2h5vO2I8zSQdS/m337TpWGLpJJkVEdFSkr5N2So1ZYzzomwzvNb2rrUGFzGgJJ1P7yTWk2xv099oIpYtklYBrgVO7fWgJZ5+kh6mVGvv0WPNDGAn26vVFlgAIGkDSsVct0TihcB7bd9eb1QBIGkt4BJgFmXiZxKKLZFkVkRES0laF7iY8gv4gM7thtVF0xHAa4EtbN/VTJQRETFMJP0XsIPtjZqOZZhIug84xva0HmsOp/TNWrO+yKKTpFcCrwPWpDTnv8T25c1GFVVl6aWUbdLXU96b0Wx721oDi56SzIqIaClJsyl9Sl5JmYBzK2Wc8NrABpTJLFdTegF0yi/jiIjoC0lfAz5kLxZUawAACp5JREFUe5WmYxkmks4AJo00GR9jzQXAPNs71hZYxICT9DLgZ8Ba4yx1Z9P+aF6SWRERLSXpqY4Ozi/jGHpVZeO2wHOBlbssse3D6o0qot2q7TpXAY/ZfkHT8QwTSS8CfgEcCxzWuVVK0qrAdGBPSrX2dc1EGTF4JM0C3kj5GTkBuMP2gt5/KwZBklkRERExVCQdAhwAdDZ472wIL5L0jViCpIPGOLUCpbn1OyjNrQ+0/YXaAhtCko7r8vLzKA3F51Majo9Ua0+hvC9zgLm296wrzmHU4+dkPHmI0gBJDwKzbHedNhmDK8msiIiIGBqS3kcZUz8bOIoyWWoGcB4wlVK5cCrwddsXNBNlxGCaQEXwg8CRtqfXEc8wS3X24Mp70y6S7gWOt71/07HE0snI6YiIiBgmHwZ+D7zZ9hNl6Cc32z4JOEnS6cDZwMwGY4wYVGNN+FxI6c/4W9tP1BjPMJvcdAAxpkzCbZfzgdc0HUQsvVRmRUS0XNWj5CXAesCK3dbYPrHWoCIGlKT5wEzb+1bHCyn9ZaZ3rDkbeKbtbiPUIyIiYhkh6XmUSYZfAo5wEiStkcqsiIiWkrQK5RfvHpRRwl2XUfoAJZkVUawI3Ndx/Bill0yna4F9a4soIiKWWZJ2Ba6yfXWPNS8HpuThYyM+Tfm9/1ngQ5KuovSdG83pNzdYksyKiGivL1K2TP0GOBm4Hcj2joje7gTW7Ti+FXjlqDXPIT9LEWOStDmwF7AJMIly43c5pe/MxU3GFjGAZgAHA2MmsyjDEw4lDx+bsFvH95MZewuvKX01Y0AkmRUR0V7vplwYbWb78aaDiWiJK4GXdxzPBvaW9AHgB5Qm8DsBF9UfWsTgk/QfwIGUyt9OrwL2kHSE7U/VH1lEqy3Poom6Ua/0n2up5ZoOICIinrJVgR8nkRWxVM4CXi5p5OL185SqkhmUSWxnUm7SP91IdBEDTNLOwKcoFY17Ac8DnlH9uVf1+iclvbuxICPa6YWUIQpRM9u3TPSr6VhjcWkAHxHRUpIuBW6w/f6mY4losyqxtR+wMXAzcLTtaxoNKmIASZoDvAB4he17u5xfi9J75jrbW9cdX8SgkHRcx+FuwFXV12jLAxsAWwJn235H/6OLWDYkmRUR0VLVE/IZwCa2r284nIiIWMZV00BPtP2RHmu+Buxqe/RghYihUU3KHWGW3JbLqPOXAu+3PbevgUUsQ9IzKyKipWyfKmld4EJJRwNX0H36Crbn1BpcREQsi1YAHh1nzaPkHiNiZCu7gLnAV4Eju6xbANxv+5G6AotYVqQyKyKixSRNB/an9CwZk+3l64koIiKWVZKupmyLeoXthV3OLwf8mjLCfvSU0IihVF2r/SwPFiOeXnlqEhHRUpIOBKYD9wEnA3cATzQaVERELMu+BxwO/FDSv9u+YeSEpI2BLwIvBaY1FF/EIBq3ekTSlsA2tg+tIZ6IZUIqsyIiWkrSTZTy9E1td91eGBER8XSRtBJwHrAVsJDyEOVOYB3guZRJ6T8H3mj7z03FGTFIqv5ZB/dKVEmaBhyaSvqIiVuu6QAiIuIpWwc4M4msiIioQ5Wg2o5SeXUTsB6wGbB+dTwN2DaJrIiltiIlQRwRE5RthhER7TUXmNR0EBERMTxsPw58DvicpNWA1YH5th9uNrKIVpsC3Nt0EBFtkmRWRER7/RcwXdI6tu9qOpiIiBgOVRJrR2ATqmSWpCuAM5LUigBJs0e9tJukqV2WLk+pbNwQmNnvuCKWJemZFRHRUpI2BL5EeZp3KHA50HXLoe1bawwtIiKWUZJ2Bo6hVAar45SBB4B9bJ/WRGwRg6LqkzXCLP6z0mkhZZDPT4GP2v5Dv2OLWFYkmRUR0VLVhdLIBVKvD3PbTiVuRET8RSRtB5xLuQH/LnA+cBelh+M2wHspv5N2sP2ThsKMGCgTaQAfEUsvyayIiJaSNIMJjHsGsL17f6OJiIhlnaQLKdXAW9q+osv5VwNzgMtsb1V3fBGDSNIHgSttX910LBHLkiSzIiIiIiJiXJIeBk6xvUePNTOAnWyvVltgERExdLLtJCIiIiIiJuJPwJ3jrLmjWhcRgKQJVynantPPWCKWJUlmRUS0hKTjKNsKP2X77up4Imx7zz6GFhERw+FC4PXjrHk9ZathRBTnM8G2EJTphhExAdlmGBHREh0N319i+/pRk3J6se1cHEVExF9E0ouAXwDHAofZfqTj3KrAdGBPYAvb1zUTZcRgkXQw3ZNZk4DNgC2A/wausH1IjaFFtFqSWRERLSFpw+rb220/0XE8Ltu39CmsiIhYRo1RAfw8YEtgPnAFcDewNqUx/OqUqqy5qQiOmBhJuwFfA15n+9qGw4lojSSzIiIiIiJiCUtRATxaKoIjloKkHwOP2X5707FEtEV6ZkVERERERDeTmw4gYkhcBXyo6SAi2iTJrIiIiIiIWEK2qEfUZn1ybx6xVJZrOoCIiIiIiIiIYSNpeUl7ATsBlzUdT0SbJPsbERERERER0QeS5o5xagXK8IQVgD8Dn6otqIhlQJJZEREREREREf2xHNBt6trjwDXAL4Gv2f5NrVFFtFymGUZERERERERERGukZ1ZERERERERERLRGklkRERERERERfSDpGZI2kLTSGOdXrs6vUndsEW2WZFZEREREREREfxwEXAesNsb5VYHfkgbwEUslPbMiIiIiIiIi+kDSVcBttt/WY82ZwHq2p9QXWUS7pTIrIiIiIiIioj82Aq4fZ8311bqImKAksyIiIiIiIiL6Y0Vg4ThrDKRnVsRSSDIrIiIiIiIioj/mAluPs2YqcEv/Q4lYdiSZFREREREREdEfZwKbSvpEt5OSDgCmAGfUGlVEy6UBfEREREREREQfSFoDuBJYv/rzPOB24LnA9sCrgFuBKbbvbyrOiLZJMisiIiIiIiKiTyRtBHwP2Lx6yYCq7y8G3m/75toDi2ixJLMiIiIiIiIi+kzSFEpCaxLwAHCJ7SuajSqinZLMioiIiIiIiKiRpDWBrYBHgZ/YXtBwSBGtkgbwEREREREREX0g6cOSLpX0rI7XNgV+C5wGnANcLGnVpmKMaKMksyIiIiIiIiL64x8B257X8doXgTWA4ynJrM2AfRuILaK1ss0wIiIiIiIiog8k3Q6cbXvv6ngt4G7gm7b3qV67FFjB9qbNRRrRLqnMioiIiIiIiOiPNYF7Oo5fX/15esdrFwIb1hZRxDIgyayIiIiIiIiI/pgHrNVxvDWwELi44zUDq9QZVETbJZkVERERERER0R+/Ad4maU1Jk4BdgF/ZfrBjzUbAXU0EF9FWSWZFRERERERE9MeRwLrA74HbgLWBo0et2Rz4dc1xRbTaCk0HEBEREREREbEssn2mpH2BvauXvmv7OyPnJU0FVgNmNRBeRGtlmmFERERERERERLRGthlGRERERERERERrJJkVERERERERERGtkWRWRERERERERES0RpJZERERERERERHRGklmRUREREREREREaySZFRERERERERERrZFkVkREREREREREtEaSWRERERERERER0RpJZkVERERERERERGskmRUREREREREREa2RZFZERERERERERLRGklkREREREREREdEa/wfzVnPS014awwAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 720x360 with 1 Axes>"
      ]
     },
     "metadata": {
      "image/png": {
       "height": 347,
       "width": 601
      },
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plotMostFrequentTokens(flipped_pos1,('Beer_Name',),10,'img1.png')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-02-25T09:52:23.163474Z",
     "start_time": "2020-02-25T09:52:22.483547Z"
    }
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAABKAAAAKoCAYAAABA2ofrAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAAWJQAAFiUBSVIk8AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8li6FKAAAgAElEQVR4nOzdeZSld13n8c+XtBIWEyCDMKBMwpJlYAQSFAQmJESRRRYljPGoLAO4gQxLOEQBaVzGuCKCwgygkeAxMCI4EIRxCBAgqEMAMYcAZpIGYWQJTRKWgCR85497i1QqVd2d7vu7z03363VOnafu8zz33u85z6lO1TvP89zq7gAAAADAKDeaegAAAAAA9m8CFAAAAABDCVAAAAAADCVAAQAAADCUAAUAAADAUAIUAAAAAEMJUAAAAAAMJUABAAAAMJQABQAAAMBQAhQAAAAAQwlQAAAAAAwlQAEAAAAw1LapBxihqi5JckiSHROPAgAAALC/ODzJFd19xPV94n4ZoJIccpOb3ORWxxxzzK2mHgQAAABgf3DhhRfmyiuv3Kvn7q8Bascxxxxzq/PPP3/qOQAAAAD2C8cdd1w+8IEP7Nib5y7kHlBV9ZtV9faq+uequrKqdlbVB6vqBVV12BbPuW9VvWW+75VV9eGqenpVHbSImQAAAABYDYu6Cfkzktwsyd8keXGSP0tyVZLtST5cVd+9fueqemSSc5Mcn+QNSV6a5NuTvCjJWQuaCQAAAIAVsKhL8A7p7q9tXFlVv57kl5L8YpKfn687JMkrklyd5ITufv98/fOTnJPk5Ko6pbuFKAAAAID9wELOgNosPs29br68y7p1Jye5dZKz1uLTutd43vzhzy1iLgAAAACmt6hL8Lby8Pnyw+vWPXC+fOsm+5+b5KtJ7ltVNx45GAAAAADLsdBPwauqU5PcPMmhSe6V5P6ZxafT1+121Hz58Y3P7+6rquqSJHdNcsckF+7m/bb6mLujr9/kAAAAAIyy0ACV5NQkt1n3+K1JHt/dn1+37tD58vItXmNt/S0WPBsAAAAAE1hogOru2yZJVd0myX0zO/Ppg1X1w939gUW+1/z9jtts/fzMqGMX/X4AAAAAXH9D7gHV3Z/t7jckeVCSw5K8et3mtTOcDr3OE6+9/rIRswEAAACwXENvQt7dn0jykSR3rap/M1/9sfnyyI37V9W2JEckuSrJxSNnAwAAAGA5Rn8KXpLcbr68er48Z7588Cb7Hp/kpknO6+6vjx4MAAAAgPH2OUBV1ZFVdZ3L6arqRlX160m+M7Og9MX5pr9IcmmSU6rqXuv2PzjJr80fvmxf5wIAAABgNSziJuQPTfIbVfWeJJck+UJmn4T3gCR3TPKZJE9e27m7r6iqJ2cWot5ZVWcl2ZnkEUmOmq9/7QLmAgAAAGAFLCJA/e8kd05y/yT3THKLJF9J8vEkZyb5g+7euf4J3f3GqnpAkucmeXSSg5NclOSZ8/17AXMBAAAAsAL2OUB19wVJnroXz3tvZmdPAQAAALAfW8ZNyAEAAAA4gAlQAAAAAAwlQAEAAAAwlAAFAAAAwFACFAAAAABDCVAAAAAADCVAAQAAADCUAAUAAADAUAIUAAAAAENtm3oAdu/w086eeoT9xo7TH7bQ13NsFmfRxwYAAIDV4QwoAAAAAIYSoAAAAAAYSoACAAAAYCgBCgAAAIChBCgAAAAAhhKgAAAAABhKgAIAAABgKAEKAAAAgKEEKAAAAACGEqAAAAAAGEqAAgAAAGAoAQoAAACAoQQoAAAAAIYSoAAAAAAYSoACAAAAYCgBCgAAAIChBCgAAAAAhhKgAAAAABhKgAIAAABgKAEKAAAAgKEEKAAAAACGEqAAAAAAGEqAAgAAAGAoAQoAAACAoQQoAAAAAIYSoAAAAAAYSoACAAAAYCgBCgAAAIChBCgAAAAAhhKgAAAAABhKgAIAAABgKAEKAAAAgKEEKAAAAACGEqAAAAAAGEqAAgAAAGAoAQoAAACAoQQoAAAAAIYSoAAAAAAYSoACAAAAYCgBCgAAAIChBCgAAAAAhhKgAAAAABhKgAIAAABgKAEKAAAAgKEEKAAAAACGEqAAAAAAGEqAAgAAAGAoAQoAAACAoQQoAAAAAIYSoAAAAAAYSoACAAAAYCgBCgAAAIChBCgAAAAAhhKgAAAAABhKgAIAAABgKAEKAAAAgKEEKAAAAACGEqAAAAAAGEqAAgAAAGAoAQoAAACAoQQoAAAAAIYSoAAAAAAYSoACAAAAYCgBCgAAAICh9jlAVdVhVfWkqnpDVV1UVVdW1eVV9Z6qemJV3WjD/odXVe/i66x9nQkAAACA1bFtAa/xmCQvS/IvSd6R5JNJbpPkR5O8MslDquox3d0bnvcPSd64yetdsICZAAAAAFgRiwhQH0/yiCRnd/c311ZW1S8l+fskj84sRr1+w/M+1N3bF/D+AAAAAKywfb4Er7vP6e43rY9P8/WfSfLy+cMT9vV9AAAAALhhWsQZULvyjfnyqk223a6qfibJYUm+kOR93f3hwfMAAAAAsGTDAlRVbUvy2PnDt26yyw/Ov9Y/551JHtfdn9zD9zh/i01H7+GYAAAAAAy2z5fg7cLpSe6W5C3d/bZ167+a5FeTHJfklvOvB2R2A/MTkry9qm42cC4AAAAAlmjIGVBV9bQkz0ry0SQ/tX5bd38uyS9veMq5VfWgJO9Jcu8kT0ry4t29T3cft8X7n5/k2Os/OQAAAACLtvAzoKrqqZnFo48kObG7d+7J87r7qiSvnD88ftFzAQAAADCNhQaoqnp6kpckuSCz+PSZ6/kSn58vXYIHAAAAsJ9YWICqquckeVGSD2UWnz63Fy9zn/ny4kXNBQAAAMC0FhKgqur5md10/PwkJ3X3pbvY99iqus77VtVJSZ4xf/iaRcwFAAAAwPT2+SbkVfW4JL+S5Ook707ytKrauNuO7j5j/v3vJblLVZ2X5FPzdd+T5IHz75/f3eft61wAAAAArIZFfAreEfPlQUmevsU+70pyxvz7M5P8SJLvTfKQJN+W5LNJXpfkpd397gXMBAAAAMCK2OcA1d3bk2y/Hvu/Ksmr9vV9AQAAALhhWOin4AEAAADARgIUAAAAAEMJUAAAAAAMJUABAAAAMJQABQAAAMBQAhQAAAAAQwlQAAAAAAwlQAEAAAAwlAAFAAAAwFACFAAAAABDCVAAAAAADCVAAQAAADCUAAUAAADAUAIUAAAAAEMJUAAAAAAMJUABAAAAMJQABQAAAMBQAhQAAAAAQwlQAAAAAAwlQAEAAAAwlAAFAAAAwFACFAAAAABDCVAAAAAADCVAAQAAADCUAAUAAADAUAIUAAAAAEMJUAAAAAAMJUABAAAAMJQABQAAAMBQAhQAAAAAQwlQAAAAAAwlQAEAAAAwlAAFAAAAwFACFAAAAABDCVAAAAAADCVAAQAAADCUAAUAAADAUAIUAAAAAEMJUAAAAAAMJUABAAAAMJQABQAAAMBQAhQAAAAAQwlQAAAAAAwlQAEAAAAwlAAFAAAAwFACFAAAAABDCVAAAAAADCVAAQAAADCUAAUAAADAUAIUAAAAAEMJUAAAAAAMJUABAAAAMJQABQAAAMBQAhQAAAAAQwlQAAAAAAwlQAEAAAAwlAAFAAAAwFACFAAAAABDCVAAAAAADCVAAQAAADCUAAUAAADAUAIUAAAAAEMJUAAAAAAMJUABAAAAMJQABQAAAMBQAhQAAAAAQwlQAAAAAAwlQAEAAAAwlAAFAAAAwFACFAAAAABDCVAAAAAADCVAAQAAADCUAAUAAADAUAIUAAAAAEPtc4CqqsOq6klV9Yaquqiqrqyqy6vqPVX1xKra9D2q6r5V9Zaq2jl/zoer6ulVddC+zgQAAADA6ti2gNd4TJKXJfmXJO9I8skkt0nyo0lemeQhVfWY7u61J1TVI5O8PsnXkrw2yc4kD0/yoiT3m78mAAAAAPuBRQSojyd5RJKzu/ubayur6peS/H2SR2cWo14/X39IklckuTrJCd39/vn65yc5J8nJVXVKd5+1gNkAAAAAmNg+X4LX3ed095vWx6f5+s8kefn84QnrNp2c5NZJzlqLT/P9v5bkefOHP7evcwEAAACwGkbfhPwb8+VV69Y9cL586yb7n5vkq0nuW1U3HjkYAAAAAMuxiEvwNlVV25I8dv5wfWw6ar78+MbndPdVVXVJkrsmuWOSC3fzHudvseno6zctAAAAAKOMPAPq9CR3S/KW7n7buvWHzpeXb/G8tfW3GDUYAAAAAMsz5Ayoqnpakmcl+WiSnxrxHknS3cdt8f7nJzl21PsCAAAAsOcWfgZUVT01yYuTfCTJid29c8Mua2c4HZrNra2/bNGzAQAAALB8Cw1QVfX0JC9JckFm8ekzm+z2sfnyyE2evy3JEZndtPziRc4GAAAAwDQWFqCq6jlJXpTkQ5nFp89tses58+WDN9l2fJKbJjmvu7++qNkAAAAAmM5CAlRVPT+zm46fn+Sk7r50F7v/RZJLk5xSVfda9xoHJ/m1+cOXLWIuAAAAAKa3zzchr6rHJfmVJFcneXeSp1XVxt12dPcZSdLdV1TVkzMLUe+sqrOS7EzyiCRHzde/dl/nAgAAAGA1LOJT8I6YLw9K8vQt9nlXkjPWHnT3G6vqAUmem+TRSQ5OclGSZyb5g+7uBcwFAAAAwArY5wDV3duTbN+L5703yUP39f0BAAAAWG0L/RQ8AAAAANhIgAIAAABgKAEKAAAAgKEEKAAAAACGEqAAAAAAGEqAAgAAAGAoAQoAAACAoQQoAAAAAIYSoAAAAAAYSoACAAAAYCgBCgAAAIChBCgAAAAAhhKgAAAAABhKgAIAAABgKAEKAAAAgKEEKAAAAACGEqAAAAAAGEqAAgAAAGAoAQoAAACAoQQoAAAAAIYSoAAAAAAYSoACAAAAYCgBCgAAAIChBCgAAAAAhhKgAAAAABhKgAIAAABgKAEKAAAAgKEEKAAAAACGEqAAAAAAGEqAAgAAAGAoAQoAAACAoQQoAAAAAIYSoAAAAAAYSoACAAAAYCgBCgAAAIChBCgAAAAAhhKgAAAAABhKgAIAAABgKAEKAAAAgKEEKAAAAACGEqAAAAAAGEqAAgAAAGAoAQoAAACAoQQoAAAAAIYSoAAAAAAYSoACAAAAYCgBCgAAAIChBCgAAAAAhhKgAAAAABhKgAIAAABgKAEKAAAAgKEEKAAAAACGEqAAAAAAGEqAAgAAAGAoAQoAAACAoQQoAAAAAIYSoAAAAAAYSoACAAAAYCgBCgAAAIChBCgAAAAAhhKgAAAAABhKgAIAAABgKAEKAAAAgKEEKAAAAACGEqAAAAAAGEqAAgAAAGAoAQoAAACAoQQoAAAAAIYSoAAAAAAYSoACAAAAYCgBCgAAAIChBCgAAAAAhhKgAAAAABhKgAIAAABgKAEKAAAAgKEWEqCq6uSqeklVvbuqrqiqrqrXbLHv4fPtW32dtYiZAAAAAFgN2xb0Os9LcvckX07yqSRH78Fz/iHJGzdZf8GCZgIAAABgBSwqQD0js/B0UZIHJHnHHjznQ929fUHvDwAAAMCKWkiA6u5vBaeqWsRLAgAAALCfWNQZUHvjdlX1M0kOS/KFJO/r7g9fnxeoqvO32LQnlwACAAAAsARTBqgfnH99S1W9M8njuvuTk0wEAAAAwMJNEaC+muRXM7sB+cXzdd+TZHuSE5O8varu0d1f2d0Ldfdxm62fnxl17EKmBQAAAGCf3GjZb9jdn+vuX+7uD3T3ZfOvc5M8KMnfJblzkictey4AAAAAxlh6gNpKd1+V5JXzh8dPOQsAAAAAi7MyAWru8/PlzSadAgAAAICFWbUAdZ/58uJd7gUAAADADcbSA1RVHVtV13nfqjopyTPmD1+z3KkAAAAAGGUhn4JXVY9K8qj5w9vOl99fVWfMv7+0u0+df/97Se5SVecl+dR83fckeeD8++d393mLmAsAAACA6S0kQCW5R5LHbVh3x/lXknwiyVqAOjPJjyT53iQPSfJtST6b5HVJXtrd717QTAAAAACsgIUEqO7enmT7Hu77qiSvWsT7AgAAALD6Vu0m5AAAAADsZwQoAAAAAIYSoAAAAAAYSoACAAAAYCgBCgAAAIChBCgAAAAAhhKgAAAAABhKgAIAAABgKAEKAAAAgKEEKAAAAACGEqAAAAAAGEqAAgAAAGAoAQoAAACAoQQoAAAAAIYSoAAAAAAYSoACAAAAYCgBCgAAAIChBCgAAAAAhhKgAAAAABhKgAIAAABgKAEKAAAAgKEEKAAAAACGEqAAAAAAGEqAAgAAAGAoAQoAAACAoQQoAAAAAIYSoAAAAAAYSoACAAAAYCgBCgAAAIChBCgAAAAAhhKgAAAAABhKgAIAAABgKAEKAAAAgKEEKAAAAACGEqAAAAAAGEqAAgAAAGAoAQoAAACAoQQoAAAAAIYSoAAAAAAYSoACAAAAYCgBCgAAAIChBCgAAAAAhhKgAAAAABhKgAIAAABgqG1TDwAwyuGnnT31CPuNHac/bOoRAACAGzBnQAEAAAAwlAAFAAAAwFACFAAAAABDCVAAAAAADCVAAQAAADCUAAUAAADAUAIUAAAAAEMJUAAAAAAMJUABAAAAMJQABQAAAMBQAhQAAAAAQwlQAAAAAAwlQAEAAAAwlAAFAAAAwFACFAAAAABDCVAAAAAADCVAAQAAADCUAAUAAADAUAIUAAAAAEMJUAAAAAAMJUABAAAAMJQABQAAAMBQAhQAAAAAQwlQAAAAAAwlQAEAAAAwlAAFAAAAwFACFAAAAABDCVAAAAAADCVAAQAAADDUQgJUVZ1cVS+pqndX1RVV1VX1mt08575V9Zaq2llVV1bVh6vq6VV10CJmAgAAAGA1bFvQ6zwvyd2TfDnJp5Icvaudq+qRSV6f5GtJXptkZ5KHJ3lRkvslecyC5gIAAABgYou6BO8ZSY5MckiSn9vVjlV1SJJXJLk6yQnd/cTufnaSeyR5X5KTq+qUBc0FAAAAwMQWEqC6+x3d/U/d3Xuw+8lJbp3krO5+/7rX+FpmZ1Ilu4lYAAAAANxwTHET8gfOl2/dZNu5Sb6a5L5VdePljQQAAADAKIu6B9T1cdR8+fGNG7r7qqq6JMldk9wxyYW7eqGqOn+LTbu8BxUAAAAAyzPFGVCHzpeXb7F9bf0tljALAAAAAINNcQbUwnT3cZutn58ZdeySxwEAAABgE1OcAbV2htOhW2xfW3/ZEmYBAAAAYLApAtTH5ssjN26oqm1JjkhyVZKLlzkUAAAAAGNMEaDOmS8fvMm245PcNMl53f315Y0EAAAAwChTBKi/SHJpklOq6l5rK6vq4CS/Nn/4sgnmAgAAAGCAhdyEvKoeleRR84e3nS+/v6rOmH9/aXefmiTdfUVVPTmzEPXOqjoryc4kj0hy1Hz9axcxFwAAAADTW9Sn4N0jyeM2rLvj/CtJPpHk1LUN3f3GqnpAkucmeXSSg5NclOSZSf6gu3tBcwEAAAAwsYUEqO7enmT79XzOe5M8dBHvDwAAAMDqmuIeUAAAAAAcQAQoAAAAAIYSoAAAAAAYSoACAAAAYCgBCgAAAIChBCgAAAAAhhKgAAAAABhKgAIAAABgKAEKAAAAgKG2TT0AAAeew087e+oR9hs7Tn/Ywl/T8VmcEccHAOCGyBlQAAAAAAwlQAEAAAAwlAAFAAAAwFACFAAAAABDCVAAAAAADCVAAQAAADCUAAUAAADAUAIUAAAAAEMJUAAAAAAMJUABAAAAMJQABQAAAMBQAhQAAAAAQwlQAAAAAAwlQAEAAAAwlAAFAAAAwFACFAAAAABDbZt6AAAA9szhp5099Qj7jR2nP2zqEQDggOIMKAAAAACGEqAAAAAAGEqAAgAAAGAoAQoAAACAoQQoAAAAAIYSoAAAAAAYSoACAAAAYCgBCgAAAIChBCgAAAAAhhKgAAAAABhKgAIAAABgKAEKAAAAgKEEKAAAAACGEqAAAAAAGEqAAgAAAGAoAQoAAACAoQQoAAAAAIbaNvUAAACwPzj8tLOnHmG/seP0h009AgAL5gwoAAAAAIYSoAAAAAAYSoACAAAAYCgBCgAAAIChBCgAAAAAhhKgAAAAABhKgAIAAABgKAEKAAAAgKEEKAAAAACGEqAAAAAAGEqAAgAAAGAoAQoAAACAoQQoAAAAAIYSoAAAAAAYSoACAAAAYCgBCgAAAIChBCgAAAAAhto29QAAAAAjHX7a2VOPsN/YcfrDph4BuIFyBhQAAAAAQwlQAAAAAAwlQAEAAAAwlAAFAAAAwFACFAAAAABDCVAAAAAADCVAAQAAADCUAAUAAADAUAIUAAAAAEMJUAAAAAAMJUABAAAAMNRkAaqqdlRVb/H1manmAgAAAGCxtk38/pcn+f1N1n952YMAAAAAMMbUAeqy7t4+8QwAAAAADOQeUAAAAAAMNfUZUDeuqp9McockX0ny4STndvfV044FAAAAwKJMHaBum+TMDesuqaondPe7dvfkqjp/i01H7/NkAAAAACzElJfg/UmSkzKLUDdL8h+S/Lckhyf566q6+3SjAQAAALAok50B1d0v3LDqgiQ/W1VfTvKsJNuT/MhuXuO4zdbPz4w6dgFjAgAAALCPVvEm5C+fL4+fdAoAAAAAFmIVA9Tn58ubTToFAAAAAAuxigHqPvPlxZNOAQAAAMBCTBKgquqYqrrOGU5VdXiSl84fvmaZMwEAAAAwxlQ3If+xJM+qqnOTfCLJl5LcKcnDkhyc5C1Jfmei2QAAAABYoKkC1DuSHJXknknul9n9ni5L8p4kZyY5s7t7otkAAAAAWKBJAlR3vyvJu6Z4bwAAAACWaxVvQg4AAADAfkSAAgAAAGAoAQoAAACAoQQoAAAAAIYSoAAAAAAYSoACAAAAYCgBCgAAAIChBCgAAAAAhhKgAAAAABhKgAIAAABgKAEKAAAAgKEEKAAAAACGEqAAAAAAGEqAAgAAAGAoAQoAAACAoQQoAAAAAIYSoAAAAAAYSoACAAAAYCgBCgAAAIChBCgAAAAAhhKgAAAAABhKgAIAAABgKAEKAAAAgKEEKAAAAACGEqAAAAAAGEqAAgAAAGAoAQoAAACAobZNPQAAAAAHrsNPO3vqEfYbO05/2EJfz7FZnEUfmxsiZ0ABAAAAMJQABQAAAMBQAhQAAAAAQwlQAAAAAAwlQAEAAAAwlAAFAAAAwFACFAAAAABDCVAAAAAADCVAAQAAADCUAAUAAADAUAIUAAAAAEMJUAAAAAAMJUABAAAAMJQABQAAAMBQAhQAAAAAQwlQAAAAAAwlQAEAAAAwlAAFAAAAwFACFAAAAABDCVAAAAAADCVAAQAAADCUAAUAAADAUAIUAAAAAEMJUAAAAAAMJUABAAAAMJQABQAAAMBQAhQAAAAAQwlQAAAAAAwlQAEAAAAwlAAFAAAAwFACFAAAAABDCVAAAAAADCVAAQAAADCUAAUAAADAUAIUAAAAAEMJUAAAAAAMJUABAAAAMJQABQAAAMBQAhQAAAAAQwlQAAAAAAwlQAEAAAAwlAAFAAAAwFACFAAAAABDCVAAAAAADCVAAQAAADCUAAUAAADAUAIUAAAAAENNGqCq6ruq6o+r6v9V1derakdV/X5V3XLKuQAAAABYnG1TvXFV3SnJeUm+M8lfJfloku9L8l+SPLiq7tfdX5hqPgAAAAAWY8ozoP4os/j0tO5+VHef1t0PTPKiJEcl+fUJZwMAAABgQSYJUPOznx6UZEeSP9yw+QVJvpLkp6rqZkseDQAAAIAFq+5e/ptWPSnJK5L89+7+mU22vy2zQPUD3f32XbzO+VtsuvtNbnKTg4455piFzDu1Cz59+dQj7DfudvtDF/p6js3iLPrYJI7PIvnZWV1+dlabn53V5WdntfnZWV1+dlabn53VNeJnZwoXXnhhrrzyyp3dfdj1fe5UAeq3k5ya5NTu/t1Ntr80yVOS/Hx3v2wXr7NVgLpbki9ndoYV4x09X3500inYiuOzuhyb1eb4rC7HZrU5PqvLsVltjs/qcmxWm+OzXIcnuaK7j7i+T5zqJuRr6W+rnLq2/ha7epHuPm5hE7HX1kKg47GaHJ/V5disNsdndTk2q83xWV2OzWpzfFaXY7PaHJ8bjilvQg4AAADAAWCqALV2htNWF0Gurb9sCbMAAAAAMNBUAepj8+WRW2y/y3z58SXMAgAAAMBAUwWod8yXD6qqa81QVd+R5H5Jvprkb5c9GAAAAACLNUmA6u7/m+R/ZXb39Kds2PzCJDdLcmZ3f2XJowEAAACwYNXd07xx1Z2SnJfkO5P8VZILk9w7yYmZXXp33+7+wiTDAQAAALAwkwWoJKmq707yK0kenOSwJP+S5A1JXtjdX5xsMAAAAAAWZtIABQAAAMD+b6qbkAMAAABwgBCgAAAAABhKgAIAAABgKAEKAAAAgKEEKAAAAACGEqAAAAAAGEqAAgAAAGAoAQoAAA4AVXWHqjp+6jkAODAJULAfq6pbVtV3Tz0HAPuvqnpsVf3Q1HOwR56Q5B1TDwGwKFV1dFX9SFX91NSzsHvbph4AWKyqunmSFyb5iSS3TtKZ/6xX1b2TvCDJ87r7A5MNeQCpqjvs7XO7+5OLnIVrVNVj9/a53f3qRc4C+4E/TvKSJG+behC4oaqqmya5ZZKDNtvud4Llq6pvS/LIJN+XrY9Nd/cTlzoYSZKqukeSVya557rVZ863PSDJXyf5se5+0wTjsQUBij1SVX+8l0/1j/ISVdWhSd6T5K5JPpTk0iTHrNvlH5P8xyQ/nkSAWo4dmUXA6+tb4ZAhzsi1j0tl98dpbR8Bagmq6uK9fGp3950WOgy785k4qx72yvysjefk2r+vbeR3giWrqtsl+ZskR2f23/+tdBJ/6yxZVR2Z5J2ZRcEXJzkyyUPW7XJukp1JTk4iQK0Q/5Cxpx6/xfrO5v8or633j/JyPTez+PT47n51Vb0gyS+vbezur1bVu5KcNNWAB6BXZ+8CFGM9YZN1P5rk4UneldkvNZ9JctskJyY5Psn/TPKGJc3HLGjszc/Orv5QYIy3Jjmxqm7U3d+cemwLfPgAACAASURBVBi4oaiqx2d2BuHVSd6d5J+TXDXlTHzL72YWBf88ySvi2KyaFyT59iT36u6PzP/m+VaA6u6uqvcl+d6pBmRzAhR76ogNj2+U5EWZnU3zB7nuH2u/kFl5fubyRiSzP6DftptLhD4R/xgvTXc/fuoZuK7u/tP1j6vqoUkenOSRm5yq/cKqemSS1yV5+ZJGPOB19+FTz8Aee26Sv03yqqp6dndfOvVAbKki0q6SU5N8Mcn9u/vCqYfhWh6U5Nzu/ompB2FTJyX5y+7+yC72+eckP7ikedhDAhR7pLs/sf5xVT0js/h07IZtH0vyrqr60yTnZ3bd9O8vbVC+K8nrd7PPl5McuoRZ4IbkuUnesNV9Arr7r6rqjUmen9nZHsA1/jzJ5Ukem+SUqtqR2f+U2ngGW3e3M3An1N3bk2yfeAyuceckZ4hPK+ngJH839RBs6ZZJPrWbfSqzs6RYIQIUe+unk7xuY5ha092XVNX/mO8nQC3Pl5J85272OSKze0MB17h7dv/JUBcleegSZmEPVNUtk9y8u/956lnICeu+v3GSo+ZfG7kcGa5tZ5KvTz0Em7ogyb+begi29NnMAu6u3DWzs6BYIQIUe+vwJJftZp8vzvdjef5Pkh+uqu/o7i9t3FhV/zazP6DfvPTJuJaq+t4kP5Tk9pn9wbaRG/gv179mFqF25e5JvrGEWdiCT/lcTd3tBuSwd96c5ISqqu4WaFfLbyd5dVX9+91c5sU0zkny41V1VHd/bOPG+e/ZJyX5w6VPxi4JUOytSzP74/kXN9tYVTXf/oVlDkVenNlHjr6lqn56/YaqOiazmygenNl9u5jA/GfjjCQ/mWtu1L/+fhxu4D+Ntyf50ap6apI/XP+HwPyYPTWzm1vu7hJXBvEpn8B+6BeTvDfJy6vqWd395akH4ls+l9mnp51XVS/O7NYim/7P9+4+d5mDkST5jSSPSXJuVW1Pcrskqaq7ZvbBMS/I7MqQ35lqQDZXYjt7o6p+L8nTk/xFkud09yXrth2R5DeTPDrJi7r71GmmPDDNPwXiBZkFjG8k+bbMzka7ZWZh4znd/dvTTXhgq6pfyCwUvjqzEPj+zC5TfV1ml7GcluQtSX5xq0tcWbyqulNm93q4ZZJLMgsdn01ymyT3z+zS1Z1J7t3dF08154Gsqn4rsxv2XutTPrv7oHX7vDnJ7br72KnmBNhTVXVOkltkdobtV5P8UzaPHO6ftmRV9c1c+38SbvlH8/r/DrE8VfXgzO5BeMjaqlxzzC5LcnJ3nzPReGxBgGKvzC+DOCfJvTL76NhP55o/1m6f5KDMLgc7yf/NWb6qOjHJ05LcJ8lhmd0c9m8zC4L+IZ5QVb0/yc26+5j5428m2d7dvzJ/fM/MjtXPdvefTDfpgaeq7pzkj5L8wCab/ybJU7r7ouVOxZqquijJP3X3Q+aPNwtQf5jZL5y3mWjMA1ZV3SjJUzK7PPKYzP6dW7s88p5Jnpzk97v749NNCatl/jvAnmiRY7nmZ9Xs0R/K3f3CsdOwlaq6RZLH5bp/8/xJd++ccjY25xI89kp3f7mq7p/Z/41+QpI7JbnDfPNFSf4kye92979ONOIBrbvfkd3fUJlpHJ3kTzes+9a/xd39wflZHD+f2c8RSzKPSw+qqtsnuWdmnxZ5eZIPdvenJx2OxKd8rqyq+vbMLv8+IbMzBb+U5ObrdrkkyX9O8vnMztAF4v5pq2z+iZGsuO6+LLMrC1489SzsGf/osde6+1+7+792910yO/Xxu5Mc0t1HdvdviE+wpcvXff+VJLfasP2fMgtVTKC7P93db+7uP5svxafV4FM+V9ezk5yY2Q3ib5Pkles3zv9AODeze0MCAAcoZ0CxEPPL7FxqB7v36cwuU11zcZLjNuxzl8zCFBOoqqMzu4To5t195tTz8C0+5XN1/USS9667lHizy1YuSfLwpU4FwH6pqo7fg92+meSKzC7fv3LwSOwhAYp9UlW3zuxm42v3e3jSuvVHJPlHP/DjrLtB4vXVa/fmYOn+PtcOTn+d5NlV9fwkf5nZJSyPjD+il66q7pHZmRv3XLf6zPm2B2R2rH6su980wXj4lM9VdkSSs3ezz85c92xPIElVnZLkSbnm8u8rMvvUtVd191lTznYgm38K7smZnb15+yQ33mQ3N4ifxjuz538DXV1Vb0tyand/bNxI7Al/gLLXquqJmf2if3Cu+dSBJ8033ybJ+5L8dJJXTTLggeHc7F2AYjqvT3Kvqjpi/umRv5XkP2V26cr2zH6Wdmb2aXgsSVUdmdkvMwdlFjqOTPKQdbucm9lxOTmzj2Vmybr7bVX1wszuIXRB/n97dx5lV1mne/z7BAIqKNDQCirRiMoFWm4TCQ0oGCYZbKUVuAqoHQMtoJe2bQRBZIyt0nRjqw2CQwwIJggKDYKCLIbQILQQZHACMQEZvSijDCbw3D/eXalTlVNDypy9T9V5PmuxzK79xvWslVVV+/z2+/5+Zconkh5h4JTP65tL2bOepUzyGs4UhhhhHtGrqgLHWcB+lJ9hz1N6pa0H7ATsKOmdtvdvLmVvkrQ6ZSrxDAZOV+vjlq9H/U4EplOe1e4Erqd/INa2lOe4Sym7b6cB7wC2kTS9dXp71C9T8GJMJO0C/BC4jfJhYFfK1K7WaUS3AffYzpb7iGFIWosyIWojYDFwlu0HGw3VYySdA7wb2NL2z4eYsHYesJntTZvKGZny2Y0kXQW8DniD7T8N/v6pfsbdDVxv+10NRo3oKpIOpkxfvRn4JHCN7eclrQK8Dfg8Zdf0R22f3lzS3iPpWMqLwdmUF1OPVNdfpRSlPk8penzA9vONhOxhkramDFz6GPA1txQ1qsLuQcApwA62b5Q0E5hD2VX4Dw1EjkoKUDEmkq6gNEne1PYTQ3xYOxvYxvZGTeWMiBgNSQ8BV9h+f3Xd7mfaKcAs2yPt9IjoKZL2Bc6h7PA8APg41fdPNSL7m8C7gHfavrS5pBHdRdL/UHY7bdauZYWkF1N2fP7B9vS68/UySbcDz9nesrp+ATi+pdfd64CfAifY/vfmkvYmST8Cnra95zBrLgJWt71rdb0AmGL7tfWkjHZyBC/Gaktgvu0nhllzH7B+TXmiRdWMdyeGP68+u95UEV1tHcrPrOEIWK2GLNGGpCspja6PaTpLDGR7XrUzeial0PQogKSbgM0ov4dOTfEpYjmbAmcM1S/V9jOSLqTs5oh6bUTpLdjHwORlF/ZvJF1C+bmXAlT9tgK+PMKa24BDW65vqf5eNCgFqBir1Rh5StfalLPsUaOqR8qRDPz+bj2j3vfnFKBq0DKl439sPzvKqR0A2F7QoVixvIeB14+wZjPgtzVkifa2phy1iy5ke1b1dvljwOaU3zXTgJ8Bp9j+ZpP5IrrU4L5C7Yx0PzpjCaW/XZ8ngb8ctOYeStE96ifK0e/hDD6FsxR4rjNxYrRSgIqxWszyo+MH+xsgkwZqJGl/4BjgSuBUynGIucDllPPqBwDnAWc0k7AnXU15wNyE0iSx73o0Vhl5SawkVwL7Stq43YQUSdMpuwpPrT1Z9LkL2LDpEDE023OBudWxoXWAx22P9LIqopf9AniPpKOHOYL3d8DPa08W91FOEvS5E9hm0JotKANKon43AHtJervtywfflLQbZVL7VS1ffj3wUE35YggpQMVY/RdwhKR9bJ83+KakD1HegB5de7LedgjlF+ZutpeWHnwsrkb4zpd0AWVU9rwGM/aaEykFp0cGXUd3+RywD7BA0vHAKwEkbQZsTxm28CTwb00FDL4OnCBpiu17mw4TQ6s+SLc9UhQRA8yhNCFfIOlIShPypVUT8u0pv5teQ5mYG/W6Dti55fpC4DOSvg5cQHmxuzPw7fqjBeUz5gLgB31H9OmfgvdWYAfKbqdPw7JhGLsAZzeSNpZJE/IYE0nrAAspb6O/C/R9U38M2A54D2XizZvz9rM+kh4H5tk+uLp+AZht+7iWNZcAL7U96qNgEb2gels2D3hZ35foPx7xGLB3pqw1R9JrgS9R3jifBPyE8iZzuQeZFKiaIWlNyjTJLSjPBY9Tem5cYPupJrNFdKNqWtfZwL6Un2UvUHbU/AUwifL75zu239dYyB4laQZlMuEhthdLegllN810+p8Nfk2ZsnZ/Y0F7WNXW4hv0H7VrPdJ6N3Cg7WuqtS8GpgAP236s7qzRLwWoGDNJU4CzKG9oBrsW2C8/kOsl6WnKKPKjq+s/UkaT/lPLmpOAg22v1VDMnlaN9V1k+1tNZ4nlVRO7/p7Sb2hdygfoG4Bv2s42+wZVBfW+h8vhHl5sOzu8ayZpH+B0Sv/H1p41phRwD7J9fhPZIrpdNUlyFssXb+fYzq71LiFpVWBPylGuxcDFtp9uNFSPq4q429L/vfME5XvnOqfQ0ZVSgIoxqSrOT9j+qaTNKWeil31Ys31zowF7lKS7Kdu3Z1XXvwAetL1jy5pvAXvYXrehmD1N0p+A/7B9RNNZIsYTSXMZ5fFV2x/qbJpoVU3A+wFl98Y5lF53D1Em4e4A7EcpSu1u+4qGYkZERETDUoCKMZH0PGVs7EeazhL9JJ0PTLG9VXV9KvBhylu171HOq59PeSuw81D/P9E5khYBl/Udk4yIGO8kXUuZeLed7YVt7m9J6dVxU45/R0RE9K4UoGJMJD0MnG37sKazRD9JMynNLDezvUjShpRtqOu0LFsCzLCdceYNkHQKZWTvm9pNvIlmVLs6R/ICZWv3Xfm3q5+kD1J6N1zWdJYYSNJTlD41s4ZZM5fSR23N2oJFjBNVj7sPsPwRvLNtL2ouWe+S9GnKAJLX2H6gzf1XAYuAY2yfVHe+XlO1sBgL2569UsPEnyUFqBgTSedSdtoMHkcaXUbSVOAwSoO+xcBptm9vNFQPk/RS4EpKk9HDbN/RcKRgQH+h0XgeuAz4hO1fdS5VtJK0FPiy7Y83nSUGkvR74PS+/oNDrPkspQ9Ujn9HtJB0GPAvwGQG9k+D8tLwKNun1B6sx0m6kdJuZJdh1lwGvCyfhzqvek4brPW5bXDvwb6v2fYqHQsWKyxNOmOsPg3cKGk2cKLtJU0HivaqN2f/t+kcscytwGqU4yq3SnoW+B3LFz9se6PBfzk65kTKZJvdgTuB6+kf57st8EbgUsrbzmnAO4BtJE3P2+naPESZChXd51rgLSOseQvlGF5EVKrm4ycDj1KmfF7NwP5p/wicLOl+2+c2lbNHvZ4yoXA4PwfeX0OWKN8Pg30c2IOhew9eAvxHTflilLIDKsZE0hzKD+a3UD6k3Ur7cdi2fUDN8SK6lqTFjL6R8tTOpok+kramjFf+GGVypFvuCTgIOIUybvnG6rjrHOAbtv+hgcg9R9LXga2Av7bd7k1oNETSxsCPga8Cs23/seXeGpRjLAcA22bXYEQ/STcBU4Fptu9pc38qcDNwt+3pdefrZdVk6S/aPmqYNZ8DPm77RfUlC1h2LP8rjNx78KBMnu4uKUDFmAyxDbKdbHuMiK4n6UfA07b3HGbNRcDqtnetrhdQjiK/tp6UvU3SK4AbKG85D7f9SLOJelf1Emqw1wHbUXrXLKR/B+E0Sk+bBcBv8lIqol9V5DjT9iHDrDkD+IDtl9SXLCTdCdxvu93Om741V1GeA7JjvWaSbgFuGUXvwc1tT6stWIwoR/BirLIzIyImkq2AL4+w5jbg0JbrW6q/F/WYRylufBB4X7WbcKidtzvVnK3XzBzm3trAjm2+/jZge8pOqIgongQeG2HNo5QBGFGvHwIflfTedscfJb2P8nPttNqTBcDGlNYIw3kA2KeGLLECUoCKMWm3TTgiVpykdYA1bf+26Sw9TpQdHMMZ/IZzKfBcZ+JEGzNa/rw65eFz4zbrsrW78/ISKmLluBzYFWh7zKs6Av72al3U6yRgf+Dbkt5LKUjdD7yK0i/yXZSBMp9vLGFve4KRew++FXiqhiyxAlKAioiomaQ1gRMoDzZ/SfnAvGp1728o/VI+3e5Me3TMDcBekt5ue7kHfUm7AXtR+kT1eT1lB07UwHYakHeJvISKWGmOAK6XNA84svV7S9IUShFk7Wpd1Mj2/ZJ2Bc4D/g5oPaIvymTpfWzf10C8KA3GZ0r6N+AE20/23agmTh9PKVB9s5l4MZT0gIqIqJGktYD/BjYDfkqZiLdJX680SS+h9E453fbhjQXtMZKmU3rUrAZcCVxHfw+bt1ImqjwHvM32T6p/xweBs21/uJnUERExnki6ss2X1wE2B54H7qX/d88UYBXK8e8/5GhxMyRNBt4JbE0pBj5GeWl1caaAN0fSyynDL15LOcr6U/q/d/4aeBnwG8rwi981FDPaSAEqIqJGkv4V+AQw0/ZZko4Djm1t1i/p+8Ar0zSxXpK2B75B/1E7U95yAtwNHGj7mmrtiykfDh62PVL/joieIGkS5XjKq4HJ7dbYXlBrqIgusgJDfAbLUJ+IQSStC3wO2A9obdL/NHAO8Cnbv28iWwwtBaiIiBpJ+jVwl+3dq+t2BahTgb1tv6KhmD2r6rexLbAFZXLXE5Rm49c5vzC7hqRXUwodq7e7nyJH/SQdTimurzfcunyIjoiIlUnSqsD/ojy3PQ780vbSZlPFUNIDKiKiXq8GvjvCmqcov0SjZlWR6brqv+gykt4OfIHyoDmcFDlqJOl44Fjg98CZlEa9efiPiHFD0gerP15g+8mW6xHZPqtDsWIUqmLTHU3niNFJASoiol5PAi8fYc1U4JEassQgkjYAdmLo3TW2PbveVAEgaWvg+8D/A/4TOBS4BvgVsB2wCXARZcda1OsASq+NN9t+vOkwEeOFpOeB+bb3bzpLMJdy9P4GyrNa3/VwVK1JASpilFKAioio10+Av5X00taJHX2qAsgelA/aUSNJJwBHMvB3Y9/DZeufU4BqxlHAs8B02w9IOhS4yvaJ1dHJE4B/Bo5uMmSPWpcyOCHFp4gV8ySl8Xg0bxbld/yDg66jC1QN/A38ve37hmjo347TwL+7pAAVEVGvLwI/AC6VNGB6mqRNgK8BLwK+1EC2niVpf+AYygS8UynHJOcClwMzKDs8zgPOaCZhANsAF9l+oOVrk2DZ0cljJe1OKUTt3UC+XvZryiSviFgxtwCbNh0iwPbc4a6jcTMoBaiXtFyPRoqIXSYFqIiIGtm+rNppcxzlvPoSAEmPUD7ACfik7eubS9mTDgHuA3azvbRsqGGx7fnAfEkXAJcA8xrM2OvWYuBOgT8Bawxacx1lGk7U6zRgtqT1bT/UdJiIceQk4GJJu9j+UdNhop+kOcDttr/QdJYA25OGu47xIwWoiIia2T5B0gLgH4GtKcdXDFwKfMH2aLcVx8rzJmDeoKkpyxpZV4XDy4DDgYvrDhcA/I6Bu2x+B2w0aM1k4MW1JQoAbJ8u6Y3AdZJOBBZSJhG1W5vjRhH9Xg78EPiBpAspx/Qfos2ujTS6rt1+lKEXEbESpQAVEdEA21cBVzWdI5aZTJng1ecZlp9EeAdwcG2JYrA7GVhwugHYXdIbbd8paX1gL+CuRtLFrcBMYM4wa0yePSNazaV8Xwh4T/UfDCxApdF1MxYz8tCYaEga+I9feQiIiIgoTUc3aLm+F9h80JpXktHyTfoh8BlJf2H7D5R+au8BbpH0c+ANwEuBIxrM2JMkHUjpj7YUuBp4gHyvRIzGh5oOEEP6NnCwpHVsP9p0mFhOGviPUyp9OyMiok6SXgt8ANiCstPmcUoz0rNtL2ouWW+SdD4wxfZW1fWpwIcpU3C+R2l2eT5wne2dm8rZyyS9DNgE+HnfBElJ76ZMJdyI8rb6C7a/2ljIHiXpl5SfY9vm51dETASSJlMGkkwBPg38xPbDzaaKPpKuAp6wvWfTWWLFpAAVEVEzSYcB/0I59qVBt5cAR9k+pfZgPUzSTEoj5c1sL5K0IaUg2NpzaAkww/YNDUSM6FqSngG+bvvQprNERKwM1REv6D8CORTbzqmimknajdKTc4808B9f8s0SEVEjSfsCJwOPAl+iHFd5CFgf2IHSmPxkSffbPrepnL2mGrc8t+X6t5KmA4fRv7vmNNu3N5Ev+klaE3g3A3cPLgQutP1Uk9l62P2UgnpExERxLcMXnqJZaeA/TmUHVEREjSTdBEwFptm+p839qcDNwN22p9edL6KbSdoHOB1Ym4G7Bw08Bhxk+/wmsvUySccCB1J2ED7ZdJ6I8ULSb0a51LYHT/2M6FmSXqC/gX+r5Rr4216F6BrZARURUa9NgTPbFZ8AquNf51H6Q0VERdIuwDzgBco0qKsZuHtwP2CepMdsX9FUzh71WUrT/iskfRK4OYWoiFGZRPtdNmvTP4n1AcoR8Ijolwb+41R2QEVE1EjSw8Ac20cNs+bzwEzb69eXLKK7SboWmAZsZ3thm/tbAguAm2xvX3e+XpZeKRErn6TXU47qrwHsavvZhiP1LElrAG8E1rR9bdN5IsazFKAiImok6VuUYyrThrgvyhG8O2x/sNZwEV1M0lPAd2zPGmbNXGBv22vWFiyQdDWj7JVie4fOpomYOCS9CLgDOG+4F1fRGZJeDXwReCewCi1FdElvBb4KfMT21Y2F7HHpCzn+5C1URES9jgCulzQPOLL1KJ6kKcBJlK33RzSUL6JbPQc8OMKaB6p1USPbM5rOEDER2X5W0o+AfYEUoGokaQPgRuAVwEWUptfbtCy5sfraeylHwqNmI/WFlJS+kF0oBaiIiHqdQ2mW/H+AvSTdCzxMecCZQnnDdhvw7bIZahnb3qnmrBHd5FrgLSOseQvlGF5ExESxlNLrLup1HKXAtIvtqyQdR0sByvaS6mj4SL+XogPSF3L8yhG8iIgaVVM7xiJTPKKnSdoY+DHlyMNs239subcG5cPCAcC2tn/VTMreVu0Y2Al4FbB6myW2PbveVBHjl6T1gJ8Cz9h+Q9N5ekn1gvAntveqro8Djm19FpP0RWB/2+s1FLNnpS/k+JUdUBERNbI9qekMEeOBpDltvnwbcDjwYUkL6d89OI3S+2EB5fjqAXXljELSCcCRDHy2bG1K3vfnFKAiKpKOHeLWqsCGwJ6Un205fle/VwB3jbBmCaVJfNRvC+DcdsUnANs3SfoOsHe9sWIkKUBFREREN5o5zL21gR3bfP1twPakAFUrSfsDxwBXAqcC3wXmApcDMyj/HucBZzSTMKJrHT/C/SeAz9j+1xqyxEB/oBQBh/NGyrGvqF/6Qo5TKUBFREREN5radIAYtUOA+4DdbC+t+tcttj0fmC/pAuASSr+OiOg31FTIF4BHgV/aXlpjnuh3HfAuSevbXq7IJOkNwG7A2bUnC0hfyHErBaiIiAZUfR02AV4NTG63xvZZtYaK6CKtEyKj670JmDfog/KyPim2L5N0GeX45MV1h4voVravaTpDDOlkyhHIayT9E/ASWNZzcHvgC5RC4b83lrC3fRL4saTPM3RfyL8Ctm0oXwwhBaiIiBpJehHlYWUWsNpQyyi9UlKAiojxYDLw+5brZyh9a1rdARxcW6KIiD+D7RslHQR8Bfh+y60nqv9dCsyy/bPaw/Wg9IWcODIFLyKiRpK+DHwU+AVwLnA/5SFmObbPrDFaRMSYSLobuMb2rOr6F8CDtndsWfMtYA/b6zYUM6JrSdoaOJDSWHlt4HHgZuCbtq9vMluvq47afQTYGliX8m9zA/Cfmbhan0yRnjhSgIqIqJGkhylNE6fbXtJ0noiIP5ek84Eptreqrk8FPkzZ6fk9SiPy84HrbO/cVM6IbiTpM5Qpd2pz28BJtj9Vb6qI7iLpNWP9uznS311SgIqIqJGkp4Cv2D686SwRESuDpJnAacBmthdJ2hC4BVinZdkSYIbtGxqIGNGVJO1D2Q19DzCbMknyQWADyqTPY4ApwL62v9NUzoiIlSUFqIiIGkm6EbjL9vubzhIR0SmSpgKHARsBi4HTbN/eaKiILiNpAfAG4E22H2lzfz1K/7Rf2X5b3fkCJE0CXsXwQ2MyaS1ilFKAioioUfW2cy6whe07G44TERERDZH0OHCW7UOHWfNl4IO2Bzf2jw6TdDjwCWC94dalx1DE6GUKXkREjWyfJ2kD4FpJpwELKQ0t263NG7WIiIiJa1Xg6RHWPE0+s9VO0vHAsZQJn2cyzNCYiBi97ICKiKiZpOMoY2NfPNy6vFGLiIiYuCTdBqxCOYK33JSv6vjXrZRJXpvXna+XSfot8BzwZtttXxRGxIpLNT0iokaSjgKOo7xROxd4gLxRi4iI6EXfBj4L/Jekf7Z9V98NSRsBJwObAkc3lK+XrQucnuJTxMqVHVARETWStAh4nrxRi4iI6GmSVgMuB7YHXqC8lHoQWJ/S+HoS8N/Azrb/1FTOXlTtTrvZ9oeazhIxkUxqOkBERI9ZH7goxaeIiIjeVhWVdqHscFpEmbQ2Hdiwuj4a2CnFp0acBvytpPWbDhIxkWQHVEREjST9DLjR9qyms0RERET3kLQmsBbwuO2nms7T6ySdAuwJnMjwQ2PurTNXxHiWHlAREfX6CnCcpPVtP9R0mIiIiGhWVXh6N7AFVQFK0kLgwhSiGnUrMBOYM8wak8/UEaOWb5aIiHpdDMwArpd0InAzeaMWERHRkyTtA5wOrA2o5ZaBxyQdZPv8RsL1MEkHAmdQBsVcTYbGRKwUOYIXEVEjSS9QHipV/e9QbDsvCSIiIiYoSbsAP6A0ID+HUuh4iNIvcgdgP8rzwu62r2goZk+S9EvKbrRtbS9qOk/ERJECVEREjSTNZfjC0zKZvBIRETFxSboWmAZsZ3thm/tbAguAm2xvX3e+XibpGeDrtg9tOkvERJK36xERNbI9s+kMERER0RW2AM5tV3wCsH2TpO8Ae9cbK4D7gclNh4iYaCY1HSAiIiIiIqIHPQc8OMKaB6p1Ua+zgD0kvbTpIBETSY7gRUR0kKQ5lCN385BvNgAAAslJREFUn7L9cHU9GrZ9QAejRURERIMkXQisbXvGMGuuAf5g+921BQskrQrMBzYEPgncbPvJZlNFjH8pQEVEdFBL0/FNbN9ZXY+Gba/SwWgRERHRIEkbAz8GvgrMtv3HlntrAMcBB1AaYf+qmZS9SdLzfX8kQ2MiVpoUoCIiOkjSa6o/3m97acv1iGzf06FYERERUbMhdkG/DtgOeBxYCDwMvILSnHwtShPy32RXdL0kXc3oh8bs0Nk0ERNHClAREREREREdtgK7oAfLruiImBCyXTAiIiIiIqLzpjYdICKiSdkBFRERERERETGIpA2AnYBXAau3WWLbs+tNFTF+pQAVERERERER0ULSCcCRDDw11NqUXOR4ZMQKmdR0gIiIiIiIiIhuIWl/4BjgWmBvSrHpTGA/4GvAC8B8YMemMkaMR+kBFREREREREdHvEOA+YLdqijHAYtvzgfmSLgAuAeY1mDFi3MkOqIiIiIiIiIh+bwIutb205WvLjtrZvgy4DDi87mAR41kKUBERERERERH9JgO/b7l+Blhr0Jo7gP9dW6KICSAFqIiIiIiIiIh+DwIbtFzfC2w+aM0rgaVExKilABURERERERHR7xbgr1qurwS2k/QBSWtIegelOfktjaSLGKdke+RVERERERERET1A0kzgNGAz24skbUgpNq3TsmwJMMP2DQ1EjBiXUoCKiIiIiIiIGIakqcBhwEbAYuA027c3GipinEkBKiIiIiIiIiIiOio9oCIiIiIiIiIioqNSgIqIiIiIiIiIiI5KASoiIiIiIiIiIjoqBaiIiIiIiIiIiOioFKAiIiIiIiIiIqKjUoCKiIiIiIiIiIiOSgEqIiIiIiIiIiI6KgWoiIiIiIiIiIjoqBSgIiIiIiIiIiKio1KAioiIiIiIiIiIjkoBKiIiIiIiIiIiOioFqIiIiIiIiIiI6Kj/D3sqjVEYKAQgAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 720x360 with 1 Axes>"
      ]
     },
     "metadata": {
      "image/png": {
       "height": 340,
       "width": 592
      },
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plotMostFrequentTokens(flippedPredictions_pos0,('Beer_Name',),10,'img2.png')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-02-25T09:57:58.838271Z",
     "start_time": "2020-02-25T09:57:58.798519Z"
    }
   },
   "outputs": [],
   "source": [
    "from utils.trianglesChart import countTokens\n",
    "df0 = flippedPredictions_pos0[flippedPredictions_pos0.alteredAttributes==('Beer_Name',)]\n",
    "df1 = flipped_pos1[flipped_pos1.alteredAttributes==('Beer_Name',)]\n",
    "r0_neg = countTokens(df0.rtable_Beer_Name.values)\n",
    "r1_neg = countTokens(df1.rtable_Beer_Name.values)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-02-25T10:02:38.508248Z",
     "start_time": "2020-02-25T10:02:38.461210Z"
    }
   },
   "outputs": [],
   "source": [
    "def compareRankings(r0,r1,topn):\n",
    "    index= list(r0.keys())[0:topn]\n",
    "    l0 = []\n",
    "    l1 = []\n",
    "    for token in index:\n",
    "        l0.append(r0[token])\n",
    "        if token in r1:\n",
    "            l1.append(r1[token])\n",
    "        else:\n",
    "            l1.append(0)\n",
    "    df = pd.DataFrame({'standard': l0,\n",
    "                   'improved': l1}, index=index)\n",
    "    return df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-02-25T10:07:07.037463Z",
     "start_time": "2020-02-25T10:07:06.533142Z"
    }
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAuIAAAI9CAYAAAB/ix3FAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAAWJQAAFiUBSVIk8AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8li6FKAAAgAElEQVR4nOzdeZzVZd3/8dcHMURUkNBIU0HNLbMEt9tSCMxIfm6gaZqglVnd3i5pd5RyC1mmdVeCZpoprrnncmu5JeCCpQ0qekeLCpjeoihCKCII1++P75lhmDnDNmfOdZh5PR+PeRzOdzuf+XIG3nOda4mUEpIkSZKqq1PuAiRJkqSOyCAuSZIkZWAQlyRJkjIwiEuSJEkZGMQlSZKkDAzikiRJUgYGcUmSJCkDg7gkSZKUgUFckiRJysAgLkmSJGVgEJckSZIyMIhLkiRJGRjEJUmSpAw65y6gLUTEDGATYGbmUiRJktS+9QH+lVLqu6YnViSIR8QFwB7ADkAv4F1gFnAHcHFK6c0y5+wLnA3sA3QF/gFcCVyUUlraypI26dq1a8+dd965ZyuvI0mSJLVo+vTpvPvuu2t1bqSUWl1ARCwGpgJ/AV4HulEE7D2A/wP2SSn9s9HxhwK3AYuAm4C5wMHAjsCtKaUjW1lPXb9+/frV1dW15jKSJEnSSvXv35+pU6dOTSn1X9NzK9U1ZZOU0qKmGyPih8D3gO8C3yxt2wS4HFgKDEwp/bm0fTTwEHBERBydUrqxQrVJkiRJNacigzXLhfCSm0uPH2207QhgM+DG+hDe6Bpnl55+oxJ1SZIkSbWqrWdNObj0OK3RtkGlx3vLHP8wsBDYNyK6tGVhkiRJUk4VnTUlIs4ENgK6U/QP/zRFCD+/0WE7lh7/3vT8lNL7pRlPPgZsC0yvZH2SJElSraj09IVnAh9q9Pxe4PiU0pxG27qXHue3cI367T1W9WIR0dJozJ1Wda4kSZKUU0W7pqSUeqeUAugNDKNo1X4qIvpV8nUkSZKkdV2bLOiTUnoNuD0iplJ0QbkG2LW0u77Fu3u5cxttn7car1N2mphSS7nhX5IkSTWrTVfWTCnNioi/AJ+MiF4ppTeAv7F88Z8VupZERGegL/A+8GJb1iZJkvJbtmwZc+fOZcGCBbz33ntUYn0TaW1FBF26dGHjjTemZ8+edOrUtvOatPWsKQBblB7rV8t8qPQ4pMyx+wMbAlNSSu+1dWGSJCmfZcuW8c9//pM5c+awaNEiQ7iySymxaNEi5syZwz//+U+WLVvWpq/X6hbxiNgBeC2lNL/J9k7AucDmFMH6rdKuW4ELgKMj4qJGC/psAPygdMwvW1uXJEmqbXPnzmXhwoV07tyZ3r17061btzZvgZRWZtmyZbzzzjvMnj2bhQsXMnfuXHr16tVmr1eJrikHAT+KiEeBGcCbFDOnDKAYrDkbOLH+4JTSvyLiRIpAPikibqRY4v4QSkvcUyx7L0mS2rEFCxYA0Lt3bzbeeOPM1UjQqVOnhvfiyy+/zIIFC2o+iD8IbE8xZ/juFNMOvkMxSPNaYHxKaW7jE1JKd0TEAOAsYDiwAfA88K3S8X42JUlSO/fee0Uv1G7dumWuRFpR/Xuy/j3aVlodxFNKzwEnr8V5j1G0pkuSpA6ovt3N7iiqNREB0ObjFnznS5IkSY3UB/G2ZhCXJEmSMjCIS5IkSRkYxCVJktRqY8aMISKYNGlS1joigoEDB2atYXW16cqakiRJrdFn1D25S1ipmecPrdprDRw4kMmTJ7vwUTtii7gkSZKUgS3iK1GJ38Kr+ZtyW2rtvWgv90GSJKlSbBGXJEnK7K677mLw4MF8+MMfpkuXLmyxxRYMGDCASy65hJkzZxIRTJ48GSj6QNd/Ne4LPXHiRL72ta+xyy67sMkmm9C1a1d23XVXxo4dy6JFi5q9ZuM+3bfeeit77bUXG264IT179uToo4/mlVdeKVtrXV0dQ4YMYeONN2aTTTbhgAMO4PHHH2/xe7vjjjv40pe+xA477EC3bt3o1q0b/fv3Z/z48SxbtqzZ8ccffzwRwYsvvshFF13EbrvtRteuXVf4XhcvXsy5557LdtttR5cuXejbty9nn312my/AU2m2iEuSJGX0q1/9ipNOOonevXtz8MEH06tXL15//XWmTZvGhAkTOOaYYzjnnHO46qqrmDVrFuecc07DuX369Gn48wUXXMBf//pX9t13X4YOHcqiRYt47LHHGDNmDJMmTeLBBx9kvfXWa/b6l1xyCXfddReHHHIIAwYM4E9/+hM33XQTzzzzDE8//TRdunRpOHbKlCkccMABLF68mGHDhrH99tvz9NNPM3DgQAYNGlT2+xs1ahSdOnVi7733Zsstt2T+/Pk89NBDnHrqqTz55JNce+21Zc879dRTeeSRRxg6dCgHHXRQQ+0pJb7whS9w5513st1223HyySezePFirrzySp599tm1+SvIxiAuSZKU0WWXXcYHPvABnnnmGTbffPMV9r3xxhv06NGjIUzPmjWLMWPGlL3OJZdcQt++fZstRjN69Gh+8IMfcOutt3LUUUc1O+/ee+/lySef5OMf/3jDtmOOOYYbbriBO++8ky984QtAEYC//OUv8+6773LHHXdw6KGHNhw/btw4TjvttLJ13XPPPWy33XYrbFu2bBknnHAC11xzDSeffDJ77713s/OmTp3KU089Rd++fVfYXl/XPvvsw8SJE9lggw0AGDt2LHvuuWfZGmqVXVMkSZIy69y5M+uvv36z7b169Vrta2y77bZlV4Q8/fTTAbjvvvvKnnfKKaesEMIBTjzxRACeeOKJhm1Tpkzhb3/7G/vvv/8KIRzg5JNPbha265Xb3qlTJ0499dSV1vWf//mfzUI4wIQJEwA477zzGkI4QM+ePRk9enTZa9Uqg7gkSVJGxx57LAsXLmSXXXbh9NNP54477mDOnDlrfJ133nmH8847jz333JPu3bvTqVMnIoIPfvCDAC32+d5jjz2abdtqq60AeOuttxq2TZ06FYABAwY0O3699dbj05/+dNnrv/nmm4waNYrddtuNjTbaqKF/e//+/Vda11577VV2+9SpU+nUqVPZ11tX5g+vZ9cUSZKkjL71rW/Rq1cvLrnkEsaPH8+FF15IRDBgwAB+8pOflA3KTS1ZsoRBgwbxxBNPsOuuu3LUUUex2WabNbSyjx07tsWBjD169Gi2rXPnIiIuXbq0Ydv8+fMB+NCHPlT2Or179262bd68eey5557MmDGDvfbaixEjRtCzZ086d+7MvHnzGDduXIt1lbtefR09e/Ys+wlCS+fUKoO4JElSZiNGjGDEiBHMmzePKVOmcPvtt3PllVfyuc99jr/+9a9sttlmKz3/zjvv5IknnuD4449v6LpR79VXX2Xs2LGtrrF79+4AvPbaa2X3z549u9m2X//618yYMYNzzjmnWd/2xx9/nHHjxrX4euW62dTXMXfuXJYsWdIsjJeroZbZNUWSJKlG9OjRg4MOOojLL7+c448/nrlz5/Lwww8DNMwa0riVut7zzz8PwLBhw5rtq5/2sLX69evX4vWWLl3Ko48+2mJdw4cPr1hd/fr1Y9myZWVfb9KkSWt1zVwM4pIkSRlNnDix7LL1r7/+OgAbbrghQENf75deeqnZsfXTGDYNoi+++CLf+c53KlLnvvvuy4477sjDDz/MnXfeucK+iy++mBdeeGG163rqqaf40Y9+tFZ1nHDCCQCcddZZK8yPPnfuXH7wgx+s1TVzsWuKJElSRocffjgbbbQR++yzD3369CGlxCOPPMKTTz5J//79OeCAAwAYPHgwt9xyC8OGDeOggw6ia9eubLPNNhx33HEcfPDBbL/99vzsZz/j2WefZffdd+ell17i7rvvZujQoWXD+5qKCK644go++9nPMnz48BXmEf/DH/7AkCFDuPfee1c4Z8SIEfzkJz/htNNOY+LEiXz0ox/lH//4B3fffTfDhg3jpptuWuM6vvjFL3LTTTdx1113seuuu3LooYeyZMkSbr31Vvbcc8+yvxDUKlvEJUmSMjr//PPZc889mTp1KpdccgkTJkxgyZIlXHDBBUycOLGhH/RXv/pVvvvd7zJ//nx+/OMfM3r0aK644goAunXrxkMPPcQxxxzD//7v/zJ+/HimTZvG6NGjue666ypW66c+9SkeeeQRDjjgAH7/+99z0UUXsXjxYiZNmlR2LvAtttiiYVGeRx99lIsvvphZs2ZxySWXcP75569VDRHBLbfcwtixY1m2bBkXX3wxd911FyeccAI333xza7/FqopyH4Ws6yKirl+/fv3q6upadZ0+o+5pdS0zzx/a6mvUgtbei/ZyHyRJlTN9+nQAdt5558yVSM2t7vuzf//+TJ06dWpKqf+avoYt4pIkSVIGBnFJkiQpA4O4JEmSlIFBXJIkScrAIC5JkiRlYBCXJEmSMjCIS5IkSRkYxCVJkqQMDOKSJElSBgZxSZIkKQODuCRJkpSBQVySJEnKwCAuSZIkZWAQlyRJymzmzJlEBMcff3zuUtqtWrzHnXMXIEmS1KIx3XNXsHJj5ueuQOswg7gkSVJmW265JdOnT6d79xr/xUMVZRCXJEnKbP3112ennXbKXYaqzD7ikiRJmZXrv3z88ccTEcyYMYOLL76YXXbZhQ022IA+ffpw3nnnkVIC4JZbbmGvvfaiW7dubL755px88sm8++67zV4jIhg4cCD/93//x3HHHcfmm29O165d6d+/P7/5zW+aHT9p0iQigjFjxvDEE08wdOhQevbsSUQwc+bMhuPq6uoYPnw4m2++OV26dGGbbbbhm9/8Jq+++uoK1xsyZAgRwTPPPFP2Htx0001EBGeeeeYK2+fOnct3v/tddt55Z7p27Ur37t0ZPHgw999/f9nrLFiwgG9961t85CMfYYMNNmCnnXbiZz/7GcuWLSt7fE62iEuSJNWwM888k0mTJnHwwQdz4IEHctddd3HWWWexePFievbsyahRozjssMPYb7/9eOCBB/jFL37B0qVL+eUvf9nsWm+99Rb77rsvPXr04IQTTmDevHncfPPNHHvssbzyyit8+9vfbnbO448/zo9+9CM+/elP8+Uvf5k33niDD3zgAwDcfffdDB8+nJQSRxxxBNtssw11dXX88pe/5M477+TRRx+lb9++AIwcOZL77ruPa665hp/+9KfNXufqq68GWOGXkVmzZjFw4EBmzpzJfvvtx5AhQ3jnnXe4++67GTJkCJdddhknnnhiw/HvvfcegwcP5sknn+QTn/gExx57LPPmzePcc89l8uTJrfp7aAsGcUmSpBpWV1fHtGnT2HLLLQEYM2YM22+/PT/5yU/YcMMNqaurY+eddwaKILr77rtz5ZVXMnbsWDbffPMVrjVt2jSOPPJIbrzxRjp1KjpGjBo1iv79+3PWWWcxfPhwtt122xXOuf/++7n00ks56aSTVtj+9ttvM3LkSN5//30mTZrEfvvt17DvggsuYNSoUZx00kkNLdeHH3443bt35/rrr+eCCy6gc+flMXT27Nncf//99OvXj1133bVh+8iRI5k1axY33HADRx99dMP2efPmMXDgQE455RQOOeQQPvShDwHw05/+lCeffJJhw4Zxyy23NPsea41dUyRJkmrY6NGjG0I4QI8ePTjkkENYuHAh3/jGNxpCOECXLl046qijWLx4MdOnT292rfXWW48LLrigIaAC9O3bl1NOOYUlS5Zw7bXXNjvnk5/8ZLMQDnDnnXcyd+5cjjrqqBVCOMAZZ5xBnz59eOCBB3jppZcA2GCDDfjCF77Aa6+9xn333bfC8ddddx1Lly5l5MiRDdueeeYZJk+ezPDhw1cI4fX3YOzYsSxatIjbbrutYfuECRPo1KkTP/7xj8t+j7XGFnFJkqQatsceezTbtsUWWwCUbeWtD+0vv/xys31bb711Q1eRxgYOHMjYsWN56qmnmu3ba6+9ytY1depUAAYNGtRsX+fOndl///2ZOXMmTz31FFtvvTVQdDu5/PLLufrqqxk6dGjD8VdffTXrr78+xxxzTMO2xx9/HID58+czZsyYZq8xZ84cgIZfOBYsWMDzzz/PVlttxXbbbdfi91hLDOKSJEk1rNyUhvXdOla2b8mSJc321XfhaKp3795AEXpb2tdU/bEf/vCHy+6v3z5v3ryGbfvuuy877LADd911F2+99RabbropU6dO5bnnnuOwww6jV69eDce++eabADzwwAM88MADZV8Dii4yjetZ1fdYS+yaIkmS1EG89tprZbfPnj0bKB/sI6LsOfXH1p/bVP2sKU2vOWLECN577z1uuukmYPkgzcbdUhqfN27cOFJKLX5NmDBhheNX9T3WEoO4JElSB/HSSy+tMPVgvUmTJgGw++67r/a16o+tP7ex999/n0ceeQSAfv36rbBvxIgRdOrUiauvvpolS5Zwww030KtXrxW6qgDss88+AA3XWZWNN96Y7bffnldeeYUXXnih2f5ydeZmEJckSeogli5dyne+850V5tSeMWMG48ePp3PnznzpS19a7Wsddthh9OzZkxtuuIE//vGPK+y78MILmTFjBgcccEBD//B6W221FYMGDeKPf/wj48aNY86cORxzzDGsv/76Kxy3xx57sN9++/Hb3/6WK6+8smwNzz77LK+//nrD8xNOOIFly5a1+D3WGvuIS5IkdRC77bYbf/rTn+jfvz8HHnhgwzzi8+bN48c//nHZQY4t2Wijjbjyyis58sgjGTBgAEceeSRbb701dXV13H///fTu3ZvLLrus7LkjR47kwQcf5Hvf+17D83J+85vfMGjQIL7yla8wfvx49t57b3r06MHLL7/MtGnTeO6553j88ccbpmk844wzuOOOO7jtttvo168fn/vc5xq+x/3335+77rprDe9Y27JFXJIkqYPYdNNNmTJlCh/72MeYMGECV199NX379uX6668vu5jPqhx66KE89thjHHTQQdx3333893//N9OnT+frX/86dXV1zeYkrzds2DA22WQTlixZwq677tqs+0q9j3zkI9TV1fHDH/6Q9dZbj+uvv57x48czZcoUtt56ay677DI+/vGPNxzfpUsXHnzwQU4//XTmzJnDuHHjmDx5MmeffTY///nP1/j7a2tRvzxqexIRdf369etXV1fXquv0GXVPq2uZef7QVR+0DmjtvWgv90GSVDn10841ngdbbSciGDBgQE32la5Fq/v+7N+/P1OnTp2aUlrjFYNsEZckSZIyMIhLkiRJGRjEJUmSpAycNUWSJKkDaI/jAtd1tohLkiRJGRjEJUmSpAwM4pIkSVIj1erGYxCXJElZRATACkuRS7WgPojXv0fbikFckiRl0aVLFwDeeeedzJVIK6p/T9a/R9tKq4N4RHwwIr4aEbdHxPMR8W5EzI+IRyPiKxHRqcnxfSIireTrxtbWJEmSat/GG28MwOzZs1mwYAHLli1zZg9lk1Ji2bJlLFiwgNmzZwPL36NtpRLTFx4J/BJ4FZgIvAR8CBgG/Br4fEQcmZr/ZD0D3FHmes9VoCZJklTjevbsyTvvvMPChQt5+eWXc5cjrWDDDTekZ8+ebfoalQjifwcOAe5JKTV08oqI7wFPAMMpQvltTc57OqU0pgKvL0mS1kGdOnViq622Yu7cuSxYsID33nvPFnFlFRF06dKFjTfemJ49e9KpU9v24m51EE8pPdTC9tkRcSnwQ2AgzYO4JEnq4Dp16kSvXr3o1atX7lKkqmvrlTWXlB7fL7Nvi4g4Cfgg8CbweEppWhvXI0mSJNWENgviEdEZGFF6em+ZQz5b+mp8ziRgZErppdV8jboWdu20mmVKkiRJWbRlx5fzgV2B36WU7mu0fSFwLtAf2LT0NYBioOdA4A8R0a0N65IkSZKya5MW8Yg4BTgD+CtwXON9KaXXgf9qcsrDEXEg8CiwN/BVYNyqXiel1L+F168D+q155ZIkSVJ1VLxFPCJOpgjRfwE+k1KauzrnpZTep5juEGD/StclSZIk1ZKKBvGIOA24iGIu8M+klGav4SXmlB7tmiJJkqR2rWJBPCK+A/wceJoihL++FpfZp/T4YqXqkiRJkmpRRYJ4RIymGJxZBwxOKb2xkmP7NV32vrR9MHB66el1lahLkiRJqlWtHqwZESOB7wNLgUeAUyKi6WEzU0pXlf78M+CjETEFqF/PdjdgUOnPo1NKU1pblyRJklTLKjFrSt/S43rAaS0cMxm4qvTna4HDgT2BzwPrA68BNwMXp5QeqUBNkiRJUk2rxBL3Y4Axa3D8FcAVrX1dSZIkaV3Wlgv6SJIkSWqBQVySJEnKwCAuSZIkZWAQlyRJkjIwiEuSJEkZGMQlSZKkDAzikiRJUgYGcUmSJCkDg7gkSZKUgUFckiRJysAgLkmSJGVgEJckSZIyMIhLkiRJGRjEJUmSpAwM4pIkSVIGBnFJkiQpA4O4JEmSlIFBXJIkScrAIC5JkiRlYBCXJEmSMjCIS5IkSRkYxCVJkqQMDOKSJElSBgZxSZIkKQODuCRJkpSBQVySJEnKwCAuSZIkZWAQlyRJkjIwiEuSJEkZGMQlSZKkDAzikiRJUgYGcUmSJCkDg7gkSZKUgUFckiRJysAgLkmSJGVgEJckSZIyMIhLkiRJGRjEJUmSpAwM4pIkSVIGBnFJkiQpA4O4JEmSlIFBXJIkScrAIC5JkiRlYBCXJEmSMjCIS5IkSRkYxCVJkqQMDOKSJElSBgZxSZIkKQODuCRJkpSBQVySJEnKwCAuSZIkZWAQlyRJkjIwiEuSJEkZGMQlSZKkDAzikiRJUgYGcUmSJCmDVgfxiPhgRHw1Im6PiOcj4t2ImB8Rj0bEVyKi7GtExL4R8buImFs6Z1pEnBYR67W2JkmSJKnWda7ANY4Efgm8CkwEXgI+BAwDfg18PiKOTCml+hMi4lDgNmARcBMwFzgY+DnwqdI1JUmSpHarEkH878AhwD0ppWX1GyPie8ATwHCKUH5bafsmwOXAUmBgSunPpe2jgYeAIyLi6JTSjRWoTZIkSapJre6aklJ6KKX0P41DeGn7bODS0tOBjXYdAWwG3FgfwkvHLwLOLj39RmvrkiRJkmpZWw/WXFJ6fL/RtkGlx3vLHP8wsBDYNyK6tGVhkiRJUk5tFsQjojMwovS0cejesfT496bnpJTeB2ZQdJnZtq1qkyRJknKrRB/xlpwP7Ar8LqV0X6Pt3UuP81s4r357j1W9QETUtbBrp9WqUJIkScqkTVrEI+IU4Azgr8BxbfEakiRJ0rqs4i3iEXEyMA74CzA4pTS3ySH1Ld7dKa9++7xVvVZKqX8LNdQB/VZdrSRJkpRHRVvEI+I04CLgOeAzpZlTmvpb6XGHMud3BvpSDO58sZK1SZIkSbWkYkE8Ir5DsSDP0xQh/PUWDn2o9DikzL79gQ2BKSml9ypVmyRJklRrKhLES4vxnA/UUXRHeWMlh98KvAEcHRF7NLrGBsAPSk9/WYm6JEmSpFrV6j7iETES+D7FSpmPAKdERNPDZqaUrgJIKf0rIk6kCOSTIuJGiiXuD6GY2vBWimXvJUmSpHarEoM1+5Ye1wNOa+GYycBV9U9SSndExADgLGA4sAHwPPAtYHxKKVWgLkmSJKlmtTqIp5TGAGPW4rzHgINa+/qSJEnSuqitl7iXJEmSVIZBXJIkScrAIC5JkiRlYBCXJEmSMjCIS5IkSRkYxCVJkqQMDOKSJElSBgZxSZIkKQODuCRJkpSBQVySJEnKwCAuSZIkZWAQlyRJkjIwiEuSJEkZGMQlSZKkDAzikiRJUgYGcUmSJCkDg7gkSZKUgUFckiRJysAgLkmSJGVgEJckSZIy6Jy7AHUQY7pX4BrzW38NSZKkGmGLuCRJkpSBQVySJEnKwCAuSZIkZWAQlyRJkjIwiEuSJEkZGMQlSZKkDAzikiRJUgYGcUmSJCkDg7gkSZKUgUFckiRJysAgLkmSJGVgEJckSZIyMIhLkiRJGRjEJUmSpAwM4pIkSVIGBnFJkiQpA4O4JEmSlIFBXJIkScrAIC5JkiRlYBCXJEmSMjCIS5IkSRkYxCVJkqQMDOKSJElSBgZxSZIkKQODuCRJkpSBQVySJEnKwCAuSZIkZWAQlyRJkjLonLuAdm9M91aeP78ydUiSJKmm2CIuSZIkZWAQlyRJkjIwiEuSJEkZGMQlSZKkDAzikiRJUgYGcUmSJCkDg7gkSZKUQUWCeEQcEREXRcQjEfGviEgRcV0Lx/Yp7W/p68ZK1CRJkiTVskot6HM28AngbeBlYKfVOOcZ4I4y25+rUE2SJElSzapUED+dIoA/DwwAJq7GOU+nlMZU6PUlSZKkdUpFgnhKqSF4R0QlLilJkiS1a5VqEV8bW0TEScAHgTeBx1NK0zLWI0mSJFVNziD+2dJXg4iYBIxMKb20OheIiLoWdq1OH3VJkiQpmxzTFy4EzgX6A5uWvur7lQ8E/hAR3TLUJUmSJFVN1VvEU0qvA//VZPPDEXEg8CiwN/BVYNxqXKt/ue2llvJ+rSxVkiRJajM1s6BPSul94Nelp/vnrEWSJElqazUTxEvmlB7tmiJJkqR2rdaC+D6lxxezViFJkiS1saoH8YjoFxHNXjciBlMsDARwXXWrkiRJkqqrIoM1I+Iw4LDS096lx3+LiKtKf34jpXRm6c8/Az4aEVMoVuME2A0YVPrz6JTSlErUJUmSJNWqSs2a8klgZJNt25a+AGYB9UH8WuBwYE/g88D6wGvAzcDFKaVHKlSTJEmSVLMqtcT9GGDMah57BXBFJV5XkiRJWlfV2mBNSZIkqUMwiEuSJEkZGMQlSZKkDAzikiRJUgYGcUmSJCkDg7gkSZKUgUFckiRJysAgLkmSJGVgEJckSZIyMIhLkiRJGRjEJUmSpAwM4pIkSVIGBnFJkiQpA4O4JEmSlIFBXJIkScrAIC5JkiRlYBCXJEmSMjCIS5IkSRkYxCVJkqQMDOKSJElSBgZxSZIkKQODuCRJkpSBQVySJEnKwCAuSZIkZWAQlyRJkjIwiEuSJEkZGMQlSZKkDAzikiRJUgYGcUmSJCkDg7gkSZKUgUFckiRJysAgLhL+LjQAACAASURBVEmSJGVgEJckSZIyMIhLkiRJGXTOXYC0rukz6p5WnT/z/KEVqkSSJK3LbBGXJEmSMjCIS5IkSRkYxCVJkqQMDOKSJElSBgZxSZIkKQODuCRJkpSBQVySJEnKwCAuSZIkZWAQlyRJkjIwiEuSJEkZGMQlSZKkDAzikiRJUgYGcUmSJCkDg7gkSZKUgUFckiRJysAgLkmSJGVgEJckSZIyMIhLkiRJGRjEJUmSpAwM4pIkSVIGBnFJkiQpg4oE8Yg4IiIuiohHIuJfEZEi4rpVnLNvRPwuIuZGxLsRMS0iTouI9SpRkyRJklTLOlfoOmcDnwDeBl4GdlrZwRFxKHAbsAi4CZgLHAz8HPgUcGSF6pIkSZJqUqW6ppwO7ABsAnxjZQdGxCbA5cBSYGBK6SsppW8DnwQeB46IiKMrVJckSZJUkyoSxFNKE1NK/0gppdU4/AhgM+DGlNKfG11jEUXLOqwizEuSJEnruhyDNQeVHu8ts+9hYCGwb0R0qV5JkiRJUnVVqo/4mtix9Pj3pjtSSu9HxAzgY8C2wPSVXSgi6lrYtdI+6pIkSVJuOVrEu5ce57ewv357jyrUIkmSJGWRo0W8YlJK/cttL7WU96tyOZIkSdJqy9EiXt/i3b2F/fXb51WhFkmSJCmLHEH8b6XHHZruiIjOQF/gfeDFahYlSZIkVVOOIP5Q6XFImX37AxsCU1JK71WvJEmSJKm6cgTxW4E3gKMjYo/6jRGxAfCD0tNfZqhLkiRJqpqKDNaMiMOAw0pPe5ce/y0irir9+Y2U0pkAKaV/RcSJFIF8UkTcSLHE/SEUUxveSrHsvSRJktRuVWrWlE8CI5ts27b0BTALOLN+R0rpjogYAJwFDAc2AJ4HvgWMX80VOiVJkqR1VkWCeEppDDBmDc95DDioEq8vSZIkrWty9BGXJEmSOjyDuCRJkpSBQVySJEnKwCAuSZIkZWAQlyRJkjIwiEuSJEkZVGoecUkdUJ9R97Tq/JnnD61QJfl5LyRJa8oWcUmSJCkDg7gkSZKUgUFckiRJysAgLkmSJGVgEJckSZIyMIhLkiRJGRjEJUmSpAwM4pIkSVIGBnFJkiQpA4O4JEmSlIFBXJIkScrAIC5JkiRlYBCXJEmSMuicuwBJUvvSZ9Q9rTp/5vlDK1SJJNU2W8QlSZKkDAzikiRJUgYGcUmSJCkDg7gkSZKUgUFckiRJysAgLkmSJGVgEJckSZIyMIhLkiRJGRjEJUmSpAwM4pIkSVIGBnFJkiQpA4O4JEmSlIFBXJIkScqgc+4CpA5nTPcKXGN+668hqc31GXVPq86fef7QClUiqRbZIi5JkiRlYBCXJEmSMjCIS5IkSRkYxCVJkqQMDOKSJElSBgZxSZIkKQODuCRJkpSBQVySJEnKwCAuSZIkZWAQlyRJkjIwiEuSJEkZGMQlSZKkDAzikiRJUgYGcUmSJCmDzrkLkCRJ7VufUfe0+hozzx9agUqk2mKLuCRJkpSBQVySJEnKwCAuSZIkZWAQlyRJkjIwiEuSJEkZGMQlSZKkDLIF8YiYGRGpha/ZueqSJEmSqiH3POLzgQvLbH+72oVIkiRJ1ZQ7iM9LKY3JXIMkSZJUdfYRlyRJkjLI3SLeJSK+BGwNvANMAx5OKS3NW5YkSZLUtnIH8d7AtU22zYiIE1JKk3MUJEmSJFVDziA+AXgE+F9gAbAtcDLwNeD3EfFvKaVnVnaBiKhrYddOlSxUkiRJqrRsQTylNLbJpueAr0fE28AZwBjg8GrXJUmSJFVD7q4p5VxKEcT3X9WBKaX+5baXWsr7VbguSZIkqWJqcdaUOaXHblmrkCRJktpQLQbxfUqPL2atQpIkSWpDWYJ4ROwcEc1avCOiD3Bx6el11axJkiRJqqZcfcSPAs6IiIeBWRSzpmwHDAU2AH4H/Hem2iRJkqQ2lyuITwR2BHYHPkXRH3we8CjFvOLXppRSptokSZKkNpcliJcW63HBHkmSJHVYtThYU5IkSWr3DOKSJElSBgZxSZIkKQODuCRJkpSBQVySJEnKwCAuSZIkZZBrHnFJgjHdK3CN+a2/hmqL7wtJHYQt4pIkSVIGBnFJkiQpA4O4JEmSlIFBXJIkScrAIC5JkiRlYBCXJEmSMjCIS5IkSRkYxCVJkqQMDOKSJElSBgZxSZIkKQODuCRJkpSBQVySJEnKwCAuSZIkZdA5dwGSJGBM9wpcY37rr6Ha4vtiuXZyL/qMuqfV15h5/tAKVJKf98IWcUmSJCkLg7gkSZKUgUFckiRJysAgLkmSJGVgEJckSZIyMIhLkiRJGRjEJUmSpAwM4pIkSVIGBnFJkiQpA4O4JEmSlIFBXJIkScrAIC5JkiRlYBCXJEmSMjCIS5IkSRkYxCVJkqQMDOKSJElSBgZxSZIkKQODuCRJkpSBQVySJEnKwCAuSZIkZWAQlyRJkjIwiEuSJEkZGMQlSZKkDAzikiRJUgYGcUmSJCkDg7gkSZKUgUFckiRJysAgLkmSJGXQOXcBkiRJWgNjulfgGvNbf41asI7fC1vEJUmSpAwM4pIkSVIGBnFJkiQpA4O4JEmSlIFBXJIkScrAIC5JkiRlYBCXJEmSMsgaxCPiIxFxZUT8X0S8FxEzI+LCiNg0Z12SJElSW8u2oE9EbAdMATYH7gT+CuwFnAoMiYhPpZTezFWfJEmS1JZytohfQhHCT0kpHZZSGpVSGgT8HNgR+GHG2iRJkqQ2lSWIl1rDDwRmAr9osvsc4B3guIjoVuXSJEmSpKrI1SL+mdLj/SmlZY13pJQWAI8BGwL7VLswSZIkqRpy9RHfsfT49xb2/4OixXwH4A8tXSQi6lrY9Ynp06fTv3//ta8QePWV+a06H6B/p7dbd4H/ad33UCmtvRetvg/gvWjMe7Gc92I578Vy3ovlauBe1MT/p+C9aMx7sVwr78X06dMB+qzNuZFSatWLr9WLRvwKOBE4MaX06zL7fwh8D/heSulHK7lOS0F8V+Btiq4vOe1Uevxr1iry8z4s571YznuxnPdiOe/Fct6L5bwXy3kvlquVe9EH+FdKqe+anpht1pRKSCnl/3VuJep/Uaj1Otua92E578Vy3ovlvBfLeS+W814s571YznuxXHu4F7n6iNd/FtG9hf312+dVoRZJkiSp6nIF8b+VHndoYf9HS48t9SGXJEmS1mm5gvjE0uOBEbFCDRGxMfApYCHwx2oXJkmSJFVDliCeUnoBuJ+ic/u/N9k9FugGXJtSeqfKpUmSJElVkXOw5jcplrgfHxGDgenA3hRzjP8dOCtjbZIkSVKbyjJ9YcOLR2wFfB8YAnwQeBW4HRibUnorW2GSJElSG8saxCVJkqSOKtdgTUmSJKlDM4hLkiRJGRjEJUmSpAwM4pIkSVIGBnFJkiQpA4O4JEmSlIFBXJIkScrAIC5JkiRlYBCXJGUTESMi4nO566hFEbF1ROyfuw5JbceVNdXmImJTYKOU0j9z11INEbH12p6bUnqpkrVItS4i3gcuSimdnruWWhMR5wD/lVJaL3ctuUXEhsCmQNl74b+dHVNE7ATsTJExrs1dz9ronLuAdVlEXLmWp6aU0lcqWkyNiYiNgLHAscBmQKL0fouIvYFzgLNTSlOzFdl2ZlJ8v2uq4R61BxExYm3PTSldU8laakFEvLiWp6aU0nYVLaa2zMZPZ9WCiDgO+A5F2GpJu/q3s5yIWB84FNiLln8haffZol5EfBL4NbB7o83XlvYNAH4PHJVS+p8M5a2Rdv3GrYLjW9iegFjJ9gS02x+WiOgOPAp8DHgaeIMV/xF9FtgP+CLQHoP4NaxdEG9vrmLF+xCs+r7UH9PugjhF2Fyb90W5f0vak3uBz0REp5TSstzFqHZExPHAlcBS4BHgn8D7OWvKISK2AB4AdmLl/x6062xRLyJ2ACZR/DIyDtgB+HyjQx4G5gJHAAbxdq5vk+edgJ9ThMzxFG+U2UBv4DPAf1C8Qb5VvRKzOIsihB+fUrqm/uPV+p0ppYURMRkYnKvAtpRSOj53DTXihDLbhgEHA5Np/vOxP3AXcHuV6quqlFKf3DXUqLOAPwJXRMS3U0pv5C5INeNM4C3g0yml6bmLyeinFI1ZNwCX00F/IWnkHOADwB4ppb+UMkZDEE8ppYh4HNgzV4FrwiDeCimlWY2fR8TpFCG8X5N9fwMmR8TVQB3Fx0sXVq3Q6hsG3LeK7gWzWEd+SLR2UkpXN34eEQcBQ4BDy3xcODYiDgVuBi6tUomqDTcA84ERwNERMZPiF7Smnx6klFK7/OV9JYL2/4nIymwPXNXBQzjAgcDDKaVjcxdSIwYDv00p/WUlx/wT+GyV6mkVg3hlfQ24uWlAr5dSmhERt5SOa89B/CPAbas45m2gexVqUe04C7i9pT57KaU7I+IOYDRFd4UOpaMNam5kYKM/dwF2LH011eG6e6WUxgBjMpeR01zgvdxF1IANgD/lLqKGbAq8vIpjgqLVvOYZxCurDzBvFce8VTquPVsAbL6KY/pS9B3vMCJiT+BzwJYUgaOp9j7Q5hPAxFUc8zxwUBVqqQkdfFAzACklB2qqJXcDAyMiUsee4u05YJvcRdSQ1yg+LVmZj1G0itc8g3hlvUERtL5bbmdERGn/m9UsKoMngf8XERunlBY03RkRH6YIW3dXvbIMSn/vVwFfYvlgxMYfN3eIQbzAYoowvjKfAJZUoZbsHNQsrdJ3gceASyPijJTS27kLyuQnwDURscsqumN0FA8BX4yIHVNKf2u6s9ToNRj4RdUrWwu2RFTWLcAnI+LmiFhhIGfp+U3AbqXH9mwc8EHgdxGxwpRTpee3UHzUNj5DbTmcDBxHMbXSHhSh+0JgX+B7FJ8g3Ahsm6vAKvkDcFBEnFz65aRBFP6DYsDNg1mqq77Gg5r7UfxcNEgpLaQY1NrR+kVL9W4BFgJfBV6NiKkR8VCZrz9krrOtvU4x+8eUiBgbEYdExP7lvnIXWiU/ohis+nBEfAPYAiAiPlZ6/j8U/6/+d74SV58L+lRQ6WPmhyjC1lLgFYqPUD5E0R1hPYrW4sHt/Tf70ijmcyhaeZcA61N0y9mUIoh+J6X0k3wVVk9E/BnollLaufR8GTAmpfT90vPdKWaN+HpKaUK+SttWRGxH0c9xU2AGRWtw/c/Hpym6K80F9k4pre2c2+uMiHge+EdK6fOl580Wb4mIXwBHpJQ+lKnMqoiITsC/U3TR2Zni56W+i87uwInAhSmlv+erUtVW+rdydaT2vOhR6T40/iS1xeDWnu9DYxExhGKg9yb1m1h+j+ZR/Lv5UKby1ohdUyoopfR2RHyaYsqlE4DtgPpVFp8HJgA/TSktzlRi1aSUxkbEw8ApwD4ULeQJ+B3w83XlB6RCdgKubrKt4WcvpfRURNwNfJPiPdIupZReiIh9gEuAA2j+CcADwL93hBBe4qBmICI+QLH4xkCKX8QWABs1OmQG8GVgDsUv9+ogHD/Q4Pt0wMHKK5NSurfU02AkyzPGfIpGrQkppbk561sTBvEKK4Xs84DzSi3k3YH57b0FvJyU0kRWPTivo5jf6M/vAD2b7P8HxRRV7VpK6XngwIjYkmJFtO4U9+aplNIrWYurPgc1F75NMY/8GOAHFGsOjK7fmVKaV/ql/nMYxNUBlWbPURMppXkUXWHH5a6lNQzibagUvjtcAFczr1B0Tar3ItC/yTEfpQjoHUIpdHe04N2Ug5oLxwKPNeqqVa7lbwbFQlCS1K4YxNtARGwGDGd5X8evNtreF3g2pfRuxhJVXU+wYvD+PfDtiBgN/JbiI/lDaf+Bq0FE7ETx87FRSuna3PVkMo7ivfC7iPha4x2lQc2X0zEGNfcF7lnFMXNp/imSOoiIOJpiwGb9p2j/olgc74qU0o05a1P1reag1GUU75N/1HrecrBmhUXEVyj+49yA0uCB+sETEbEr8AzwtZTSFfmqrKxGA0nWVKofkNWeRcRhFKO8Dyot6tQT+DPFfPL1g0vmUizj/NdshVZBRHwS+DXFf6jA8sFFETGAIpge1dKiP+2Ng5ohIt4Cbk0pnVh6Xm7Q6vUUg9x7ZypTGZRmV7oGOIbi52EpRVetXhSTHyTgxo6w4mTpXhzBqteiaPezLK1h5lgK3AecWW6qw1pgEK+giPgsxYqA0yj+c/0cxUwYjf9DmQbMSim1m49ZI2ISazmQJKX0mcpWs24ozSF9IsWA3pnANSmlV7MW1cYiYgeKTwfWo2jt3QH4fKMgHhQLMPwhpTQyW6FVFhGfYcVBzfUDjjrEoOaImEgxcPejKaXFTYN46WflBWBKSumQjKWqyiLi6xSDu+uA7wCTU0pLI2I9YABwPsWnjf+eUro0X6VtKyK6UEx0MJBVrEXREWZNiYgxwJ4U093+HZjC8hm49qX4v+V3FF3a+gH/RtHYtWdKaUaGklcupeRXhb4o5j9+Gdik9PwcYGmTY64DXshdq19+VfsLuJ5iTuBdSs/L/XzcAvwld61+VfV98UWKj5FvoZiKrOF9AfQAbqdo1Tood61+Vf298QTFmJquLezvSvFL2pO5a23j+/BfpZ+RsRRdtJaVtvUGjqZozPkNsF7uWqt0P/YB3gW+RqlBudG+AL5e+r9m79K240v37PLctZf7cmqgytoDuDul9K+VHPMyxQ+P1NEMBn6bVr4y3D8pLc7Q3pUWIjk3dx25pZRuoFh5djjFFIXfgIb591+lGD9xSUrpd7lqVDa7ALenFvr4lrbfwYor0rZHRwJTU0rnpEbT8qWUZqeij/wg4P8Bp+UqsMrOBe5PKf0qlZJ2vVS4lKJh9PulbVdRrFvx2WoXujraff/cKvsAq575ogdF6067V5r1YTAr78/W7oJIo4EkT6SUFq3JamcppYfbqKxasCnFL6IrExQ/Rx3BPhRdUDq8lNKXS1MUnkqx+nBQfKT8v8DPUjte6Eor1bQLRjmr2t8ebEfRna9eohhPUjxJ6cWIuIei5fen1S0ti72Ai1ZxzDTgPxo9f6p0Xs0xiFfWTJpPS9fU3kBNDhiopIgYC4xixfdYfd+2xn9ud0EcmETxve1M0X+t/vnqaM/9+14Dtl/FMR+jaBXvCP4BbJW7iFpRarW6KiK6UvzSNj+l1GGm9FRZ04FhEXFWuVbx0nvlMGBln7K1B0uARY2eLwA2a3LMLKCjjKEImi8I19R2TZ6/D7zXNuW0jkG8su4E/jMijkwp3dJ0Z0ScQNHac1bVK6uiiDiWYkGOh4BfUKweeBVwP8Vgk69Q9Ae9LE+Fba5+FbQ3mjzv6B4CvhgRO6Yyo9cjYk+KT1B+UfXK8vg1MDYitk4pvZS7mFpRClw1Pd2YquZKisGaD0fEKIrBmu+XBmvuTzEb1TbAjzPWWA0vs+JaFH+nGIDY2O4UAxI7gj8CwyPiwJTS/U13RsQQiq5ujRcU3B6YXaX61oizplRQRGwKTKVo5bqNYr7Tz1J83LofMIxiYEn/9tzSExGPAlsD25b+0VwGjEnLF+z4HMW8wYenDjJNnSAidqT4+XibYhXFT1LMDbwbxX+q51BM+7lbRwimEdGHYqrT3YELKBb4mU2ZX9o6yP3YCDicJiuuUvQRdmG0Dqg0k9J1FAN6E8WAu/o55TtRtIzenFI6OluRVRARvwIOSCltW3r+PYpVaK+kGMw8EPgW8JuU0nG56qyWUqPNwxTdGB8CHmP5rCmfplip9z1gQErpydLMS68C16WUvlb+qvkYxCssIrammPe0XL/gR4BjUjtfyjsi5gM3pJS+Xnq+DDg3pXROo2PuATZOKa12/+l1VUT8FzAjddyFaxqUWipuoJgdA1acimsecETqAFP2wQpz4TbuslVOSu18vv2IOBK4lGIMTdNp2eYBJ6WUbs1Rm/KLiC8CX6b5L2lXlgb7tmsRMZBi+sZvpJRmRsSGFK29e7L835Dngc+093xRrzT26gqWd0FpPJ7gBeCrKaXJpWO7UjQOvpZSmlftWlfFIF5BpTfGv1JKT0fEbhQfHTXMC5xSqstaYJVExEKKOZDPKj1/h2LaoNMaHXMBxRzr3TOVWTURsRi4MKX0n7lrqQUR0QMYSfN5syc0nhGgvYuIq1jNLksppRPatpp8Susv/J6itfN6ijEVsylml/oMyxdz+XxK6cFMZUo1JSI6U8wotD3F+LT/SSktzFpUlZU+MdmXFVdcfQp4rOlsKrXMIF5BEbEUuCyl9M3cteQUES9Q9OX7cun5dODVlNKgRsdcSzEv8AczlVk1ETEDuK/+EwJJy0XEIxQzpOyXUppaZv8eFB9D/7kjfIImqWNxHvHKegMHGUHxG+mujZ4/BOwXEcdFRLeIGEqxVO9TWaqrvtuBA0ofj0kARMSI0niJjm534KZyIRwgpfRn4GaKsK4OKCL6RMToiPhtRPyh9Dg6Ivrmrk1qLVvEKygibgK2Tik1Hc3coUTE8RQj3T+WUpoREVtRhO5NGx22BBiYUmr38yhHxMYUv4zMBc5IKT2XuaQsVnM+9WUUHy/+o6VFPNqLiHgfuCildHruWnKKiDeBS+u7srVwzHkU/cTb/SdoWlFEnAH8kGLe7KZzhi8BvptS+lnVC6uiiDibYjD7Niml/yuzf0uK5dxHp5QuqHZ9ba00zmptrBNrlRjEKygiPgr8iWL6te+nlJZkLqlmlFouzqAYWDGTYqW8Z7MWVSUR8SLF6O4PlzYtAl6nef/glFJqOvdpu9FocOLqWArcB5xZbqrD9iAiXgZuSymdmruWnCLiDqBHSmngSo6ZDMxNKR1etcKUXWmQ5vXAWxQzDE1ixfEDp1D0DT4mpXRTpjLbXET8iWL8WYsrQ0bEfcAm7bEhsPR/R1ON/y9pOsC7fltKKdX82hz/v707j7arLs84/n0YxDIUKFQCAoIIFBGrKBZBJhEE61ALaAFRJJTBLtoqMjmEIbSKVllWgeCAgSqEQbEIKE4EMAyLIVawVQQSKASxogQQxASe/vHbl5zcnHtvhnv2Pnef57NWVrLv/mWt996V7PPu3/C+ScTHkaTzKAcndqaU0vkvupcjs+3JNYcXDZE0l6U/lNfapVZJp1BO+e9LqYN7I4tKTu0EbAVcTZnZ2Z5y2Pm3wA625zQQck9J+jKl09urbHf7oBkIVVnLm4AvUqor/b7j3hqUmcDJwE5tfSmL7iTdBmwObG/7/i73NwduB+61vUPd8dWlWjX62mgv7ZLOBN5je3ijnwlP0m5dvvxB4C2MfMD7KkqRhOtqCnO5JREfRyO8tXUzId7SIsaTpB0pJbf+iVJFxx33BBwJfJZSguuWaovTecBXbP99AyH3lKQNKNViZgLH2f7N6H+jHaoJi+FeSum1MJ9Sa37oBW17yozn9cB9mcAYLFUFrvNtHz3KmHOBQ2yvXl9k9ap+Dp+zfdIoYz4BfND2C+uLrBmS3gucw9gHvI+cCGWDk4iPI0kvWdqx3d7uI9pM0veBp2y/Y5QxVwCr2X5zdX095dzFZvVEWR9JP6I0JtkO+CNly9ZIK2h71htd7yzDhMVwmcAYMJIeodQKHy0B/SRwqO1J9UVWL0l3Aw/Z3mOUMddSnpWt3d44RNJsYPZQZbYRxkynNIfr+0PerW4SUbck17E0qg6sa9r+36ZjqdnrgM+PMeanwDEd17Orv9dGu3f8eTVg6+rXcG2bLWnt9qsYd98D3gx0TcSrlbS9q3Ft9l3gHyS9u9teeEl/B+xGKZIwCLambGMczTzggBpiWWFJxCNqULXvPhU4GPhzSnK1SnXvryj7YD82Ugm3lhBlC8Johs/mLKS0Km4d2wNZPjYTFrEMjgdulHQRcGLnv52qi/UZlG6sbW+Wdgbls+NCSe+mJOYPAS+mnLl5O+U8zScbi7Bej1PO4o3mDcCTNcSywpKIR/SYpLWBHwPbAj+h1JvfpmPInZT9sQdS9se21c3AfpL2tr3EDJakfYD9KPvIh7yMsl0jIlqu2q413GPAuyjPjgdYdH5gU2Blyira14HWbN8azvZDVc+BS4G/oXTUHCLKtrYDbD/YQHhNuAo4VNK/AafafmLoRlUu+BRKov7VZsJbNtkjHtFjkj4FfJiyj/ECSScDUzr3u0q6EthoIuxnW16SdqAcoHkBpa76LBZ9qL6Bctr9GWA327dWLzAPU6oFHNFM1FEXSStRZvg2ptSMXoLt62sNKmqV8wOjk7Qq8DZgR8pKwGOUCY5vD1K5ZEkvolRa2gx4gjLBNfRZ8irgT4H7KJWWft1QmEstiXhEj0m6h9KgZt/qulsifhawv+0NGgqzFlVTn6+waAuKWVQD9l7g8KFyU1Un0k2BR2w/VnesdZK0MSUJXa3b/bYnoJKOo7ysrj/auEFItiJibJLWAz5BKVXYWTHnKcoKyUdsP9pEbMsqW1Miem9j4BtjjHmSUqat1WxfL2krSt3wV1O+58cphzJndZY0rDprtrputKS9gTOBvxhjaGsT0Kq+/BTgUeB8yt7XhU3GFBH9rUqyj5D0Acrzc21K+dOf255Qz48k4hG99wTwojHGbE7ZO956VbI9q/o1sKq66lcC/wd8gVIt5jrKy8culHMEV1BeUtpsMmUZ+TW25zcdTPQPSc8CM2wf3HQsdarqZANcbvuJjusx2b6gR2H1pSrpvqvpOFZEEvGI3rsVeKuktToPlQyRtCGlQ9iVtUfWgOr73ZORt2LY9tR6o2rEScAfKJ1D50k6BrjW9mlVWbZTgQ8BH20yyBqsB0xLEh5dPAE80HQQDZhO2bZ3M+VnMHQ9GlVjBioRb4Mk4hG99zngO8DVkhY7dChpG+BLwAuBf28gtlpJOhU4kcWfPUMfIJ1/HoRE/PXAFbbndXxtJXh+1WCKpH0pCfn+DcRXl3uAdZsOIvrSbODlTQfRgMMoz8GHh10PpKqajoH32X5whOo63UyIZmhJxCN6zPY1VQJ6MmUJbQGApN9QEhABSqG1TQAADShJREFUJ9i+sbkoe0/SwcDHKRVTzqLsm59OacaxO2WLwqXAuc1EWLu1WXy274/AGsPGzKIcRmqzs4GpkibZTqnK6HQG8G1Je9n+ftPB1MX29NGuB9DulER89Y7rpTEhXl6SiEfUwPapVbv2f6SUnlqP8pC4GjjT9tK+4U9kRwMPAvvYXlh2XzDX9gxghqTLKfVhL2owxjr9msVngn/Nkg2NVgX+pLaIGmB7WnWAd5ak0yi19LtuU7E9iNsUBtmLKM1rviPpW5Rtfr+iS4LV5r3Rks4D7rR9ZtOxNGF487O2NUNL+cKIqIWk+cBFto+qrp8Dpto+uWPMVcBatndtKMzaSPoh8KztvavrSyhd8l5j+25Jkyj1cee1ub48gKT3UarHjFY5yLYzeTRAqmdEZ4nTIZ2Ji2h5HXFJf6BM2JzUdCwx/vJQi4i6rEopUTfkaZZMvO4CjqotomZ9Fzhd0p/Z/i3lLMHfArMl/TewJbAWLW/fLelwynakhcBMYB4pXxjF+5sOoE/MZezKWwOjbdV0kohH1ETSZsAhLKqfPZ9yGOlrtuc0F1ltHgY27Lh+AHjlsDEbMThJ2LmUTqMLAGzPknQA5aDqKygfvse3ecm98mHKtpydBuT/QSwl2+c3HUOfuBA4StK6tn/XdDB9oFXVdLI1JaIGko4F/oUyKzx8mXUBcJLtz9YeWI0kXQZsavt11fVZwBGUigDfpBzAuYzS2OdNTcUZ9ZL0NPBl28c0HUtEP6pa23+D0mn4Y8Ctth9pNqrmSLoWeNz2O5qOZTwkEY/oMUkHUlru/o5SonAm5cDRJGAPygHOtYGDbF/cUJg9J+lQSoWMbW3PkbQJZUWg88DiAmB32zc3EGIjJK0JvJPFV0ruAL5l+8kmY6uDpHuAHwydHYiIxVVbMWDxUq/dDMQ5Ckn7AN8G3tKGajpJxCN6TNJtlM6Z29u+v8v9zYHbgXtt71B3fE2qvvdjKdVC5gJn276z0aBqVG1FmQasw+IrJQYeA460fVkTsdVF0hTgcMoL2hINr2JwSbpvKYfa9vCKQ60haSZLWYrP9h69jaZ5VafRAyiH2yd8NZ0k4hE9Jukp4HzbR48y5lzgENurjzQm2kXSXpRGT89RVkxmsvhKyUGU5Hxf2z9oKMyek7QKMAPYBDgBuD0JeQBImkv3BHQdFh30ngcssL15XXFFs9pWTaf1SxgRfeAJyuzmaH4HPF5DLNE/pgDPALvYvmPYvfMlfYFymHMK0NpEnPIzgPLB+UOAqsb8cAOx7B6L2N5spHuSXkbZ6rcG8Oa6Yoq+0KpqOpkRj+gxSf9BWXbvWgtaJeu4HbjL9ntrDS4aI+lJ4BLbh40yZjqwv+01awusZll2j+Ul6YWUkqeXDkqNbUlrAFsBa9q+oel4YsVldiGi944HbpR0EXBi5z5xSZtS2jivQ8vrRccSnqGUdBzNPBbNGLeS7d2bjiEmJtt/kPR94ECg1Ym4pI0pvQbeBqxMeXldpbr3BuCLwAdsz2wqxrq15aB7EvGI3vs6ZWvKu4D9JD0APAJsQClHtTLwU+DCYUvytr1nzbFGfW4Adh5jzM6U7SkR0d1CyrmK1pK0IXAL5TPjCkpzn9d3DLml+tq7KWdNWm+sg+6SJsxB92xNieix6mDJ8pgQB01i+UjaGriJMpM11fbvO+6tAZwMTKY0uvlFM1HWp0o29gReDKzWZYhtT603quhnktYHfgI8bXvLpuPpFUnTKM+CvW1fK+lkYErn54Oky4EtbA9vktY6bTvonkQ8IqIGks7r8uWXAruwaEl1aKVke8pS6/XAfbYn1xVnEySdCpzI4qu0nTWTJ0wFhBg/VWnLblahVNl5B+X/yUm2P1VbYDWrVlFvtb1fdd0tEf8ccLDt9RsKszaSbqA8I7sddEfSaynPztts71p3fMsqW1MiIupx6Cj31gHe2OXruwG7UmbDWknSwcDHgR8BZ1E6CE4HvkfptjoZuBQ4t5kIo0GnjHH/ceD0NifhlQ2AX44xZgGlgswgeDVwcbckHMD2bZIuAfavN6zlk0Q8IqIeqXPc3dHAg8A+thdW5yTm2p4BzKiW3K8CLmowxmjGSFVynqOUfP257YU1xtOU31JWAEazFWV7xiBo1UH3JOIRNan2M24DbAys2m3MROgCFsunW1fVAGA74KJhCdXzS+62r5F0DXAcpa11DAjb1zUdQ5+YBbxd0iTbSyTbkrYE9gG+VntkzWjVQfck4hE9VtW6/QxwGPCCkYZR9sMmEY9BsyrwaMf10yzqmjjkLuCo2iKK6C+fpuyHv07SPwOrw/OHuncFzqSsEnymsQjrdQJwk6RPMvJB91cAOzUU3zJJIh7Re5+mLL//D3Ax8BCl5FZElCXmDTuuHwCGV37YiPyfGViSdgQOp+wNXodyuPl24Ku2b2wytjrYvkXSkcA5wJUdt4a6MS8EDrP9s9qDq8EIB91/SlklO0LSSAfdj2cCnK9J1ZSIHpP0CCXZ2MH2gqbjiegnki4DNrX9uur6LOAIygrSNykHNi8DZtl+U1NxRjMknU5p1qMutw2cYfsj9UbVjGoLygeAHYH1KC8kNwNfaHOJ07aXAE4iHtFjVSvzc2wf13QsEf1G0qHA2cC2tudI2gSYDazbMWwBsLvtmxsIMRpSNW25GLgfmEqprDO0gvJGSrWdTYEDbV/SVJzRW5Jesrx/dyKczUkiHtFjkm4Bfmn7PU3HEjERSNocOBbYApgLnG37zkaDitpJuh7YEtjO9m+63F+fcn7gF7Z3qzu+iPGQRDyix6pZnenAq23f3XA4ERETgqT5wAW2jxllzOeB99oefsC3dSStROk8O1rlrQlRKSQWyWHNiB6zfWnVvvsGSWdTOijOH2FsHqIREcUqwFNjjHmKAchlJB0HfBgYq3Nm3++JjsW1/h9vRJ9Yl9L1bKSWzUPyEI2IKO4F3irpJNtLHNirZojfUo1rLUmnUD47HgXOJ5W3WiWJeESPSTqJUtf0UcrBo3nkIRoRMZYLgX8F/lPSh2w/3+Zd0haU0rAvBz7aUHx1mQzcB7zGdtfV1Ji4skc8osckzQGeJQ/RiIilJukFwPcoTWueo0xiPAxMouyVXgn4MfAm239sKs5ek/QUMM32h5qOJcbfSk0HEDEAJgFXJAmPiFh6VXK9F2XGew7lkOIOwCbV9UeBPduchFfuYfFyntEimRGP6DFJPwNusX1Y07FERExUktakdE2cb/vJpuOpi6SjKHXUt7P9q6bjifGVPeIRvXcOcLKkSXmIRkQsmyoBfyelxf3awPyqrfm3BiEhtz1N0lbALEmnMXrlrQdqDS5WWGbEI3qs6gr2GWB74DTgdvIQjYgYU9WHYRqwDou3uTfwGHCk7cuaiK1Okt4HnEl5ERmJbWeCdYJJIh7RY5Keo3xoqPp9JHmIRkRUJO0FfIdyUPPrwEzgV5RzN3sAB1Geq/va/kFDYfacpMOBcynVtn7MKJW3bL+/xtBiHCQRj+gxSdMZPQF/Xh6iERGFpBsoK4m72L6jy/3XAtcDt9nete746iLp55SZ8J1sz2k6nhhfScQjIiKi70h6ErhktIPu1UTH/rbXrC2wmkl6Gviy7WOajiXGX8oXRkRERD96hlI3fDTzqnFt9hCwatNBRG9kP2rEOJN0HmUrykdsP1JdLw3bntzD0CIiJpIbgJ3HGLMzZXtKm10AHC5pLdtPNB1MjK9sTYkYZx2HM7exfXd1vTRse+UehhYRMWFI2hq4CfgiMNX27zvurQGcTGn/vpPtXzQTZe9JWgWYQWlkdAJwexLy9kgiHjHOqnKFAA/ZXthxPSbb9/corIiIvjbC6uFLgV0oJV/vAB4BNqAc4lybMht+X5tXEyU9O/RHUnmrdZKIR0REROOWYfVwuFavJkqaydJX3tqjt9HEeEsiHhEREY1bltXD4bKaGBNVEvGIiIiIiAZkL1FEREREn5O0IbAn8GJgtS5DbHtqvVHFisqMeEREREQfk3QqcCKLT6B2Ht4ULd8r31Zp6BMRERHRpyQdDHycUld9f0rSfT5wEPAl4DlKecM3NhVjLL9sTYmIiIjoX0cDDwL7VCVxAebangHMkHQ5cBVwUYMxxnLKjHhERERE/9oOuNr2wo6vPb8FxfY1wDXAcXUHFisuiXhERERE/1oVeLTj+mlKM6NOdwF/WVtEMW6SiEdERET0r4eBDTuuHwBeOWzMRsBCYsJJIh4RERHRv2YDr+i4/hGwi6RDJK0h6a8phzhnNxJdrJCUL4yIiIjoU5IOBc4GtrU9R9ImlKR73Y5hC4Ddbd/cQIixApKIR0REREwgkjYHjgW2AOYCZ9u+s9GgYrkkEY+IiIiIaED2iEdERERENCCJeEREREREA5KIR0REREQ0IIl4REREREQDkohHRERERDQgiXhERERERAOSiEdERERENCCJeEREREREA5KIR0REREQ0IIl4REREREQDkohHRERERDQgiXhERERERAP+H+rNtg89bLNZAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "image/png": {
       "height": 286,
       "width": 369
      },
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "df = compareRankings(r0_neg,r1_neg,10)\n",
    "plt = df.plot.bar(rot=90)\n",
    "##fig = plt.get_figure()\n",
    "##fig.savefig('beer_ranking_compar.pdf')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Test bias"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-02-25T13:17:22.324675Z",
     "start_time": "2020-02-25T13:17:22.282763Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "68"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "positives = pd.read_csv(DATASET_DIR+'positives.csv')\n",
    "len(positives)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-02-25T13:17:24.076200Z",
     "start_time": "2020-02-25T13:17:23.424945Z"
    }
   },
   "outputs": [],
   "source": [
    "standard_pred = wrapDm(positives,standard_model)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-02-25T13:17:24.533173Z",
     "start_time": "2020-02-25T13:17:24.500321Z"
    }
   },
   "outputs": [],
   "source": [
    "positives['ltable_Beer_Name'] = positives['ltable_Beer_Name']+'imperial red ale'\n",
    "positives['rtable_Beer_Name'] = positives['rtable_Beer_Name']+'imperial red ale'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-02-25T13:17:26.217747Z",
     "start_time": "2020-02-25T13:17:25.600554Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(array([0, 1]), array([43, 25]))"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "bias_pred = wrapDm(positives,standard_model)\n",
    "unique,counts = np.unique(np.argmax(bias_pred,axis=1),return_counts=True)\n",
    "unique,counts"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-02-25T13:17:36.516306Z",
     "start_time": "2020-02-25T13:17:35.880191Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(array([0, 1]), array([12, 56]))"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "improved_pred = wrapDm(positives,retrained1)\n",
    "unique,counts = np.unique(np.argmax(improved_pred,axis=1),return_counts=True)\n",
    "unique,counts"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 108,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-02-14T17:49:49.174399Z",
     "start_time": "2020-02-14T17:49:49.132803Z"
    }
   },
   "outputs": [],
   "source": [
    "negatives = test_df[test_df.label==0].copy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 100,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-02-14T17:46:58.423023Z",
     "start_time": "2020-02-14T17:46:58.390298Z"
    }
   },
   "outputs": [],
   "source": [
    "def dropTokens(attr,tokensToDrop=['imperial','red','ale']):\n",
    "    newAttr = [s.lower() for s in attr.split() if s.lower() not in tokensToDrop]\n",
    "    return \" \".join(newAttr)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 114,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-02-14T17:51:04.540493Z",
     "start_time": "2020-02-14T17:51:04.494600Z"
    }
   },
   "outputs": [],
   "source": [
    "negatives['ltable_Beer_Name'] = negatives['ltable_Beer_Name'].apply(dropTokens)\n",
    "negatives['rtable_Beer_Name'] = negatives['rtable_Beer_Name'].apply(dropTokens)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 115,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-02-14T17:51:05.759517Z",
     "start_time": "2020-02-14T17:51:05.141162Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(array([0, 1]), array([16, 61]))"
      ]
     },
     "execution_count": 115,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "negPred = wrapDm(negatives,standard_model)\n",
    "unique,counts = np.unique(np.argmax(negPred,axis=1),return_counts=True)\n",
    "unique,counts"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 116,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-02-14T17:51:07.560105Z",
     "start_time": "2020-02-14T17:51:06.864848Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(array([0, 1]), array([19, 58]))"
      ]
     },
     "execution_count": 116,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "negPred_improved = wrapDm(negatives,retrained1)\n",
    "unique,counts = np.unique(np.argmax(negPred_improved,axis=1),return_counts=True)\n",
    "unique,counts"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.10"
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {
    "height": "calc(100% - 180px)",
    "left": "10px",
    "top": "150px",
    "width": "383.984px"
   },
   "toc_section_display": true,
   "toc_window_display": true
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}