Bias_Calculation_NB.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# This notebook will generate the Bias based metrices results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# model choices include 'bert','bert_supervised','birnn','cnngru','birnn_att','birnn_scrat'\n",
    "# attention lambda for bert_supervised is 0.001 and birnn_scrat is 100\n",
    "#python testing_for_bias.py <model> <att_lambda>\n",
    "!python testing_for_bias.py bert_supervised 0.001"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##  The bias score calculation is based on the work by Borkan et al. (2019)\n",
    "### 1. Nuanced Metrics for Measuring Unintended Biaswith Real Data for Text Classification [https://arxiv.org/abs/1903.04561]\n",
    "### 2. Kaggle competition: https://www.kaggle.com/c/jigsaw-unintended-bias-in-toxicity-classification/overview/evaluation\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "from collections import Counter,defaultdict\n",
    "from tqdm.notebook import tqdm\n",
    "import json\n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/saha/anaconda3/envs/explain_hate/lib/python3.7/site-packages/ekphrasis/classes/tokenizer.py:225: FutureWarning: Possible nested set at position 2190\n",
      "  self.tok = re.compile(r\"({})\".format(\"|\".join(pipeline)))\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Reading twitter - 1grams ...\n",
      "Reading twitter - 2grams ...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/saha/anaconda3/envs/explain_hate/lib/python3.7/site-packages/ekphrasis/classes/exmanager.py:14: FutureWarning: Possible nested set at position 42\n",
      "  regexes = {k.lower(): re.compile(self.expressions[k]) for k, v in\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Reading english - 1grams ...\n"
     ]
    }
   ],
   "source": [
    "# get_annotated_data method is used to load the dataset\n",
    "from Preprocess.dataCollect import get_annotated_data  "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Load the dataset. For bias calculation, we will need to group hatespeech and offensive into one class (toxic) and normal as another class (non-toxic)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "dict_data_folder={\n",
    "      '2':{'data_file':'Data/dataset.json','class_label':'Data/classes_two.npy'},\n",
    "      '3':{'data_file':'Data/dataset.json','class_label':'Data/classes.npy'}\n",
    "}\n",
    "\n",
    "params = {}\n",
    "\n",
    "# We need to load the dataset with the labels as 'toxic' and 'non-toxic'. \n",
    "# We consider hatespeech and offensive as toxic and normal as non-toxic.\n",
    "params['num_classes']=2  \n",
    "params['data_file']=dict_data_folder[str(params['num_classes'])]['data_file']\n",
    "params['class_names']=dict_data_folder[str(params['num_classes'])]['class_label']\n",
    "\n",
    "data_all_labelled=get_annotated_data(params)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>post_id</th>\n",
       "      <th>text</th>\n",
       "      <th>annotatorid1</th>\n",
       "      <th>target1</th>\n",
       "      <th>label1</th>\n",
       "      <th>annotatorid2</th>\n",
       "      <th>target2</th>\n",
       "      <th>label2</th>\n",
       "      <th>annotatorid3</th>\n",
       "      <th>target3</th>\n",
       "      <th>label3</th>\n",
       "      <th>rationales</th>\n",
       "      <th>final_label</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1179055004553900032_twitter</td>\n",
       "      <td>[i, dont, think, im, getting, my, baby, them, ...</td>\n",
       "      <td>1</td>\n",
       "      <td>[None]</td>\n",
       "      <td>normal</td>\n",
       "      <td>2</td>\n",
       "      <td>[None]</td>\n",
       "      <td>normal</td>\n",
       "      <td>3</td>\n",
       "      <td>[None]</td>\n",
       "      <td>normal</td>\n",
       "      <td>[]</td>\n",
       "      <td>non-toxic</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1179063826874032128_twitter</td>\n",
       "      <td>[we, cannot, continue, calling, ourselves, fem...</td>\n",
       "      <td>1</td>\n",
       "      <td>[None]</td>\n",
       "      <td>normal</td>\n",
       "      <td>2</td>\n",
       "      <td>[None]</td>\n",
       "      <td>normal</td>\n",
       "      <td>3</td>\n",
       "      <td>[None]</td>\n",
       "      <td>normal</td>\n",
       "      <td>[]</td>\n",
       "      <td>non-toxic</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1178793830532956161_twitter</td>\n",
       "      <td>[nawt, yall, niggers, ignoring, me]</td>\n",
       "      <td>4</td>\n",
       "      <td>[African]</td>\n",
       "      <td>normal</td>\n",
       "      <td>2</td>\n",
       "      <td>[None]</td>\n",
       "      <td>normal</td>\n",
       "      <td>3</td>\n",
       "      <td>[African]</td>\n",
       "      <td>hatespeech</td>\n",
       "      <td>[]</td>\n",
       "      <td>non-toxic</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1179088797964763136_twitter</td>\n",
       "      <td>[&lt;user&gt;, i, am, bit, confused, coz, chinese, p...</td>\n",
       "      <td>1</td>\n",
       "      <td>[Asian]</td>\n",
       "      <td>hatespeech</td>\n",
       "      <td>4</td>\n",
       "      <td>[Asian]</td>\n",
       "      <td>offensive</td>\n",
       "      <td>3</td>\n",
       "      <td>[Asian]</td>\n",
       "      <td>hatespeech</td>\n",
       "      <td>[[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,...</td>\n",
       "      <td>toxic</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1179085312976445440_twitter</td>\n",
       "      <td>[this, bitch, in, whataburger, eating, a, burg...</td>\n",
       "      <td>4</td>\n",
       "      <td>[Caucasian, Women]</td>\n",
       "      <td>hatespeech</td>\n",
       "      <td>2</td>\n",
       "      <td>[Women, Caucasian]</td>\n",
       "      <td>hatespeech</td>\n",
       "      <td>3</td>\n",
       "      <td>[Women, Caucasian]</td>\n",
       "      <td>offensive</td>\n",
       "      <td>[[0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,...</td>\n",
       "      <td>toxic</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20143</th>\n",
       "      <td>9989999_gab</td>\n",
       "      <td>[if, ur, still, on, twitter, tell, carlton, i,...</td>\n",
       "      <td>217</td>\n",
       "      <td>[Men, Women, Other]</td>\n",
       "      <td>offensive</td>\n",
       "      <td>199</td>\n",
       "      <td>[None]</td>\n",
       "      <td>offensive</td>\n",
       "      <td>215</td>\n",
       "      <td>[None]</td>\n",
       "      <td>normal</td>\n",
       "      <td>[[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 0,...</td>\n",
       "      <td>toxic</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20144</th>\n",
       "      <td>9990225_gab</td>\n",
       "      <td>[when, i, first, got, on, here, and, said, i, ...</td>\n",
       "      <td>220</td>\n",
       "      <td>[African]</td>\n",
       "      <td>offensive</td>\n",
       "      <td>223</td>\n",
       "      <td>[African, Other]</td>\n",
       "      <td>offensive</td>\n",
       "      <td>231</td>\n",
       "      <td>[None]</td>\n",
       "      <td>normal</td>\n",
       "      <td>[[0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 0, 0,...</td>\n",
       "      <td>toxic</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20145</th>\n",
       "      <td>9991681_gab</td>\n",
       "      <td>[was, macht, der, moslem, wenn, der, zion, geg...</td>\n",
       "      <td>206</td>\n",
       "      <td>[Islam]</td>\n",
       "      <td>offensive</td>\n",
       "      <td>203</td>\n",
       "      <td>[Other]</td>\n",
       "      <td>normal</td>\n",
       "      <td>211</td>\n",
       "      <td>[None]</td>\n",
       "      <td>normal</td>\n",
       "      <td>[]</td>\n",
       "      <td>non-toxic</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20146</th>\n",
       "      <td>9992513_gab</td>\n",
       "      <td>[it, is, awful, look, at, world, demographics,...</td>\n",
       "      <td>209</td>\n",
       "      <td>[Hispanic]</td>\n",
       "      <td>hatespeech</td>\n",
       "      <td>253</td>\n",
       "      <td>[Asian]</td>\n",
       "      <td>hatespeech</td>\n",
       "      <td>222</td>\n",
       "      <td>[Asian]</td>\n",
       "      <td>offensive</td>\n",
       "      <td>[[0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 0, 0, 1, 1,...</td>\n",
       "      <td>toxic</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20147</th>\n",
       "      <td>9998729_gab</td>\n",
       "      <td>[the, jewish, globalist, elite, have, only, im...</td>\n",
       "      <td>200</td>\n",
       "      <td>[African, Islam]</td>\n",
       "      <td>hatespeech</td>\n",
       "      <td>202</td>\n",
       "      <td>[Islam, Jewish]</td>\n",
       "      <td>offensive</td>\n",
       "      <td>207</td>\n",
       "      <td>[African, Islam, Jewish]</td>\n",
       "      <td>offensive</td>\n",
       "      <td>[[0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 0, 0,...</td>\n",
       "      <td>toxic</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>20148 rows × 13 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                           post_id  \\\n",
       "0      1179055004553900032_twitter   \n",
       "1      1179063826874032128_twitter   \n",
       "2      1178793830532956161_twitter   \n",
       "3      1179088797964763136_twitter   \n",
       "4      1179085312976445440_twitter   \n",
       "...                            ...   \n",
       "20143                  9989999_gab   \n",
       "20144                  9990225_gab   \n",
       "20145                  9991681_gab   \n",
       "20146                  9992513_gab   \n",
       "20147                  9998729_gab   \n",
       "\n",
       "                                                    text  annotatorid1  \\\n",
       "0      [i, dont, think, im, getting, my, baby, them, ...             1   \n",
       "1      [we, cannot, continue, calling, ourselves, fem...             1   \n",
       "2                    [nawt, yall, niggers, ignoring, me]             4   \n",
       "3      [<user>, i, am, bit, confused, coz, chinese, p...             1   \n",
       "4      [this, bitch, in, whataburger, eating, a, burg...             4   \n",
       "...                                                  ...           ...   \n",
       "20143  [if, ur, still, on, twitter, tell, carlton, i,...           217   \n",
       "20144  [when, i, first, got, on, here, and, said, i, ...           220   \n",
       "20145  [was, macht, der, moslem, wenn, der, zion, geg...           206   \n",
       "20146  [it, is, awful, look, at, world, demographics,...           209   \n",
       "20147  [the, jewish, globalist, elite, have, only, im...           200   \n",
       "\n",
       "                   target1      label1  annotatorid2             target2  \\\n",
       "0                   [None]      normal             2              [None]   \n",
       "1                   [None]      normal             2              [None]   \n",
       "2                [African]      normal             2              [None]   \n",
       "3                  [Asian]  hatespeech             4             [Asian]   \n",
       "4       [Caucasian, Women]  hatespeech             2  [Women, Caucasian]   \n",
       "...                    ...         ...           ...                 ...   \n",
       "20143  [Men, Women, Other]   offensive           199              [None]   \n",
       "20144            [African]   offensive           223    [African, Other]   \n",
       "20145              [Islam]   offensive           203             [Other]   \n",
       "20146           [Hispanic]  hatespeech           253             [Asian]   \n",
       "20147     [African, Islam]  hatespeech           202     [Islam, Jewish]   \n",
       "\n",
       "           label2  annotatorid3                   target3      label3  \\\n",
       "0          normal             3                    [None]      normal   \n",
       "1          normal             3                    [None]      normal   \n",
       "2          normal             3                 [African]  hatespeech   \n",
       "3       offensive             3                   [Asian]  hatespeech   \n",
       "4      hatespeech             3        [Women, Caucasian]   offensive   \n",
       "...           ...           ...                       ...         ...   \n",
       "20143   offensive           215                    [None]      normal   \n",
       "20144   offensive           231                    [None]      normal   \n",
       "20145      normal           211                    [None]      normal   \n",
       "20146  hatespeech           222                   [Asian]   offensive   \n",
       "20147   offensive           207  [African, Islam, Jewish]   offensive   \n",
       "\n",
       "                                              rationales final_label  \n",
       "0                                                     []   non-toxic  \n",
       "1                                                     []   non-toxic  \n",
       "2                                                     []   non-toxic  \n",
       "3      [[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,...       toxic  \n",
       "4      [[0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,...       toxic  \n",
       "...                                                  ...         ...  \n",
       "20143  [[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 0,...       toxic  \n",
       "20144  [[0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 0, 0,...       toxic  \n",
       "20145                                                 []   non-toxic  \n",
       "20146  [[0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 0, 0, 1, 1,...       toxic  \n",
       "20147  [[0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 0, 0,...       toxic  \n",
       "\n",
       "[20148 rows x 13 columns]"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data_all_labelled"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### This function is used to extract the target community based on majority voting. If at least 2 annoatators (out of 3) have selected a community, then we consider it."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "def generate_target_information(dataset):\n",
    "    final_target_output = defaultdict(list)\n",
    "    all_communities_selected = []\n",
    "    \n",
    "    for each in dataset.iterrows(): \n",
    "        # All the target communities tagged for this post\n",
    "        all_targets = each[1]['target1']+each[1]['target2']+each[1]['target3']  \n",
    "        community_dict = dict(Counter(all_targets))\n",
    "        \n",
    "        # Select only those communities which are present more than once.\n",
    "        for key in community_dict:\n",
    "            if community_dict[key]>1:  \n",
    "                final_target_output[each[1]['post_id']].append(key)\n",
    "                all_communities_selected.append(key)\n",
    "        \n",
    "        # If no community is selected based on majority voting then we don't select any community\n",
    "        if each[1]['post_id'] not in final_target_output:\n",
    "            final_target_output[each[1]['post_id']].append('None')\n",
    "            all_communities_selected.append(key)\n",
    "\n",
    "    return final_target_output, all_communities_selected"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "target_information, all_communities_selected = generate_target_information(data_all_labelled)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['African',\n",
       " 'Islam',\n",
       " 'Jewish',\n",
       " 'Homosexual',\n",
       " 'Women',\n",
       " 'Refugee',\n",
       " 'Arab',\n",
       " 'Caucasian',\n",
       " 'Asian',\n",
       " 'Hispanic']"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "community_count_dict = Counter(all_communities_selected)\n",
    "\n",
    "# We remove None and Other from dictionary\n",
    "community_count_dict.pop('None')\n",
    "community_count_dict.pop('Other')\n",
    "\n",
    "# For the bias calculation, we are considering the top 10 communites based on their count\n",
    "list_selected_community = [community for community, value in community_count_dict.most_common(10)]\n",
    "list_selected_community"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Based on the top 10 communities, we filter the target_information\n",
    "# This will remove the other communities from the calculation\n",
    "\n",
    "final_target_information ={}\n",
    "for each in target_information:\n",
    "    temp = list(set(target_information[each])&set(list_selected_community))\n",
    "    if len(temp) == 0:\n",
    "        final_target_information[each] = None\n",
    "    else:\n",
    "        final_target_information[each] = temp"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Add a new column 'final_target_category' which will contain the selected target community names\n",
    "data_all_labelled['final_target_category'] = data_all_labelled['post_id'].map(final_target_information) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>post_id</th>\n",
       "      <th>text</th>\n",
       "      <th>annotatorid1</th>\n",
       "      <th>target1</th>\n",
       "      <th>label1</th>\n",
       "      <th>annotatorid2</th>\n",
       "      <th>target2</th>\n",
       "      <th>label2</th>\n",
       "      <th>annotatorid3</th>\n",
       "      <th>target3</th>\n",
       "      <th>label3</th>\n",
       "      <th>rationales</th>\n",
       "      <th>final_label</th>\n",
       "      <th>final_target_category</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1179055004553900032_twitter</td>\n",
       "      <td>[i, dont, think, im, getting, my, baby, them, ...</td>\n",
       "      <td>1</td>\n",
       "      <td>[None]</td>\n",
       "      <td>normal</td>\n",
       "      <td>2</td>\n",
       "      <td>[None]</td>\n",
       "      <td>normal</td>\n",
       "      <td>3</td>\n",
       "      <td>[None]</td>\n",
       "      <td>normal</td>\n",
       "      <td>[]</td>\n",
       "      <td>non-toxic</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>13851720_gab</td>\n",
       "      <td>[laura, loomer, raped, me, while, screaming, a...</td>\n",
       "      <td>1</td>\n",
       "      <td>[Jewish]</td>\n",
       "      <td>hatespeech</td>\n",
       "      <td>2</td>\n",
       "      <td>[Jewish]</td>\n",
       "      <td>hatespeech</td>\n",
       "      <td>3</td>\n",
       "      <td>[Jewish]</td>\n",
       "      <td>hatespeech</td>\n",
       "      <td>[[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 0, 0,...</td>\n",
       "      <td>toxic</td>\n",
       "      <td>[Jewish]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>1178818409812746240_twitter</td>\n",
       "      <td>[&lt;user&gt;, what, did, the, old, lady, do, was, s...</td>\n",
       "      <td>9</td>\n",
       "      <td>[None]</td>\n",
       "      <td>normal</td>\n",
       "      <td>10</td>\n",
       "      <td>[None]</td>\n",
       "      <td>normal</td>\n",
       "      <td>4</td>\n",
       "      <td>[None]</td>\n",
       "      <td>normal</td>\n",
       "      <td>[]</td>\n",
       "      <td>non-toxic</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25</th>\n",
       "      <td>19346774_gab</td>\n",
       "      <td>[as, much, as, i, appreciate, a, format, of, o...</td>\n",
       "      <td>9</td>\n",
       "      <td>[None]</td>\n",
       "      <td>normal</td>\n",
       "      <td>13</td>\n",
       "      <td>[None]</td>\n",
       "      <td>normal</td>\n",
       "      <td>4</td>\n",
       "      <td>[Hispanic]</td>\n",
       "      <td>offensive</td>\n",
       "      <td>[]</td>\n",
       "      <td>non-toxic</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>36</th>\n",
       "      <td>1165819983701643266_twitter</td>\n",
       "      <td>[sex, be, so, good, a, bitch, be, slow, stroki...</td>\n",
       "      <td>4</td>\n",
       "      <td>[Women]</td>\n",
       "      <td>offensive</td>\n",
       "      <td>7</td>\n",
       "      <td>[Women]</td>\n",
       "      <td>offensive</td>\n",
       "      <td>16</td>\n",
       "      <td>[None]</td>\n",
       "      <td>normal</td>\n",
       "      <td>[[0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0], [0, 0, 0, ...</td>\n",
       "      <td>toxic</td>\n",
       "      <td>[Women]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20105</th>\n",
       "      <td>9773208_gab</td>\n",
       "      <td>[it, always, women, trying, this, shit, like, ...</td>\n",
       "      <td>200</td>\n",
       "      <td>[Women]</td>\n",
       "      <td>hatespeech</td>\n",
       "      <td>202</td>\n",
       "      <td>[Women]</td>\n",
       "      <td>offensive</td>\n",
       "      <td>203</td>\n",
       "      <td>[Women]</td>\n",
       "      <td>offensive</td>\n",
       "      <td>[[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,...</td>\n",
       "      <td>toxic</td>\n",
       "      <td>[Women]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20109</th>\n",
       "      <td>9802943_gab</td>\n",
       "      <td>[because, women, would, never, lie, about, bei...</td>\n",
       "      <td>228</td>\n",
       "      <td>[Women]</td>\n",
       "      <td>offensive</td>\n",
       "      <td>222</td>\n",
       "      <td>[Women]</td>\n",
       "      <td>offensive</td>\n",
       "      <td>209</td>\n",
       "      <td>[Women]</td>\n",
       "      <td>normal</td>\n",
       "      <td>[[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,...</td>\n",
       "      <td>toxic</td>\n",
       "      <td>[Women]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20117</th>\n",
       "      <td>9826963_gab</td>\n",
       "      <td>[how, is, all, that, awesome, muslim, diversit...</td>\n",
       "      <td>228</td>\n",
       "      <td>[Islam]</td>\n",
       "      <td>offensive</td>\n",
       "      <td>222</td>\n",
       "      <td>[Islam]</td>\n",
       "      <td>offensive</td>\n",
       "      <td>209</td>\n",
       "      <td>[Islam]</td>\n",
       "      <td>offensive</td>\n",
       "      <td>[[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,...</td>\n",
       "      <td>toxic</td>\n",
       "      <td>[Islam]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20128</th>\n",
       "      <td>9867117_gab</td>\n",
       "      <td>[well, my, dear, lgbtq, brothers, and, sisters...</td>\n",
       "      <td>207</td>\n",
       "      <td>[Islam, Other]</td>\n",
       "      <td>offensive</td>\n",
       "      <td>223</td>\n",
       "      <td>[Islam]</td>\n",
       "      <td>hatespeech</td>\n",
       "      <td>231</td>\n",
       "      <td>[Islam]</td>\n",
       "      <td>hatespeech</td>\n",
       "      <td>[[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,...</td>\n",
       "      <td>toxic</td>\n",
       "      <td>[Islam]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20130</th>\n",
       "      <td>9869181_gab</td>\n",
       "      <td>[but, it, on, hbo, more, violence, and, sex, c...</td>\n",
       "      <td>223</td>\n",
       "      <td>[None]</td>\n",
       "      <td>normal</td>\n",
       "      <td>200</td>\n",
       "      <td>[None]</td>\n",
       "      <td>normal</td>\n",
       "      <td>233</td>\n",
       "      <td>[None]</td>\n",
       "      <td>normal</td>\n",
       "      <td>[]</td>\n",
       "      <td>non-toxic</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>1924 rows × 14 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                           post_id  \\\n",
       "0      1179055004553900032_twitter   \n",
       "5                     13851720_gab   \n",
       "17     1178818409812746240_twitter   \n",
       "25                    19346774_gab   \n",
       "36     1165819983701643266_twitter   \n",
       "...                            ...   \n",
       "20105                  9773208_gab   \n",
       "20109                  9802943_gab   \n",
       "20117                  9826963_gab   \n",
       "20128                  9867117_gab   \n",
       "20130                  9869181_gab   \n",
       "\n",
       "                                                    text  annotatorid1  \\\n",
       "0      [i, dont, think, im, getting, my, baby, them, ...             1   \n",
       "5      [laura, loomer, raped, me, while, screaming, a...             1   \n",
       "17     [<user>, what, did, the, old, lady, do, was, s...             9   \n",
       "25     [as, much, as, i, appreciate, a, format, of, o...             9   \n",
       "36     [sex, be, so, good, a, bitch, be, slow, stroki...             4   \n",
       "...                                                  ...           ...   \n",
       "20105  [it, always, women, trying, this, shit, like, ...           200   \n",
       "20109  [because, women, would, never, lie, about, bei...           228   \n",
       "20117  [how, is, all, that, awesome, muslim, diversit...           228   \n",
       "20128  [well, my, dear, lgbtq, brothers, and, sisters...           207   \n",
       "20130  [but, it, on, hbo, more, violence, and, sex, c...           223   \n",
       "\n",
       "              target1      label1  annotatorid2   target2      label2  \\\n",
       "0              [None]      normal             2    [None]      normal   \n",
       "5            [Jewish]  hatespeech             2  [Jewish]  hatespeech   \n",
       "17             [None]      normal            10    [None]      normal   \n",
       "25             [None]      normal            13    [None]      normal   \n",
       "36            [Women]   offensive             7   [Women]   offensive   \n",
       "...               ...         ...           ...       ...         ...   \n",
       "20105         [Women]  hatespeech           202   [Women]   offensive   \n",
       "20109         [Women]   offensive           222   [Women]   offensive   \n",
       "20117         [Islam]   offensive           222   [Islam]   offensive   \n",
       "20128  [Islam, Other]   offensive           223   [Islam]  hatespeech   \n",
       "20130          [None]      normal           200    [None]      normal   \n",
       "\n",
       "       annotatorid3     target3      label3  \\\n",
       "0                 3      [None]      normal   \n",
       "5                 3    [Jewish]  hatespeech   \n",
       "17                4      [None]      normal   \n",
       "25                4  [Hispanic]   offensive   \n",
       "36               16      [None]      normal   \n",
       "...             ...         ...         ...   \n",
       "20105           203     [Women]   offensive   \n",
       "20109           209     [Women]      normal   \n",
       "20117           209     [Islam]   offensive   \n",
       "20128           231     [Islam]  hatespeech   \n",
       "20130           233      [None]      normal   \n",
       "\n",
       "                                              rationales final_label  \\\n",
       "0                                                     []   non-toxic   \n",
       "5      [[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 0, 0,...       toxic   \n",
       "17                                                    []   non-toxic   \n",
       "25                                                    []   non-toxic   \n",
       "36     [[0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0], [0, 0, 0, ...       toxic   \n",
       "...                                                  ...         ...   \n",
       "20105  [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,...       toxic   \n",
       "20109  [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,...       toxic   \n",
       "20117  [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,...       toxic   \n",
       "20128  [[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,...       toxic   \n",
       "20130                                                 []   non-toxic   \n",
       "\n",
       "      final_target_category  \n",
       "0                      None  \n",
       "5                  [Jewish]  \n",
       "17                     None  \n",
       "25                     None  \n",
       "36                  [Women]  \n",
       "...                     ...  \n",
       "20105               [Women]  \n",
       "20109               [Women]  \n",
       "20117               [Islam]  \n",
       "20128               [Islam]  \n",
       "20130                  None  \n",
       "\n",
       "[1924 rows x 14 columns]"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# The post_id_divisions file stores the train, val, test split ids. We select only the test ids.\n",
    "postpost_id_divisions_path = './Data/post_id_divisions.json'\n",
    "\n",
    "with open(postpost_id_divisions_path, 'r') as fp:\n",
    "    post_id_dict=json.load(fp)\n",
    "\n",
    "data_all_labelled_bias = data_all_labelled[data_all_labelled['post_id'].isin(post_id_dict['test'])]\n",
    "data_all_labelled_bias"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.metrics import roc_auc_score\n",
    "\n",
    "# The names of the files which will contain the bias output for each model\n",
    "bias_score_file_mapping={\n",
    "    'CNN-GRU':'bestModel_cnn_gru_bias.json',\n",
    "    'BiRNN':'bestModel_birnn_bias.json',\n",
    "    'BiRNN-Attn':'bestModel_birnnatt_bias.json',\n",
    "    'BiRNN-HateXplain':'bestModel_birnnscrat_bias.json',\n",
    "    'BERT': 'bestModel_bert_base_uncased_Attn_train_FALSE_bias.json',\n",
    "    'BERT-HateXplain': 'bestModel_bert_base_uncased_Attn_train_TRUE_bias.json'\n",
    "}\n",
    "\n",
    "# The parent folder path of the bias output files\n",
    "parent_path = './explanations_dicts/'\n",
    "\n",
    "# The bias methods that will be considered\n",
    "method_list = ['subgroup', 'bpsn', 'bnsp']\n",
    "\n",
    "community_list = list(list_selected_community)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "# This function is used to convert the classification into a [0-1] score \n",
    "# with a value of 0 meaning non-toxic and 1 meaning toxic\n",
    "def convert_to_score(label_name, label_dict):\n",
    "    if label_name=='non-toxic':\n",
    "        return 1-label_dict[label_name]\n",
    "    else:\n",
    "        return label_dict[label_name]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Function to divide the ids into postive or class class based on the method.\n",
    "def bias_evaluation_metric(dataset, method, community):\n",
    "    positive_ids = []\n",
    "    negative_ids = []\n",
    "    if method=='subgroup':\n",
    "        for eachrow in dataset.iterrows():\n",
    "            if eachrow[1]['final_target_category'] == None:\n",
    "                continue\n",
    "            if community in eachrow[1]['final_target_category']:\n",
    "                if eachrow[1]['final_label'] =='non-toxic':\n",
    "                    negative_ids.append(eachrow[1]['post_id'])\n",
    "                else:\n",
    "                    positive_ids.append(eachrow[1]['post_id'])\n",
    "            else:\n",
    "                pass\n",
    "    elif method=='bpsn':\n",
    "        for eachrow in dataset.iterrows():\n",
    "            if eachrow[1]['final_target_category'] == None:\n",
    "                continue\n",
    "            if community in eachrow[1]['final_target_category']:\n",
    "#                 print(eachrow[1]['final_label'])\n",
    "                if eachrow[1]['final_label'] =='non-toxic':\n",
    "                    negative_ids.append(eachrow[1]['post_id'])\n",
    "                else:\n",
    "                    pass\n",
    "            else:\n",
    "                if eachrow[1]['final_label'] !='non-toxic':\n",
    "                    positive_ids.append(eachrow[1]['post_id'])\n",
    "                else:\n",
    "                    pass\n",
    "    elif method=='bnsp':\n",
    "        for eachrow in dataset.iterrows():\n",
    "            if eachrow[1]['final_target_category'] == None:\n",
    "                continue\n",
    "            if community in eachrow[1]['final_target_category']:\n",
    "                if eachrow[1]['final_label'] !='non-toxic':\n",
    "                    positive_ids.append(eachrow[1]['post_id'])\n",
    "                else:\n",
    "                    pass\n",
    "            else:\n",
    "                if eachrow[1]['final_label'] =='non-toxic':\n",
    "                    negative_ids.append(eachrow[1]['post_id'])\n",
    "                else:\n",
    "                    pass\n",
    "    else:\n",
    "        print('Incorrect option selected!!!')\n",
    "                \n",
    "    return {'positiveID':positive_ids, 'negativeID':negative_ids}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "bbe087e676d3459f9d9410782723d112",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=6.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "final_bias_dictionary = defaultdict(lambda: defaultdict(dict))\n",
    "\n",
    "# We load each of the model bias output file and compute the bias score using each method for all the community\n",
    "for each_model in tqdm(bias_score_file_mapping):\n",
    "    total_data ={}\n",
    "    with open(parent_path+bias_score_file_mapping[each_model]) as fp:\n",
    "        for line in fp:\n",
    "            data = json.loads(line)\n",
    "            total_data[data['annotation_id']] = data\n",
    "    for each_method in method_list:\n",
    "        for each_community in community_list:\n",
    "            community_data = bias_evaluation_metric(data_all_labelled_bias, each_method, each_community)\n",
    "            truth_values = []\n",
    "            prediction_values = []\n",
    "\n",
    "\n",
    "            label_to_value = {'toxic':1.0, 'non-toxic':0.0}\n",
    "            for each in community_data['positiveID']:\n",
    "                truth_values.append(label_to_value[total_data[each]['ground_truth']])\n",
    "                prediction_values.append(convert_to_score(total_data[each]['classification'], total_data[each]['classification_scores']))\n",
    "\n",
    "            for each in community_data['negativeID']:\n",
    "                truth_values.append(label_to_value[total_data[each]['ground_truth']])\n",
    "                prediction_values.append(convert_to_score(total_data[each]['classification'], total_data[each]['classification_scores']))\n",
    "\n",
    "            roc_output_value = roc_auc_score(truth_values, prediction_values)\n",
    "            final_bias_dictionary[each_model][each_method][each_community] = roc_output_value"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'%.4f'"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%precision 4"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### To combine the per-community Bias AUCs into one overall measure, we calculate their generalized mean as defined below:\n",
    "\n",
    "\n",
    "\n",
    "\\begin{equation*}\n",
    "M _ { p } \\left( m _ { s } \\right) = \\left( \\frac { 1 } { N } \\sum _ { s = 1 } ^ { N } m _ { s } ^ { p } \\right) ^ { \\frac { 1 } { p } }\n",
    "\\end{equation*}\n",
    "where, $M_p$  = the $p^\\textrm{th}$ power-mean function, $m_s$ = the bias metric $m$ calculated for subgroup $s$ and\n",
    "$N$ = number of identity subgroups (10). We use $p=-5$ as was also done in the kaggle competition.\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CNN-GRU subgroup 0.6466978879207836\n",
      "CNN-GRU bpsn 0.5693649805755786\n",
      "CNN-GRU bnsp 0.6724306506355714\n",
      "BiRNN subgroup 0.6810314474607603\n",
      "BiRNN bpsn 0.6050351375344387\n",
      "BiRNN bnsp 0.6805879961427638\n",
      "BiRNN-Attn subgroup 0.6663809312442344\n",
      "BiRNN-Attn bpsn 0.6181856498477524\n",
      "BiRNN-Attn bnsp 0.649972808754877\n",
      "BiRNN-HateXplain subgroup 0.6025971308391711\n",
      "BiRNN-HateXplain bpsn 0.48742670844544606\n",
      "BiRNN-HateXplain bnsp 0.6989208969036491\n",
      "BERT subgroup 0.7567999065169655\n",
      "BERT bpsn 0.6768417449576831\n",
      "BERT bnsp 0.7639961576439063\n",
      "BERT-HateXplain subgroup 0.7801852254796021\n",
      "BERT-HateXplain bpsn 0.6994490257746223\n",
      "BERT-HateXplain bnsp 0.7803493711295375\n"
     ]
    }
   ],
   "source": [
    "# To combine the per-identity Bias AUCs into one overall measure, we calculate their generalized mean as defined below:\n",
    "power_value = -5\n",
    "num_communities = len(community_list)\n",
    "\n",
    "for each_model in final_bias_dictionary:\n",
    "    for each_method in final_bias_dictionary[each_model]:\n",
    "        temp_value =[]\n",
    "        for each_community in final_bias_dictionary[each_model][each_method]:\n",
    "            temp_value.append(pow(final_bias_dictionary[each_model][each_method][each_community], power_value))\n",
    "        print(each_model, each_method, pow(np.sum(temp_value)/num_communities, 1/power_value))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### To plot the score for individual communities"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "import pandas as pd\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "tuple_community = []\n",
    "for each_model in final_bias_dictionary:\n",
    "    # Select the metric which you want to view. Possible values are subgroup, bpsn, bnsp\n",
    "    each_method = 'bnsp' \n",
    "    \n",
    "    for each_community in final_bias_dictionary[each_model][each_method]:\n",
    "        tuple_community.append((each_model, each_community, final_bias_dictionary[each_model][each_method][each_community]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_community_score = pd.DataFrame(tuple_community, columns=['Model', 'Community', 'AUCROC'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "bnsp\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAW8AAAGQCAYAAABh+OHYAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+j8jraAAAgAElEQVR4nOzdeXgURfrA8e+bg0C4NNxyq4EQCIhEXC/wQtEVXAVXcf2xurKeCKyKx6KoLN7oKoio6wnet4isLK6oeOAa5Y7hEJAbIzcCgZD6/VE1pDNMIGQ6menwfp4nT9I9nanq6+3qqupqMcaglFIqWBJinQGllFIHT4O3UkoFkAZvpZQKoKSyLPT99983TEpKehbogAZ8pZSqTEXAvMLCwgFdunT5JTSzTME7KSnp2caNG7dr0KDBxoSEBG3hVEqpSlJUVCT5+fmZa9eufRboHZpf1lJ0hwYNGmzRwK2UUpUrISHBNGjQYDO25qN4ftn/XwO3UkrFgou/JeK11l8rpVQAlanOO1yXoeO7+JmJ7x/u/31Zllu+fHnSdddd12L27NmpderU2VO/fv3dY8aMWdGpU6cOI0eOXDFs2LBfAPr3798iOzv7t0GDBq3v06dPq+nTp9f5+eef59aoUcOsWbMmKTs7u92qVavmRkpj7ty5KYMGDWq+ePHi6nXq1NlTq1atPXfffffqc845Z9vo0aPr3XXXXc0aNWq0u6CgQC6//PL8u+666xeAPn36tDrvvPM2X3HFFRtD35Wamtp5+/btM/3YRl4njTnJ1+3/1Q1fHXD7JyYmdklPT99hjCExMdE8/vjjy3v06PHbsmXLkq+55prmH3/88ZJJkybV7tev31FNmzbdVVBQID169Nj8zDPPrAQYPXp0vSFDhrT65ptvco8//vgdAOnp6e0nTZq0qG3btruaNm2a1aFDh+1Tpkz5CeCFF144fNKkSXXfeeedZZHyM2HChMP69+9/1A8//DC/c+fOOwG+/vrrGitWrKh28cUXbwaYNGlS7ZSUlKIePXr85tOmCpwnbvrQ12Nl4CO9yn2sLFiwoFqnTp06tGrVaufe7xs4cN3AgQPXN23aNKtmzZp7AOrWrbvnlVdeWXr99dc3X7FiRcr27dsTNm7cmNS0adNdAGPGjPn5UN6nIYEpeRcVFdG7d++ju3XrtnXFihXz5s+f/+MDDzywavXq1clpaWmFTz/9dMOdO3dKpP9NTEw0o0ePrn+gNLZv3y69evVKHzBgQH4ojSeeeGL5okWLUkLL9OrVa2NeXl7uN998k/fYY481Wbx4cbKf6xmvUlJSivLy8nIXLFiQ+49//GPV3//+92YArVq12v3xxx8vCS2XnZ29LS8vL3fu3Lm5U6dOrfuf//ynZuizRo0a7RoxYkST0tKYN29e6vfff1+9LPl5/fXX04499tht48ePTwvNy8nJSf3oo4/qhqY//fTT2tOnT691sOuqolPasQLQvHnzgry8vNzQz8CBA9eHPvv8888XLly4MPfkk0/eOnz48CZTp079KS8vL/fJJ5/8OXRc5eXl5WrgtgITvCdNmlQ7KSnJ3HLLLfmheSeccMKO1q1b70pLSys8+eSTt44dO7ZepP+9+uqrfxk3blyj3bt37zeNp59+ut6xxx677U9/+tPm0Lzjjjtu56BBg9aHL9u4ceM9LVq0KFixYsUhEby9Nm/enFi3bt1CgAULFlRLT09vH75MrVq1TPv27XcsX768WmjeGWecsXnhwoU1Zs+enRK+PMB111237p577ik1uHvST/juu+9qvfDCC8vee++9NICdO3fK/ffff8SHH354eEZGRuawYcMajx8/vsFTTz3VKCMjI/Pjjz+u1adPn1aXX355886dO2c0a9Ys64UXXji8/FtBlYX3WCmrk046aduaNWsOufPqYJWr2iQW5syZU6NTp07bS/v8jjvuWHPuuee2GTx48K/hn7Vs2XLXcccdt+3JJ5+s98c//nFzpP8HmD9/fvXOnTuXmobXokWLqhUUFCSEqgCquoKCgoSMjIzMgoIC+fXXX5MnT568cH/L5+fnJy5dujTlrLPO2hqal5CQwODBg9fec889Td59991l4f/Tv3//Dc8991yDefPmRQzuIa+++uphp5566uaOHTsWHH744YXTp09PPeWUU7bffvvtq3NycmqOHz9+OcCOHTsSatWqtWfEiBHrAP71r3/VX7duXXJOTk7erFmzql9wwQVHe6u5lD/2d6ysWLEiJSMjIzM0/dhjjy3v2bPnNu//T548uW6vXr02VWaegygwJe8DyczM3NW5c+dtTz/9dFqkz4cPH75m9OjRjYuKisr8nT169DgqPT29/VlnnXVUaN6HH354eJs2bTIzMzM7DBgw4JfU1FQDILJvjU2keUEVuhVeunTp/Pfee2/RFVdc0TrStszJyanVtm3bzBYtWnQ87bTTtrRo0aJEqevqq69e/8MPP9TKy8urFv6/SUlJDBo0aO2IESMa7y8vb775Zlq/fv02AvTp02fDhAkTIu7zSHr37r0pMTGRLl267Fy/fr2W7irA/o6V8GoTb+Du3r17m4YNG3b89NNP61555ZUbYrYCARGY4J2VlbVj9uzZqftbZvjw4Wsfe+yxJpFGSszKyirIzMzc/tJLL+29Vb7hhhuaZmRkZIZKAu3bt985c+bMvWlMnTr1p+eee27ppk2b9t6h9OrVa+PChQtz//vf/+b94x//aLZ8+fIkgLS0tMINGzYkhpZbt25d4mGHHXZQt4tBceaZZ/62cePGpDVr1uxz55adnb1twYIFuTNnzpz/2muv1f/6669reD9PTk5m4MCBpQboa6+9dsO3335be/ny5XsD68knn5yekZGRefHFF7dct25d4owZM2pff/31LZs2bZr1xBNPNP7www8PL+tFuXr16nsPDh1Rs+Lt71gJ9/nnny9ctWrVnPbt228fOnToEZWRvyALTPDu1avX1l27dsmoUaP2Njx+++23NZYuXbq3BNe5c+ed6enpOz755JO6kb7jrrvuWjN27Ni9QWPMmDGrQiUAgL/+9a/rc3Jyar3yyit7//+3336LuI26deu2/cILL1z/4IMPNgI47bTTtr7zzjtpoUbTcePG1T/xxBO3RvrfoJs5c2b1oqIiGjVqVOrFKSMjY9egQYPW3H///fsE6YEDB67/8ssv62zYsGGfEzolJcVce+2165566qlGoXlffvnlory8vNw33njj5wkTJhx+wQUXbFi9evXcVatWzV27du2cZs2a7ZoyZUqtOnXq7Nm2bdve/VW7du09W7duTQxPQ1WeshwrXsnJyTz55JMr3nnnnXrr1q3Tfbcf5arzLmvXPj8lJCQwceLEn6677rrmjz/+eOOUlBTTrFmzgjFjxqzwLnfnnXeuOemkkzIjfUd2dvbO9u3bb58/f37EEnytWrXMBx98sHjIkCHNbr311hb169ffXbNmzT1///vfV0da/q677lqbnZ2dOXLkyDX9+vXbnJOTk9qxY8d2CQkJtGzZsuCFF174Ofo131dZuvb5LVSPCbbEOm7cuGVJSfs/fG666ab8I488svGCBQtKVJFUr17dXHXVVb/ceeedzSP93+DBg3999NFHIzZcvvXWW2lDhw5d6513/vnnb3z55ZfTHn300VWjRo1qkpGRkXnTTTet6dOnz6a+ffse9e9///uwxx57bPlBrXAVUZaufX7b37ESXud92WWX/XrHHXf84v3/li1b7u7du/eGUaNGNXz44YfXVGrmA0TKcus4e/bsZZ06ddqnIVAppVTlmD17dv1OnTq1Ck0HptpEKaVUMQ3eSikVQBq8lVIqgDR4K6VUAGnwVkqpANLgrZRSAVSuft7LR2T5Osxki+FzAzckaSx93q27r9u/+xefB2r7d+3ate2oUaNWdOvWbTvYwbHOO++89EWLFs0vLf8LFiyoNm3atFrXXHPNfh+7/uKLL1L/8pe/tJ43b15u9erVzfz581POPvvsNnPmzJmflpZW6mOcZRn+t3PnzhkzZ87M298yfrv3sr6+HivDXn477oaEDd/2o0ePrucd4yaSsg4XfPfddzeaP39+jbfeemsZwLhx49LeeOONtM8++2zx/r77kUceaTRt2rRSl/niiy9Sn3/++XovvvjiitKWOZDAlLzjbUjSQ03Qt/+iRYtS3njjjQOOgdKtW7ftv/vd77befffdjQCuueaaFnfeeeeq/QXusqrswB0rQRgStqzDBQ8bNmzd3LlzU//zn//U/PXXXxNHjhzZ9Mknn4z6ga9u3bptjyZwQ4CCt1eshyQ91MXz9l+wYEG1Ll26tM3MzGyXmZnZburUqTUBhg0b1jQnJ6dWRkZG5j333NOwsLCQq6++ulmHDh3atWnTJvPhhx/eO+zCY489tmrChAkN7rjjjkZ79uyRq6++egPYEt0ZZ5xxVNeuXdu2bNmyw0033bRPXjdv3pxwwgkntMnMzGzXpk2bzJdffvmw0GepqamdwZbMunbt2rZnz55Htm7dun3v3r0jDvJVFcR6SNhXX321bseOHTPatWuXeeKJJ7ZZsWJF0oIFC6qFDxe8evXqpLPPPvuoDh06tOvQoUO7UKEjOTmZMWPG/Dxo0KCWN9xwQ7NLL73018zMzF1gX8By6aWXtujQoUO7Vq1adXjttdf2GZZj2rRpqcccc0xGu3btMjt37pwROvYnTZpU+7TTTjsa4MYbbzzioosuatW1a9e2zZo1yxo5cmTDsqxbXAwJO3fu3PZZWVn73PIaY1i2bFnzrVu31nWP3HYoKCgwsR6S9FAUT0PCumWPrF69ehHA7t27JSHBlkOOOOKIwunTpy9MTU01c+fOTenXr9+R8+bN+/Hee+9d5b2VHTVqVP26devumTdv3o87duyQ4447LqNXr15bMjIydtWvX3/PkCFD1tx2220tf/jhh3nedOfMmVNz7ty582vVqlXUuXPnzPPPP39zqPoGIDU1teijjz5anJaWVrRmzZqk448/PuPSSy/dFMpfyI8//lhj1qxZS1q1arW7S5cuGVOnTq119tlnlxgaNagqe0hY7+P4YC8YPXr02AzQo0ePbZdcckleQkICjz76aP0RI0Y0/te//rWyf//++d7hgnv16tX6xhtvXHf22WdvW7RoUbWzzz47fcmSJfPdd/yWnp6+Y/r06XUWLVpU4nhYsWJFyuzZs3/Mzc1NOfPMM9uef/75Jd7Q1alTp53fffddXnJyMu+//37tW265pVmoatBr8eLF1b/++usFmzZtSmzXrl2HoUOH5qekpOz38fe4CN6l2bhxY92CgoLqWVlZ81JSUo595513Ctu3b5/3ySef1LziiitaL1y4cJ+AHxqSdPny5SlXXnnlL5GGJB01alSTAw1Jes4552ypyHULmtCtMEA8bP/x48cvCa/zBti1a5dceeWVLXNzc2skJCTw888/R7wQfPLJJ3Xy8vJSJ06ceDjA1q1bE3Nzc6tnZGTsApgyZUrdevXq7Z49e3aNTp06FYT+7+STT97SuHHjPQC///3vN3722We1vMG7qKhIhgwZ0mzGjBm1EhIS+OWXX6qtXLkyKXw7ZGVl/XbUUUftBmjfvv32n376aZ/tEVT7O1ZC1SaR/q979+5tNm3alJSamlr0yCOPrCpPelBc5w2wdOnSan/4wx+a5efnJ+/atSuhefPmBZG+46uvvqqzaNGivSNgbtu2LXHz5s0JdevWLdq8eXPCnDlzahYWFsrq1auTQvsN7JDEiYmJZGVlFTRv3rxg1qxZJar9NmzYkHjxxRe3XrZsWXURMbt37444TvRZZ521qUaNGqZGjRqFaWlpu1euXFkinUjiutpk06ZNh9WrV2+9Gxfb7NmzJ6mgoCC5sockVSXFckjYA+Xt3nvvbdSwYcPdP/74Y+7cuXNzd+/eHfEYN8bII488sjxUj7pq1aq5F1544RaA1157re7WrVsTJ02atOiOO+5otnXr1r3fET5Ge/j0008/nbZ+/fqkuXPn/piXl5dbr1693Tt27NgnD95SVWJiIoWFhVVn8HcPv4eEXbx4cXJoGOeHHnqowYG+c+DAgS2uu+66XxYuXJj7xBNP/FxQUFDa8cAPP/zwY+h4+OWXX+bUrVu3CODmm28+om/fvutvvPHGNQMHDiwxmNqBjodbb721affu3bcuWrRo/ocffrh4165dEdMvz/EQ1yXv3bt3J1erVm1XaDo5OXnXrl27knNzcxNDw0xu27atGsDatWvrr1u3rlFhYWHyvHnzunTo0IE77riDMWPGZJ544onUqVOHOnXqADQYNGgQmZmZbN26lZSUlCz33SQnJ3dKSUnh1ltv5YEHHmh++umnA0R8tVoVc8AeCe62vwtAXl4eIkLjxo07rVixgpSUFIAutWrVombNmgBdMjIyfNv+X375ZSgbNYD6qamppKamtgvNTElJ2ZsHYwxt27YlMTGx0fjx49mzZw9AlyZNmuDqlbsA9OnThwkTJhzWu3dvkpOTWbhwIU2bNiUhIYG77rqL999/n8zMzMyLLrqIBx54oPO9995LnTp1mDFjBhs2bKhXo0YNPv30U55//nmA5qHtY4zhyCOPJCUl5fBp06axevXqvesYWsa7nYDQtmkAtCrHvqtsvhwr4f8T2v/Jyck8++yzZGVl8dBDDzVKS0vDu72OPvpo8vL2tvu2AFp40wNKHGu7d+8mIyPjaIB33nmH1NRUgC5NmzZly5YtAM0AzjnnHMaOHXvs0KFDAZg1axbHHHMMc+fOZfr06cyaNYtq1arx1ltvMXXq1C49evSgdu3aTJkypd4NN9zQaunSpaxdu5bOnTtnzpgxY29+i4qKaNeuXT2g+WuvvUZysn9lwnIF77J07SurnTt3Vtu8aFH6gZYrKChI+MMf/lBTRI40xhSFD0nauHHjXxs1alSQlJTUKCEhoS7ANddcw6hRo1i2bFmJ76pWrRqDBg1i8ODBEdO68sorGTlyZDSrVaG6f/F5pae5Y8cOjjnmGMCWUl566SUSE/c/3HIstv91111Hnz59GD9+PD179gydRHTs2JHExEQ6derE5ZdfzuDBg1m2bBnHHnssxhgaNGjA+++/z4MPPsgFF1xAZqatQr377rv3/g9A165d6dOnDytXruSyyy4jOzu7RPp/+tOf6NWrF1lZWWRnZ5ORkVGu9fDLsJffrvQ093es/PTTT3s/A/jLX/7CoEGDSvx/kyZN6NevH2PHjuXOO++MKi933303F110EYcffjinn346S5cuBaBXr1707duXDz74gDFjxjB69Giuv/56OnbsSGFhId26dWPcuHFce+21/POf/6R6dVsbMm7cOPr378+sWbMAaNGiBV27dmXLli089dRTe5cLueWWW/jzn//MyJEj+f3vfx/VuoSL+ZCwO3furLZo0aL0SA2WS5YsaVm7du2tDRo02AAwZ86cDm3btl2QkpKy37qg3NzcLqGTTym/vPjii+Tk5PDEE0/EOisqDlx++eWcd9559O3b92D/tVyF30ANCXvYYYdtWr9+fT1jDFu2bKmZmJi450CBWymlDgUxLXkvWrSo9W+//VZ7z549SYmJiYVNmjRZbYwRgMaNG+e7roIttm7dWkdEilq1arWsdu3aB3y7u5a8lVJxzJeSd1nrvIuKiookISHB1ze2pqenL93f5yJC69atD8nXVymlVEhRUZEAJZ7kKmu1ybz8/Py67guUUkpVkqKiIsnPz68LlHhAqEwl78LCwgFr1659du3atR2I83pygPXr1+/T31IppeLBrl276h94qRKKgHmFhYUDvDPLVOcdNNnZ2SYnJyfW2VBKqUh8KVnGfSlaKaXUvjR4K6VUAGnwVkqpANLgrZRSAaTBWymlAkiDt1JKBZAGb6WUCiAN3kopFUAavJVSKoA0eCulVABp8FZKqQDS4K2UUgGkwVsppQJIg7dSSgWQBm+llAogDd5KKRVAGryVUiqANHgrpVQAlfXt8UqpKuLey/qWmB728tsxyknV8cRNH5aYHvhIrwpPU0veSikVQBq8lVIqgLTapArpMnR8ienvH+4fo5wopSqaBm9VKq0brRrC62NV1aDVJkopFUAavJVSKoA0eCulVABp8FZKqQDS4K2UUgGkwVsppQIo5sFbRHqKyAIRWSwit0X4vKWI/FdE5ojIZyLSLBb5VEqpeBLTft4ikgiMBXoAK4HvRGSiMSbXs9goYLwx5iUROR24H/i/ys9t8CwfkVViusXwuTHKiVLKb7EueXcFFhtjlhhjdgGvA+eHLZMJfOr+nhbhc6WUOuTEOng3BVZ4ple6eV6zgQvd3xcAtUWkXvgXichVIpIjIjn5+fkVklmllIoXQXg8/mbgCRG5HPgCWAXsCV/IGPMM8AxAdna2qcwMVhX6GLVSwRHr4L0KaO6Zbubm7WWMWY0reYtILaCPMWZTpeVQKaXiUKyrTb4D0kWktYhUAy4BJnoXEJH6IhLK5+3A85WcR6WUijsxLXkbYwpFZCAwBUgEnjfGzBeREUCOMWYicCpwv4gYbLXJ9THLMDrSnlIqPsS62gRjzGRgcti84Z6/3wY0QiqllEfMg7dSSlU1lXGHrsFbKaUO0ufdupeccdzNlZ6HWDdYKqWUKgcteR9CThpzUonpr274KkY5UUpFS0veSikVQBq8lVIqgKp8tUmXoeNLTH//cP8Y5UQppfyjJW+llAqgKl/yrgr0qU6lVDgteSulVABpyfsQFg8PGiilykdL3kopFUAavJVSKoA0eCulVABp8FZKqQDSBkulAmb5iKwS0y2Gzy0xrQ3RhwYN3gcQi5fy6ouAlVIHotUmSikVQBq8lVIqgDR4K6VUAB1ydd7hjT39Dq9TYvq+t8I2iTb2KKXikJa8lVIqgDR4K6VUAB1y1SZKVTXh7ya9T0/rQ4KWvJVSKoA0eCulVADp/ZVSSh1APFZNaclbKaUCSIO3UkoFUOzL/kqp/eoydHyJ6fdqxygjKq5oyVsppQJIg7dSSgWQBm+llAqgmAdvEekpIgtEZLGI3Bbh8xYiMk1EZorIHBE5Nxb5VEqpeBLT4C0iicBY4BwgE+gnIplhi90BvGmM6QxcAjxZublUSqn4E+veJl2BxcaYJQAi8jpwPpDrWcYAoXFb6wKrKzWHlUDfOVjs3sv6lpge9vLbMcqJUvEt1tUmTYEVnumVbp7X3cBlIrISmAzcEOmLROQqEckRkZz8/PyKyKtSSsWNWJe8y6If8KIx5hEROQGYICIdjDFF3oWMMc8AzwBkZ2ebGORTKRVQ4X3pv3+4f4xyUnaxLnmvApp7ppu5eV5XAm8CGGO+AaoD9Ssld0opFadiHby/A9JFpLWIVMM2SE4MW2Y5cAaAiLTDBm+tF1FKHdJiGryNMYXAQGAK8CO2V8l8ERkhIr3dYjcBfxWR2cBrwOXGGK0WUUod0mJe522MmYxtiPTOG+75Oxc4Kfz/lFLqUBbrahOllFLlEPOStzq0PXHTh7HOglKBpMFbKaXCLB+RVXLG4XUiLxhDWm2ilFIBpMFbKaUCSIO3UkoFkAZvpZQKIG2wVFEL4rgQSgWdlryVUiqAtOStlKoywp8b2LzupRLTVWl8eA3eqkKdNKbkyAb3vRV2yB3CL55QKhpabaKUUgGkJe9KEP60Vovhc2OUE6VUVaElb6WUCiAteVcQb/e592rHMCNKqSpJS95KKRVAWvKOgX16YOhuUEodJC15K6VUAGnwVkqpANLgrZRSAaTBWymlAkiDt1JKBZAGb6WUCiAN3kopFUDawVj5rsRYLnH41u1I7r2sb4npqjR0qKqatOStlFIBpMFbKaUCSIO3UkoFkNZ5K6Xilo6FXzoteSulVABp8FZKqQDS4K2UUgEU8+AtIj1FZIGILBaR2yJ8/k8RmeV+ForIpljkUyml4klMGyxFJBEYC/QAVgLfichEY0xuaBljzN88y98AdK70jCpVAfTBoH15Xx8I+75CMPxFJl/d8FVFZyluxbrk3RVYbIxZYozZBbwOnL+f5fsBr1VKzpRSKo7FOng3BVZ4ple6efsQkZZAa+DTUj6/SkRyRCQnPz/f94wqpVQ8iXXwPhiXAG8bY/ZE+tAY84wxJtsYk92gQYNKzppSSlWuqOq8RaQ6UNsYkx82vwGw1Riz8wBfsQpo7plu5uZFcglwfXnzqpTXEzd9GOssKBWVaEveo4FTIsw/GfhnGf7/OyBdRFqLSDVsgJ4YvpCIZACHA99EkVellKoyou1t0sUYc1X4TGPMeyIy8kD/bIwpFJGBwBQgEXjeGDNfREYAOcaYUCC/BHjdGGOizK+qIsJ7JXz/cP8Y5aTstLSv/BRt8E7dz2dlKtUbYyYDk8PmDQ+bvvugc6aUUlVYtNUmv4hI1/CZInIcoF0+lFKqgkRb8h4KvCkiLwLfu3nZQH9sVYdSceHzbt1Lzjju5thkRCmfRFXyNsb8DzgeEOBy9yPA8caYb6PNnFJKqciifjzeGLNORO4HjnazFpehi6BSSqkoRFXyFpEkEXkI+5TkS8B4YIWIPCQiyX5kUCml1L6ibbB8GEgDjjTGdDHGHAscBRwGjIo2c0oppSKLNnifB/zVGLM1NMMYswW4Fjg3yu9WSilVimiDt4n04Iwbf0QfqFFKqQoSbfDOFZF9Hm0TkcuAvCi/WymlVCmi7W1yPfCuiPyFkv28awAXRPndSimlShFV8DbGrAKOF5HTgfZu9mRjzH+jzplSSqlS+fIaNGPMp3hekiAihwHXG2Pu9eP7lVJKlRRtP+/mIvKMiEwSkQEiUlNEHgEWAQ39yaJSSqlw0Za8xwOfA+8APYEcYBaQZYxZG+V3K6WUKkW0wTvNM1zrFBG5CPiTMaYoyu9V6qAsH5FVYrrF8Lkxyokqzb2X9d3797CX345hTqqGqOu8ReRw7GBUAOuBuiIiAMaYDdF+v1JKqX1FG7zrYrsIimfeD+63AY6M8vuVKpeTxpxUYvo+f9rmlYob0XYVbOVTPpRSSh2EaHubnC0ifSPM7yMiPaL5bqWUUqWL9vH44djeJuE+B0ZE+d1KKaVKEW1FYIoxZp93VRpjfhWRmlF+t1IqwJ646dmu6rEAACAASURBVMNYZ6FKi7bkXUdE9rkAuBcx1Ijyu5VSSpUi2uD9LvAvbylbRGoBT7nPlFJKVYBog/cdwDrgZxH5XkR+AJYC+e4zpZRSFSDaroKFwG0icg8lX0C8I+qcKaWUKlVUwVtELgybZYDDRGSW99VoSiml/BVtb5NeEealAR1F5Eo3VKxSSimfRVttckWk+SLSEngTOD6a71dKKRVZtA2WERljfgaSK+K7lVJKVVDwFpEMoKAivlsppVT0DZYfYhspvdKAJsBl0Xy3UlXB5926F08cd3PsMqKqnGgbLEeFTRtgAzaAXwZ8E+X3K6WUiiDaBsu9g1KJSGfgUuAi7IM675TlO0SkJ/A4kAg8a4x5IMIyfwTuxl4cZhtjLo0m30qpqqHEnQ0cUnc30VabtAH6uZ9fgTcAMcacVsb/TwTGAj2AlcB3IjLRGJPrWSYduB04yRizUUT0xcZKqUNetA2WecDpwHnGmJONMWOAPQfx/12xT2QuMcbsAl4Hzg9b5q/AWGPMRgBjzC9R5lkppQIv2uB9IbAGmCYi/xKRMyj5SrQDaQqs8EyvdPO82gBtROQrEZnhqln2ISJXiUiOiOTk5+8zSq1SSlUpUQVvY8z7xphLgAxgGjAEaCgi40TkLD8yiK3aSQdOxVbP/EtEDouQl2eMMdnGmOwGDRr4lLRSSsUnX/p5G2N+M8a8aozpBTQDZgK3luFfVwHNPdPN3DyvlcBEY8xuY8xSYCE2mCul1CHL94d0jDEbXSn4jDIs/h2QLiKtRaQacAkwMWyZ97GlbkSkPrYaZYmPWVZKqcCpkCcsy8oNKTsQmAL8CLxpjJkvIiNEpLdbbAqwXkRysVUzQ40x62OTY6WUig/RPqQTNWPMZGBy2Lzhnr8NcKP7UUrFqUO5z3UsxLTkrZRSqnw0eCulVADFvNpEKRWfugwdX2L6+4f7xygnKhIteSulVABp8FZKqQDS4K2UUgGkwVsppQJIg7dSSgWQ9jZRKgrLR2SVmG4xfG6McqIONVryVkqpANLgrZRSAaTBWymlAkiDt1JKBZA2WCqlykQbZ+OLlryVUiqANHgrpVQAafBWSqkA0uCtlFIBpMFbKaUCSIO3UkoFkAZvpZQKIA3eSikVQBq8lVIqgPQJS6V8dNKYk0pM36enmKogWvJWSqkA0uCtlFIBpMFbKaUCSIO3UkoFkAZvpZQKIA3eSikVQBq8lVIqgLQTqlIHocvQ8SWm36sdo4yoQ56WvJVSKoBiHrxFpKeILBCRxSJyW4TPLxeRfBGZ5X4GxCKfSikVT2JabSIiicBYoAewEvhORCYaY3LDFn3DGDOw0jOolCqVDgUQW7EueXcFFhtjlhhjdgGvA+fHOE9KKRX3Yh28mwIrPNMr3bxwfURkjoi8LSLNI32RiFwlIjkikpOfn18ReVVKqbgR6+BdFh8CrYwxHYGpwEuRFjLGPGOMyTbGZDdo0KBSM6iUUpUt1sF7FeAtSTdz8/Yyxqw3xhS4yWeBLpWUN6WUiluxDt7fAeki0lpEqgGXABO9C4hIE89kb+DHSsyfUkrFpZg2DxtjCkVkIDAFSASeN8bMF5ERQI4xZiIwSER6A4XABuDymGVYKaXiRMz79hhjJgOTw+YN9/x9O3B7ZedLKaXiWayrTZRSSpWDBm+llAogDd5KKRVAGryVUiqANHgrpVQAafBWSqkA0uCtlFIBpMFbKaUCSIO3UkoFkAZvpZQKIA3eSikVQBq8lVIqgDR4K6VUAGnwVkqpANLgrZRSAaTBWymlAkiDt1JKBZAGb6WUCiAN3kopFUAavJVSKoA0eCulVABp8FZKqQDS4K2UUgGkwVsppQJIg7dSSgWQBm+llAogDd5KKRVAGryVUiqANHgrpVQAafBWSqkA0uCtlFIBpMFbKaUCKObBW0R6isgCEVksIrftZ7k+ImJEJLsy86eUUvEopsFbRBKBscA5QCbQT0QyIyxXGxgMfFu5OVRKqfgU65J3V2CxMWaJMWYX8DpwfoTl/gE8COyszMwppVS8inXwbgqs8EyvdPP2EpFjgebGmI/290UicpWI5IhITn5+vv85VUqpOBLr4L1fIpIAPArcdKBljTHPGGOyjTHZDRo0qPjMKaVUDMU6eK8Cmnumm7l5IbWBDsBnIrIM+B0wURstlVKHulgH7++AdBFpLSLVgEuAiaEPjTGbjTH1jTGtjDGtgBlAb2NMTmyyq5RS8SGmwdsYUwgMBKYAPwJvGmPmi8gIEekdy7wppVQ8S4p1Bowxk4HJYfOGl7LsqZWRJ6WUinexrjZRSilVDhq8lVIqgDR4K6VUAGnwVkqpANLgrZRSAaTBWymlAkiDt1JKBZAGb6WUCiAN3kopFUAavJVSKoA0eCulVABp8FZKqQDS4K2UUgGkwVsppQJIg7dSSgWQBm+llAogDd5KKRVAGryVUiqANHgrpVQAafBWSqkA0uCtlFIBpMFbKaUCSIO3UkoFkAZvpZQKIA3eSikVQBq8lVIqgDR4K6VUAGnwVkqpANLgrZRSAaTBWymlAkiDt1JKBZAGb6WUCqCYB28R6SkiC0RksYjcFuHza0RkrojMEpEvRSQzFvlUSql4EtPgLSKJwFjgHCAT6BchOL9qjMkyxhwDPAQ8WsnZVEqpuBPrkndXYLExZokxZhfwOnC+dwFjzBbPZE3AVGL+lFIqLiXFOP2mwArP9Erg+PCFROR64EagGnB65WRNKaXilxgTu4KsiPQFehpjBrjp/wOON8YMLGX5S4GzjTF/jvDZVcBVbrItsKCc2aoP/FrO/y2vWKQZq3QPpXWNVbq6rvGd7q/GmJ7RZiDWJe9VQHPPdDM3rzSvA+MifWCMeQZ4JtoMiUiOMSY72u+J9zRjle6htK6xSlfXteqm6xXrOu/vgHQRaS0i1YBLgIneBUQk3TP5e2BRJeZPKaXiUkxL3saYQhEZCEwBEoHnjTHzRWQEkGOMmQgMFJEzgd3ARmCfKhOllDrUxLraBGPMZGBy2Lzhnr8HV3KWoq56CUiasUr3UFrXWKWr61p1090rpg2WSimlyifWdd5KKaXKQYO3UkoFkAZvpZQKIA3eyjciIrHOg1KHCg3eVUh48BSRCt+/ItJURM4FMMaYygrg4eumFw51qIl5V8GKJiKJxpg9MUg3wRhTFDZPjM/de0SkmjFmV2g9RaQ+UBvYaoz5tSLS9KSdCNwHHCkiycaYD0IBvKLSdOkmGWMK3d+1gV3GmIJI27wC81Ch6xiWVrIxZndlpBWWbqWfO7E6X71pi0iKMaYgFnk4GFW65O3ZGSIiXUSkUyWlm2SMKXLpthGRBuB/yVREzgNuFpEmbj07Aj8AbwJzReSUigww7iT7O7AW+KuI/MHNr7ASuAvQhSKSICJTgQ+Ab0SkTWUE7lCJP3y7+r2+ItJQRDJcWrtFJElEbheR0SJyaeiziuI5dxJE5CoR6e2Or8pIU0TkRBHJqsj0Skk7E3ikMtMuryobvF3JaI872WYC/wJ+EJEHRKRWRabtgksiMB34CHhTRG5xn/kZ2I7CjrYYOpmfBx4BrsQG8E9CVRoVwQXSVcAgYBcwoCIDuDvBQgH6I2AD8DiwFPifiOwzImVFpO8uyA+5Y+lW2DeYR5lOLWA8cKuIHO1m/w84FagDDAPuEZEefqUZln6C59yZC1yDHUf/IRG5sBLSnA2MAWaLyAgRaVkRaXrSDsWKDthz9mfs09zxzRhT5X6ARM/fTwDPAnWxY6Nsxw5uVacC0hXP3+OwwfRobNXCNOB+z+cJPqV5DfZgGw48HvbZA8AO7MiNFbKuofUAjgDeAyYBf4i0TfxKF+gLPBU2/zlgE3BcBR9bmcA64EngLrftPwaq+5zOJdhAMhq4DXjG89lxwNvYp/yq+bmNw47hc4Fn3d/HA48BXwF9fF7XBM/fDwIvuP18kdu+jwKtKni/1gI+B26qyHR8zXOsM1CBO0OAm4B3gcae+b8DdmKv7HV9TC/J83cqcDfQ1E2nYUvI04F/VEB6g4H1wE/AkWHL3QcUAb/zcV0Tw6ZT3O9mFR3AgSvc+iwFjgr77Bn32bEVdDxVxw6cNswzf3b4hSTadDx/Xwh8iS11P+bmhS6WZ7njOMvvdXXf/wl24LhLPPOysIWhz4B+FbB9b8MOldHcM/9cYBnwcPj+jiKtAcCFYfOauPU92k0nuTyJN37E00/MM1BhKwbpwCygALg87LOu7iR/xI/A4jmhEtzJ/ZoLpr/zLHOYC7JzgeFRppfofnsvSn/GDqd7M9AobPlrvcHep7QTsBfAV7Elo2Pc/FAAfx+42K9tGzbvRmANcDVweNhnY4C2FXRMCbaUfaSbng284v7uDJznUzrekuj5wHxsibdT2HKfUTEXqhRswWcL8FDYZ+2BF912qO1jmpnAVHe+Dgj77GxgK3CvH8exO35ODptXH5gD/CnC8vdW1DEV1XrEOgM+7vzECPPauwPiDeC0sM+6ABl+pes5sf8NXA/kuhOutmfZNOA6orgF9KSX5dK40vPZDcByYCjQMML/RnXg46kqwbYjvI+9zZ2AvSgd6z5vBnyKHX+9VhTpJXm2bSs8pUzgTmzp+5rwAF5Bx1eC+5kN/AN7FzXe8/lLRHlXFekYdvP/AMzA3ll0d/MGYqtvjvDrGA6bl+KO4z3AwLDP2hFWQPApzc7uXJ0EnBv22Zm4UrGP+zQTuMD9XQNb6HoXOJHiAtl4bJWnL9WcvuY/1hnwaSd4T/JuwMlAfTcvG/iv2zHdKyh9Af4PGOeZ18AFtK/DAni5DgJKVpN0wL7F42/Y2ztvSe0qF9RGVFRQwzb+vu2ZfhvbeLgI6OzmHQE0iyIN793MJ2475roTqYX7bDiwEBgCHObzOoYukqkuD3XddH9gNfCDZ9nx2F4+5b44eo7hBGxV1yhgpOfzi4AcbAn0HWypu7MP6+lN9zJsibsNxVVhN2CHY77Ox23rPV9PAXoA1dy847EX/fepgLaasHwMwd6BX+KmW2EvytOwd+3vuW2e7D0m4+Un5hnwYQd4b+O/xN5iznOBM3SSZ2PHDP+IsNsln/JwI/CLS7e+Z35jd1LnUs4SKNAndGC76WRsT5LhnvX+G/AQxaWyW7AlCF/qminZAFwPWyJr6aZfdAf677CNS7/gU6OhO7mnYBt+a7lAWgQ84FnmYZe+b8Gb4gtHBxdEQhf/Xm7+bdhb7K+w1WTTPCd4xNLzgdbTsy9nunV+Flva/tSz3DnYC/NDRF/yFUpeIL/HNtjlYevYB+Ea9d3+LgL+6tex5NL82p2vP2Ev/M3dZ12Bl7F3bz183K/7nA9uXxbiqkuARthqmr+5cy+UX1+qHf38iXkG/Nop7kR62U23wvY9Xgm0dvNOcCdiUx/Si1QPex82eF9LyZJ2E+xFpVU50qmDLXUe65lXC9u3+UFsiWUOtiQ2A1sirBfaJt7fPm3j0IHcHPvyjCtc4AzNfwMYi0+3t9jb88886/IkttoikZKNWvX9SC8s7bZue96EbRy8ywWwI7EXkSOwpeHj/DrBsRfltzzTb7g0czzzLsRdOMuZRp2w6VB13wTPvHnYd8DeFDqWsXd07Xw8lvZWOwE1sV3z5uHqloGTsD2Iyn33FpZmaB/Vc8eVt0B0BzaAX7q//423n5hnwKcd05OS9Y/jsbc7U7At1aEAHnV3LkpWX9TEU7fsAtfn2HrYmn7s/FCesX26Q7eW12Lrtl8BHnbzqmFLgJluWvC3l8fT2Aui95b3TuAJT54+BdJ8TPNEYLb7+znshSq0DUYDF4XyUgHH1N3AI57pb4EX3N+R2hMO+pbasy2TsHdpQ4AmnmN4ptsGu0PbIcp1aoW9gzjaMy8Le9Go4dnO/8P2n1/uAlu572qw9covhs07E3jNM/2iS/M/2JJ4e++x78N6hy7+x2DvLH7G3k319uyDYdjG0iv8PpYq6ifmGSjnzvAG0ESgIdDVTT/uDvpk4K/YkstPbjqqOitK3vK9jS09TAeu9iwz1gWxIXgCeLTriy2Br6G4LrIlJUsPL2MvHBVSL4dt/P0GW9IOHfDXue37PrCNKOpgidyAVQN78V0CfOWZPxj4EXdRrqD1fQsY6v6eCbzu/q6HrZaKqrTvCSiJ2Oq8LCDds36zQsth615nENYNtBxpNsHThdMzP8P9vseTbksX5N7C3c2VM80MwurLsVUTod5J/3THVCK2zr0IWwL3pf86xVVD9bF3cX/D3lW9i71YXOo5nh8EvqioY8r3YzTWGSjHzvAG0NHYW1px0/WxvUuOd8sMxT5tGFWLPCXrfBOwpfrXgV7uYCjyHqDYngcf4UM9LNDA/W6NrRrKpTiAV8fedUzEVif40rBS2v+7g/5bbAk4dMBf7LZxuk/79ApsdVDoDuJCbPB+FlsXeieQjw+NdeHri+3Omer+HoS9CP+Ipx83tmvkC9EElrDj6Z/AB2Gf3wvc7f6+CltQKHevnfB96s6Xh/BUE2Crgj6guPfFDcBI3J2AH9sYuNcznYRt1P8UONHNGwr8EZ+qSjxpNcE2sj/kmVcLW+D5D9DPczz7fhdXUT8xz8BB7oTwhp233I4JnXxHYUtqw7C9LfKJon8m9unIUB1cKMAMB97zLPM8tm60CLjWMz/qjv3uhPoCuNmzfqFGnlAAPwvbhWzvLbhf2xpblx7eHzYdewH53K+0PPt0jvtZAHwInOo+OwVbJfSGW9f2fqbrfjdyJ/MQbDe5LGxBYCH2drs29qLsvUhGE8DFHTsvUVxNENqHTwCbsQWEzfjTq8QbvBti7xg/AM73zH/TbftHsE/mRlXH7d0+2MJHkUsjdB63whaEHsBeKDYDbfzat57tPADb0JsX9lkdinsK9YiU73j+iXkGyrlDRuEaJyN8dq8LcN9Ec9C7YDLOHXChUmB1Fyyz3PRz7mROw1ZrFAE3RLlu3pOsNvZW9nmKS+BtsCXwObh6Ss/yUTWshP+/24Y/46qkQst4tsuXbl65Dvawdb0E9wg4dgyPp1yAOTPsf3y9YLjf6diL/TpsFdtf3Enf3QXSfGzp930OsldJWAALH1Ig123HSA+G3Ip94CrqRkJKtlO0dH83xVYdfIirSsFeJEe7de3oR5ru79Cx2x5b0PF2M30Ye5c6I5rztbRt7qZrY6tQ52CrVb13PnWxBbK46gZYpvWMdQYOsBOOBp4L3yHYEthg93cixdUmoYOkNj7UN2NLKC9gx8wIlY7qYOvPz8ResUP9fx92J1y5TzbPyd0aV9WDrTNcgqtXd+uaju1a9Yr3/3za5gl4HsrAls5WYrtbhoLALcCfcF0xy5mO9+GmNtjRCa/wfH4ctqrkU6B3+DbycX3bAr9hu3v2xVZB/Q9bFRQKsq2xd0HizftBpnM79qKQFjb/K+ydVLvQ9vd5/bxVUtPdNg0ds82wF6TJePpU42lL8SHNt7BtI6E022Mbvr13r76cr2Fp18f2DDrKTadiG9U/w47REqnbYKACeMwzcIAdcTZwWdi82tgGnLsjLD+IsCcpfchDfezt9N4A7uZf5ALLkdin3RYQoRdCOdKrCyzGPup+NdACO6DWOqCLZ7lm5QkiZUi/H65E7Zn3PvYCMhzbqLOOKBrPKFn9NQs7psQmXM8Vz3LZ2KqDj3B10T6uZ2jcipER0n0OWwIfEB5UIp30ZUjrLGwJey220XkwrkrIfT4dW7ce9RO/paSfgC1ojCesygfby+UtbHfW88q7jqWkOQtbig+/Q+zgju//+ryeocDdEXtB/B5b/XOD29c1XACfRlgPmCD+xDwDZdwZQsmnzQZiu/WcS3EXp+uxD4hE3QMhPChiG7JKBHBsPeh2d+L9QhRjTFCy+iAZ2yi2BduA8wm2kW4ctpEpdX959WFde7sT6zBcvbqb/0/sRXMarqdAtOlhux8+hy393oDta3tj2PKd8anRLHQshU0/gq3bTg7b3+uw7Q1/9mEbH+W26ShsCf9H7Ng3zwMnuGWmYi/avo+hgW30/MAzfQO2euQWN93MHd/NfUzzYUr2Hb/I5aO3mw512/O7cTIDe5G8GdsoOQDb1fJ293kN99lYAlbS3mddY52B/eyEUAkhCfvI+yLgTc/no7C3u1+7oPJzNAHU873ex4Vvp7hLU33sGB6bcPWBLuic6McBiH3o4xT3dxr2ojAU+7hwHrausIAKKJ1hL46dXbrZ2DuK5AjLVSOsFBVFen/DNoge5pn/Z+xYGkMq+NhqgOu9ge2e9ilwOsXVJEnY6rK33fEV9fDB2AvwEuztezp2CIdt2ALAv7FVKkuJ4gGc/aR9E7Z65ny3zWe586cIN7wr5WhLYP/1+Y9j75qaYRspc7FVcEW4J3CJsnomQn6SsFUij7rpZHcefemOq1vceZ0Snt8g/sQ8A6XsBO9ju+Ow9Z/nYRs1vI0d57qDvp8fB31Yuj9gbyebeXb04diHHH7F365qNbDVEZtcwG7gAtkDLi+N3YH3JhVTVTIe26timTuxi7Alk79huyL29jm9TGxf3p2ENdZR3Nf3+gpYz9Ct81ps17EENz0ZW/K90eXtZRd4Utw+udKHtE/Alrj7ueljsaX7oS79j/Dn6d9I/eUbuu09AVvnHSoYvQz0D22bKNLcpz7fHTdLsE9vvu0CaRo2gJf7zq2Mx9ZRbt/OwD0MhL1AF+EZYTSadY6Hn5hnwLMhm2K7a3lvqyeEgrXb+b2wjUnvVED6oYGsQuNpvOD57GiKe5wkY1volxHFgwSEXfGxF4Y+wAp3gj2DLZH1cJ97qzCivY0PT7uhW+8TsT0sFmNLSi9ie9MsIIpb6lICyjHuhHoVd8fh+exifHoUu5T8nIUt8Y5207WwJcWvsfWkU3F13dhqq14+pfuhC2a/wzYCX+X5zI9nArx3jQOwT/qe4ebVwDXuu+nB2ItYtA/+hNfnD6H4gbnG2DvKULfW692xFfVFyntchR/Pbt6llBwX5h7gcuJwjJJyr3+sM+A2bBb2tiZ0oIUGZn+Zko+fV8MG8K+AKT6mfye2pHmkO5GnUfygzwvupJ5H8dgp9YjiwR/PQdccO9xnb4oH0WqLfTT7PXdSrPHrYHff7+021hroFH7wY+84Qt3HahJFVQmeBjLgNOACigcgOsZt65fxecAw7EUok+K7qephn5+KrQsd46YT3Lp6x0i/BhtkW0WZl1DAPAnblrEFuCbsWIiqFEjJu8bv3c+n2HaEv3iW64LthrkOf6oZI9Xnb8C2Z5zllqmBbRje6EeaYeubiR1P/m3s3WqoTepibB/y87CFwG88+6FKBPDYZ8CWZJ/FvSLMTdfH9q0tYt83XlTDllA/8SuoYeuVZwD3u4DyDLaEMBEbuNtiuwG+He2O9xx0HbGl9/9iS3qbcH2asfWizbBBdCo+1cuFneBfYp8u2+pOZu8bU97E1Rv6lF6i246hB4y8wauLW8cPcI13PqR7OfadmqExbZq7fXp62HKnYAP4PyhZ994U+/BMVA3REfKVhq2O+8iv7wz7fnH79BXPvBnuPBrkycOj+PugU3h9/inY+vzNLv1HsHXPmX6tp/t9lEvjIew4P/92P52xDZefuuN8Oj48WBVvPzHPgNugrwIfu7/nYLsStXMn+ivhBxo2gEf1uHCEPHR2B9o/sI/onoft9xsaCOl2F2QOqstapMCLraaYhXsiE1vaL8Lzei3PsuXuW7yfPE2luC6wLfaWd+/DC9jR+56PMo3bKL6beAPXX99ND8XeUfR106dgqxT8eLFADZfeX910Pew4HTOx1UChRuHQheVzt+3DXzjwRyqmcbgv9oLpVwnU22iY7vZjqE/1y+6YHoK9mP2ttGMyyjzsrz7/XewFy9dtiS3gjcLzzkl3HL1C8WBpTbBVsXsbov3en7H8iW3ixYGpAfb2dB0luxd1wta3vkQF1oF60uuCrZK5j+I68GTs4/YbCXsN1UF8b3XsLV1DN30U8G/3d4o7uCe46ePZ90EO3042bP39VM/0WIqHWU3D9jPvThQlM+yDTEXAKDc9lbCB9bGlpYUUd/X0awS5BGy98vsuiOzBXiyPJ0IVjdu351NJY1tg64A/wZ8LlfcpxtDx6h1k6gdsD4wOFHdP9G3Ux7C87K8+39eXfbvjawK240B419IB2MJIw7D5ge1VUup2qPQEI3cvSsZ2BSwEXg1b/hhsffO7VMJ75FwA/x+2CuVU7JN2c4juUfur3frdhL1QdcbekjfFViW87ln2JTxVGD6sT3h9dhvsiyqaY+vzZ1PcoDSCKBvnPPv0T9gHXf6MvX29MrSv3e/u2Coj39667kk7EVvtthHXl9nN7+oC5wfY8blfxdYNV2pdqB/rTMmnGN/D9t32jiP/Mq6u2312IxXwZiUqoT4/0ndh32j0A/aC7B3itq47pypsxMl4+UmgkpnQHhC5HbhCROphG4ruwAa1U0XkXc/ys7B1mM2wt5sVnb/vsQ1VJ2EbEldjG1JnRvGdT2O74/XBBrOfsaWUn4ClxphLAETkFWyp/K1o1iFERJKMMUVipYuIYLfhdmxJqYMxppMxpkBEbsB205sfRXp1QvuX4tEH22Hvnh4WkU7YEjnYi6RgA60vjDHGrWPorTvVgCwRSXSf/w97AV2MvdWvhn1JtBERMcYU+pWXA+Rzpw/fsUdEErAFjQLsAz+/AYhILWwh4RIReRbbWPiJMWZjtOlGyEdof/+I3a7TjTFPhfIYtky5iEiiW992wDB3nI3HPghUCNwjIieKSCr2TlKw51jVFosrBiW7F63GlgxCHfdPxTZ+vB/2PymVnMfjsV0GG0T5Pd6nJ+/C1kHegr0gvYOtpgk1hnrfl+fLk5MUv27qnxS/ZecC7Ak/EvvuzduxVVbR3F08jy1Je8cnuRTboHQ89kTbjL2Deg5751GuaqhS0vfe0dXCXhzS3Hq9E7YfkvG8rIKA1oViH/V+q5TPumP7sk8grkIXNQAADBxJREFUykGmDiI/vtbnu+8M1Vd3cvHiAUo2LvfDDq9QgO2b/yLFd5JVrqrE+xM6eCuViByFffT4NWzD1QBsw8Ib2IcVNmN7QCwxxvRy/yOmkjMrItWND6Ukb95F5B7sifVvbFDtgi0lrQOeNMYUuhJz1KVAVzLLwZam/2yMKfJ8dir2ZGuCfbLvBWNMuUrdIlIX+zBVR2zD4FHYt9ysEpFHsO9e7Iy982iDLS29aYxZWM5VC08/VDJriO1bvAP4xRiz2ZX2p2B7HFxs7J1IQmhbxOK48ouI3I+tMrjITYe2Q3NgozFmW2heJeXnCOwdZn9jzOoovqeuMWazZ7opttfIA8bexSIiHYHfjDE/iUhPbAN5HraX1EIRqWaM2RXN+sS7mARvABG5Ezvwfgds3W9jbEDbha0fXoItIbY2xqyJSSZ9FBbAR2D7PL+PDdg7PMv5drKJyJXYHgBnuulbsT0vfgP+GTrB/EhTRM7FBvDe2OqX47BVQ/nAGdi3rY+KJo1S0k1wAbkjtoS9BnsxzAGeNcZ8LiLtsQF8oTHmdL/zUBm8Fxw3Ldh67G7Y9oRfPZ/dhG2ofcKPQsBB5jOqAo+I/AF7J3pa6HtEpAv2rvV6bMn+A2xD7OHA48aYp92x3hfb/Xa0MebHqFYkACq9ztvjE+ytzvnGmEXYgPIb9rZoJXag9o5VIXBDifpYjDHDsVUMFwHXikgdz3J+lpK2AojIEBF5A9uIuAX7AMNp0aYpIsd4vmMytkrkXmPMUGzf3trYANMd6Ccijd3/SbnWJgIXuJtiL4RPGWO6Yeu1z8cOP4q7ozgP2ObuRgLHc6fwHxHp6QoCn2DHSLlTRI4RkWoiMhDbfjSlsgO3y2e5Arfn3HgfezHaKSLVQl+LbfN6BjuUwFLs/pyL7T2FMeY5bCPtMcBVIpIczXoEQizrbNh/96K6scxbBa6zt272LmzD3knhn5XjeyM9gt4G+8DN88AYirvDvU5Yv+ZypHc5tt3iHtwj1ti+t69iqyfAdunq6fbtdqIctY7SX8/WjeJhFMRt0xc926DCul7G4Ph5CXsBDj2N3B0bxH7EVi3Mxcdxdyp53WpT3A8/1BMrNM7577GvxDvHs/yr2EKCd0iNi4linPkg/cRqJ1VK96J4/QkL4B9TyluBDuL7vI2TN7pAfSWex7opbvgZhG34OTrKNA/DluTXYqu7Qq9qGwK8EbZsK6Ic1tWzjjWxDaxjsaWvFHdiz8GOfZ5DyTeTj8bzJqAg/VBKozV25LwdFI970xTbdtIFH8aUj+H6PuICdi83Pc/9pHuWSXbHwBvuM19f/xekn5jcQhq3tanA7kXxzBhjPLfvM4AdIpIUxfeFuo19h72NPgzbnvCciJzvFjtWRB7APpRyrjFmcfnXAIwxm4wxr2AHs8oFrhSRT7BjSJwoInd5ll1moqj+8jTE1cEG527YLogfYHu0/IQdXuAzYIExpp/7vwnYcXO+L2/aseTWWUTkItcYG5o/BNug/76InGKMWWWM+d79/BK7HEftUew4KQNFpJsxpgN2v34gIhlumaOwT5HWw95hFLrjo9KriGIu1lcPKqB7UVB+sEH2M9w7MaP8rmF4uo1h6/7GYatNamIfyrmDihnsPxUbTKdRPJbEamwfar/SqIV9InOUZ957FD+pOtClOQzbe+lVbM+XUNfLQFaVYB/T34m9Y6ofduysxFZd+TqoVyWtV/hLMUIl6COw3Rv/S3EVynQ8JXBsr6Yq+cj7wfyUu7Tno6+xdZRrY52RymaM2eQan6LujogdITDUeyTBGDNLRN7G9h9vY4yZKSL3mwroNmaM2Y69izrNNZj1wg4JsNKP73d3Fa+6yZGej5YDnUTkcmz7SW1s3frJLj/9jY9dLytDeK8SY8ybItICO7Z6oohMMMb86o6d17EX5l9L+754ZYwxrlGxoTFmFbDHde9bLSJfYEclNCIy0hhzioh8BswQkWOMMXNg77YKxH6tCDHrKlgiEz71pz5UhJ/gbt6N2Lrfa42n/7SITAOGGmNyKitPrvfHbuPjLbyIZGPrej/HDl1wMvaZgOnYgN0Je7H4CXjQGDPb/V+l9XOOlvci4/pMY4q7c96KffL3KWwbw3HYF/ueaoyp8CeP/eYuyKHG10eNMT+5+X/EtmcMxj4f0BC4z9gun08CNwRlf1a0uAjequzCTvBmxpiV7u8TsN0sf8A22P3PlYL/hu3NUuF3NhX9wIuIHIs9sddg+473NMZ84z47Adtg2Qfbtz1QJ7inXj8BW5ffAhvYvjXG3OyWGYx9IrYG9i7jAmOHcwgkETke++TvF9ieV2dgu/tdZoyZ7AoBj2PvKgcYN0RFkC7IFUmDd4CEgqM7wT/F3jKDLY3MEJG+2JO7O7YhNAN7gpd7XJZ44/qWv4Gtz77CeB5wCltun7uTeBLpQid2DJZvsQ+pPYR9KvU+7MuDr3bLtMfWc282UTzFGC/CLshnYt9g/4WIpBg75k5T7MM5d8Tz/owFDd4BEfaE5utu9ghs96qm2LdjfyQiadinVouwwwsE/gQPJyKdsWOOfw78K3TL7T6L+8fdw/blKdhqn5+BS7CNdBe7zyZgx4UpAv5rjLk+RlmuUGEX5MuNfUBHsF0lCz3LxfUFubIF8mmzQ42rKvEGpJ+wt5G5xphzsC3x94nI74HtxpgvjDFfVsXADeDuJK7Hdhm8MVQ/7D4LUuB+Efso+HnY0RVnYS/GiB0NsAP2AbaZ2Cdxx8UizxXN2JFDL8EO3XC3iBxlrMKw5TRwe2jJO85J8dgdCdi3hCRj+y73CmuYnIB9k8g1xpiPY5PbyuXqTAdgn8wN1IEsIu9hn0jtiS1hbnHzk7CNr2Ox7xFdKyL3YasVPjTGLItRliucq0J5AnuxureqFj78oiXvOCae8bixD+DUwHap3AU8FFbi/D/s+wt9GakvCIwx3+ICd2hsjCAQkb9h+zN3M8b8Bmz1jO1RiO1h0RCoJyI3Y5+F+KAqB24AY8wP2Ab2atiLldoPLXnHOVfiPh07lsXtbt6FwF+wdaWDKqMnSTwLQj23l4iMxQ5Ze494hi71NEgLdrzzZdjRNv8Q5F4lB8u7HYK0Xyublrzj31BsibqriKQAGGPexfaRTQFeFJFGMcxfzAXpBHfVIl2wj3djPGNOe3oSnYUd3mAAkH0oBW4oHoEzSPs1FjR4x5kIt/8PYQdX+h1uiFMAY8xb2NelbYa4eFJWlU0RtmdJexE5LDRTil/VVoRtwDzcGDPbGLMuNtmMLQ3cB6bVJnEk7AGc2gChp+dE5GXsE5Q9vE9LikjtID5hdygTkZOx/fRHAfcYYwo8n12Lrfc92xizNEZZVAGgwTtOhD1h9zL29WRFwFxjR5ELBfCzgN7GmBmxy62Klohch3168D1gIvZpyt9hH4GP6oXX6tCgwTtGSnnCTrADdS3AvgswC7ga+4b537tl3gGOxQ5Sr+PBBJTb1+dgx2mpix1cajEw0hgzL5Z5U8GgwTvGRORY10UKETkNuMsYc6qbFqAr9vVPjxljXnDzm5gq8nq4Q52IpGK7gO7EDuZVpV+aq/yjDZYx5AaOGisiF7hZtYEjwxabi+021jw0QwN31WGM2W6MWW+M+U0DtzoYGrxj6z3sGNx/Efv29a+BLSJyD9gWd2PHyv4J+2COry/vVUoFl1abxEioZ4mINAMexPb7fR775F0f7HglL2BHCLwFO6zrIfP0pFJq/zR4VzIRaWCMyXd/V3cjqHXFvvtxIvaFxFuwLxLe5X6GGPdyAaWUAg3elUpE7sQ+7vyIMWaJm3c0MBV4B1vqro4d5vTf7mm8FDf+hVJK7aV13pXrP9hHo68SkVoi0gD70t4XjX1byt+wfbvvFJHfG2MKNXArpSLRkncl87xIYCZwIfBPY8yDnqFfj8C+zmyYMWZFLPOqlIpfGrxjwAXw14B1wDnGmO1ubAtxjZj6jj6l1H5ptUkMuEefL8aOW3yHiBxtjNkTGtdEA7dS6kC05B1D7s0hY+D/27uDE4BBKIiCW0+KtbP0lJPgMYeAWZmpwNPjo6K5k4zTH9sHvmPy3mj5OeRK4mASeM3k/QPzvvfudQA9xBugkG0TgELiDVBIvAEKiTdAIfEGKCTeAIUeoVkQXDuQGAUAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 360x360 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "ax = sns.catplot(x=\"Community\", y=\"AUCROC\", hue=\"Model\",\n",
    "             data=df_community_score,\n",
    "                 legend=False,\n",
    "                kind=\"bar\");\n",
    "ax.set(ylim=(0.3, 1.0))\n",
    "ax.set_xticklabels(rotation=45, size=13, horizontalalignment='right')\n",
    "\n",
    "# sns.set(font_scale = 0.1)\n",
    "\n",
    "\n",
    "\n",
    "handles = ax._legend_data.values()\n",
    "labels = ax._legend_data.keys()\n",
    "\n",
    "ax.fig.legend(handles=handles, labels=labels, loc='upper right', ncol=3)\n",
    "ax.fig.subplots_adjust(top=0.92)\n",
    "\n",
    "ax.set(xlabel=\"\")  \n",
    "\n",
    "print(each_method)\n",
    "# plt.savefig('bias-'+each_method+'.pdf', dpi=300, transparent=True, bbox_inches='tight')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python [conda env:explain_hate]",
   "language": "python",
   "name": "conda-env-explain_hate-py"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}