AFM analysis.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import os\n",
    "import pickle\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "import scipy.stats as sps\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import tol_colors as tol"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "from collections import OrderedDict "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "import copy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 194,
   "metadata": {},
   "outputs": [],
   "source": [
    "class PixelLists():\n",
    "    def __init__(self):\n",
    "        self.pixel_list_file = {\n",
    "            \"DeepTaylor\": \"richard_1px_deletion_game_CIFAR-10-original_mean_DeepTaylor_1563260594.pkl\",\n",
    "            \"Gradient\": \"richard_1px_deletion_game_CIFAR-10-original_mean_Gradient_1563289738.pkl\",\n",
    "            \"InputTimesGradient\": \"richard_1px_deletion_game_CIFAR-10-original_mean_InputTimesGradient_1563244779.pkl\",\n",
    "            \"Shap\": \"richard_preservation_game_CIFAR-10-original_mean_Shap_1562776643.pkl\",\n",
    "            \"EdgeDetection\": \"richard_preservation_game_CIFAR-10-original_mean_EdgeDetection_1566395882.pkl\"\n",
    "        }\n",
    "        \n",
    "        self.explanation_names = {\n",
    "            \"Shap\": \"SHAP\",\n",
    "            \"DeepTaylor\": \"Deep Taylor\",\n",
    "            \"Gradient\": \"Sensitivity\",\n",
    "            \"InputTimesGradient\": \"Gradient x Input\",\n",
    "            \"EdgeDetection\": \"Sobel Edge Detector\"\n",
    "        }\n",
    "        \n",
    "        self.pixel_list_dict = dict()\n",
    "        for method in self.explanation_names.keys():\n",
    "            self.pixel_list_dict[method] = []\n",
    "        \n",
    "    def load_pixel_list(self, method):\n",
    "        fname = os.path.join(\"pixel_lists\", self.pixel_list_file[method])\n",
    "        with open(fname,\"rb\") as f:\n",
    "            self.pixel_list_dict[method] = pickle.load(f,encoding='latin1')\n",
    "            temp = []\n",
    "            for l in self.pixel_list_dict[method]:\n",
    "                d = {str(loc[0])+\",\"+str(loc[1]): loc[2] for loc in l}\n",
    "                d = OrderedDict(sorted(d.items(), key=lambda item: -item[1]))\n",
    "                temp.append(d)\n",
    "            self.pixel_list_dict[method] = temp\n",
    "    \n",
    "    def create_ordered_pixels_list(self, pix_weights, return_abs=False):\n",
    "        pixel_weight_dict = OrderedDict()\n",
    "        for i in range(pix_weights.shape[0]):\n",
    "            for j in range(pix_weights.shape[1]):\n",
    "                if(len(pix_weights[i][j].shape) > 0):\n",
    "                    attribution_value = sum(pix_weights[i][j])\n",
    "                else:\n",
    "                    attribution_value = pix_weights[i][j]\n",
    "                if return_abs:\n",
    "                    attribution_value = np.abs(attribution_value)\n",
    "                pixel_weight_dict[str(i)+\",\"+str(j)] = attribution_value\n",
    "        #return sorted(pixel_weight_list,key=lambda x: x[2],reverse=True)\n",
    "        return OrderedDict(sorted(pixel_weight_dict.items(), key=lambda item: -item[1]))\n",
    "            \n",
    "    def generate_same_random_pixel_weights(self, images_shape, rseed):\n",
    "        pixel_weight_size = list(images_shape[1:])\n",
    "        pixel_weight_size[-1] = 1\n",
    "\n",
    "        dataset_pixel_weight_lists = []\n",
    "        random_generator = np.random.RandomState(rseed)\n",
    "        pix_weights = random_generator.uniform(size=pixel_weight_size)\n",
    "\n",
    "        for image_i in range(images_shape[0]):\n",
    "            if(image_i % 100 == 0):\n",
    "                print(\"Generating Random Pixel List for:\" + str(image_i))\n",
    "            pixel_weight_dict = self.create_ordered_pixels_list(pix_weights,return_abs=False)\n",
    "            dataset_pixel_weight_lists.append(pixel_weight_dict)\n",
    "\n",
    "        return dataset_pixel_weight_lists\n",
    "            \n",
    "    def generate_random_pixel_lists(self, rseed_list, images_shape=[1,32,32,3]):\n",
    "        for rseed in rseed_list:\n",
    "            rpl = self.generate_same_random_pixel_weights(images_shape, int(rseed))\n",
    "            self.pixel_list_dict[\"random_\" + str(rseed)] = rpl\n",
    "\n",
    "\n",
    "class DeletionResults():\n",
    "    \n",
    "    def __init__(self, filter_list=[]):\n",
    "        self.class_map = {\n",
    "            0 : \"airplane\",\n",
    "            1 : \"automobile\",\n",
    "            2 : \"bird\",\n",
    "            3 : \"cat\",\n",
    "            4 : \"deer\",\n",
    "            5 : \"dog\",\n",
    "            6 : \"frog\",\n",
    "            7 : \"horse\",\n",
    "            8 : \"ship\",\n",
    "            9 : \"truck\"\n",
    "        }\n",
    "        \n",
    "        self.explanation_names = {\n",
    "            \"Shap\": \"SHAP\",\n",
    "            \"DeepTaylor\": \"Deep Taylor\",\n",
    "            \"Gradient\": \"Sensitivity\",\n",
    "            \"InputTimesGradient\": \"Gradient x Input\",\n",
    "            \"EdgeDetection\": \"Sobel Edge Detector\"\n",
    "        }\n",
    "        \n",
    "        self.filter_list = filter_list\n",
    "        \n",
    "        self.experiment_prefix = \"richard\"\n",
    "        self.method_arr = [\"Gradient\", \"Shap\", \"InputTimesGradient\", \"DeepTaylor\", \"EdgeDetection\",\"random\"]\n",
    "        self.baseline_arr = {\n",
    "            \"\": [\"123\",\"234\",\"345\",\"456\",\"567\",\"678\",\"789\",\"890\",\"901\",\"120\"],\n",
    "            \"1px\": [\"123\",\"234\",\"345\",\"456\",\"567\",\"678\",\"789\",\"890\",\"901\",\"120\"]\n",
    "        }\n",
    "        for idx in range(1,91):\n",
    "            self.baseline_arr[\"1px\"].append(str(idx*1234))\n",
    "        self.perturb_arr = [\"mean\", \"random\"]\n",
    "        self.rate_arr = [\"\", \"1px\"]\n",
    "        self.game_arr = [\"deletion_game\", \"preservation_game\"]\n",
    "        self.dataset = \"CIFAR-10-original\"\n",
    "        self.basedir = \"paper_results\"\n",
    "        self.score_arr = dict()\n",
    "        self.method_relevance = dict()\n",
    "        \n",
    "    def get_experiment_name(self, rate, game, dataset, perturb, method, baseline=\"\"):\n",
    "        if baseline != \"\":\n",
    "            baseline = \"_\" + baseline\n",
    "        if rate != \"\":\n",
    "            rate = \"_\" + rate\n",
    "        return self.experiment_prefix + rate + \"_\" + game + \"_\" + dataset + \"_\" + perturb + \"_\" + method + baseline\n",
    "    \n",
    "    def filter_correct(self, df):\n",
    "        index = np.where(df[\"ground_truth\"] == df[\"original_prediction_i\"])\n",
    "        return index, df.loc[index]\n",
    "\n",
    "    def filter_incorrect(self, df):\n",
    "        index = np.where(df[\"ground_truth\"] != df[\"original_prediction_i\"])\n",
    "        return index, df.loc[index]\n",
    "\n",
    "    def filter_confident(self, df, confidence=0.99):\n",
    "        colnames, _ = self.cols_to_plot(df)\n",
    "        index = np.where(df[colnames[0]] >= confidence)\n",
    "        return index, df.loc[index]\n",
    "\n",
    "    def filter_not_confident(self, df, confidence=0.8):\n",
    "        colnames, _ = self.cols_to_plot(df)\n",
    "        index = np.where(df[colnames[0]] < confidence)\n",
    "        return index, df.loc[index]\n",
    "    \n",
    "    def filter_no_change(self, df, ratio=0.95):\n",
    "        colnames, _ = self.cols_to_plot(df)\n",
    "        index = np.where(df[colnames[-1]] / df[colnames[0]] < ratio)\n",
    "        return index, df.loc[index]\n",
    "\n",
    "    def normalise_outputs(self, df):\n",
    "        colnames, _, colvals = self.cols_to_plot(df, return_colvals=True)\n",
    "        initial_scores = df[colnames[0]]\n",
    "        for idx in range(colvals.shape[1]):\n",
    "            colvals[:,idx] /= initial_scores\n",
    "        df_normalised = df.copy()\n",
    "        df_normalised[colnames] = colvals\n",
    "        return df_normalised\n",
    "\n",
    "    def aopc_outputs(self, df, rate, num_pixels=1024):\n",
    "        colnames, _, colvals = self.cols_to_plot(df, return_colvals=True)\n",
    "        initial_scores = df[colnames[0]]\n",
    "        for idx in range(colvals.shape[1]):\n",
    "            colvals[:,idx] = initial_scores - colvals[:,idx]\n",
    "        if rate == \"1px\":\n",
    "            L = np.arange(1,colvals.shape[1]+1,1)[None,:]\n",
    "        else:\n",
    "            L = np.arange(1,colvals.shape[1]+1,1)[None,:] #* (0.05*num_pixels)\n",
    "        colvals = np.cumsum(colvals,1) / L\n",
    "        df_normalised = df.copy()\n",
    "        df_normalised[colnames] = colvals\n",
    "        return df_normalised\n",
    "    \n",
    "    def cols_to_plot(self, df, return_colvals=False):\n",
    "        \"\"\"\n",
    "        Returns the columns to plot from Dan's results CSVs, once loaded as dataframes\n",
    "        \"\"\"\n",
    "        first = 0\n",
    "        for idx, col in enumerate(df.columns):\n",
    "            try:\n",
    "                test = float(col)\n",
    "            except ValueError:\n",
    "                pass\n",
    "            else:\n",
    "                if test == 0:\n",
    "                    first = idx\n",
    "                    break\n",
    "        if return_colvals:\n",
    "            return (list(df.columns[first:]), \\\n",
    "                    np.array([float(val) for val in df.columns[first:]]), \\\n",
    "                    df[df.columns[first:]].to_numpy())\n",
    "        return (list(df.columns[first:]), np.array([float(val) for val in df.columns[first:]]))\n",
    "    \n",
    "    def load_results(self, aopc=True):\n",
    "        baseline_dict = dict()\n",
    "        exmethod_dict = dict()\n",
    "        baseline_dict_raw = dict()\n",
    "        exmethod_dict_raw = dict()\n",
    "        for perturb in self.perturb_arr:\n",
    "            print(\"Loading perturbation type \" + perturb)\n",
    "            if perturb == \"grid\":\n",
    "                my_rate_array = [\"1px\"]\n",
    "            else:\n",
    "                my_rate_array = self.rate_arr\n",
    "            for rate in my_rate_array:\n",
    "                print(\"Loading deletion rate type \" + rate)\n",
    "                for method in self.method_arr:\n",
    "                    print(\"Loading method \" + method)\n",
    "                    if method == \"random\":\n",
    "                        my_baseline_array = self.baseline_arr[rate]\n",
    "                    else:\n",
    "                        my_baseline_array = [\"\"]\n",
    "                    for baseline in my_baseline_array:\n",
    "                        if baseline != \"\":\n",
    "                            print(\"Baseline: \" + baseline)\n",
    "                        for game in self.game_arr:\n",
    "                            if (baseline != \"\" and game == \"deletion_game\") or (baseline == \"\"):\n",
    "                                print(\"Loading game \" + game)\n",
    "                                f = self.get_experiment_name(rate, game, self.dataset, perturb, method, baseline=baseline)\n",
    "                                fname = os.path.join(self.basedir, f + \"_results.csv\")\n",
    "                                df = pd.read_csv(fname)\n",
    "                                index = np.arange(0,df.shape[0],1)\n",
    "                                idx_list = []\n",
    "                                if len(self.filter_list) != 0:\n",
    "                                    for filt in self.filter_list:\n",
    "                                        idx, df = getattr(self, filt)(df)\n",
    "                                        idx_list.append(idx)\n",
    "                                    for idx in idx_list:\n",
    "                                        index = index[idx]\n",
    "                                if aopc:\n",
    "                                    dfaopc = self.aopc_outputs(copy.deepcopy(df), rate=rate, num_pixels=32*32)\n",
    "                                else:\n",
    "                                    dfaopc = None\n",
    "                                if baseline == \"\":\n",
    "                                    exmethod_dict_raw[f] = df\n",
    "                                    exmethod_dict[f] = copy.deepcopy(dfaopc)\n",
    "                                else:\n",
    "                                    baseline_dict_raw[f] = df\n",
    "                                    baseline_dict[f] = copy.deepcopy(dfaopc)\n",
    "        self.exmethod_dict_raw = exmethod_dict_raw\n",
    "        self.baseline_dict_raw = baseline_dict_raw\n",
    "        self.exmethod_dict = exmethod_dict\n",
    "        self.baseline_dict = baseline_dict\n",
    "        self.sample_index = index\n",
    "        self.data_shape = index.shape[0]\n",
    "    \n",
    "        \n",
    "class Results():\n",
    "        \n",
    "    def __init__(self, filter_list=[], loaded_results=None, pixel_lists=None):\n",
    "        \n",
    "        self.colors = list(tol.tol_cset(\"high-contrast\"))\n",
    "        self.colors.append(\"#888888\")\n",
    "        self.linestyle = [\"-\", \"-\", \"-\", \"-\", \"--\"]\n",
    "        \n",
    "        if pixel_lists is not None:\n",
    "            self.link_pixel_lists(pixel_lists)\n",
    "        if loaded_results is not None:\n",
    "            self.link_loaded_results(loaded_results)\n",
    "        \n",
    "        self.relevance_score_dict = dict()\n",
    "        \n",
    "        \n",
    "    def link_loaded_results(self,loaded_results):\n",
    "        self.loaded_results = loaded_results\n",
    "        self.class_map = self.loaded_results.class_map\n",
    "        self.explanation_names = self.loaded_results.explanation_names\n",
    "        self.filter_list = self.loaded_results.filter_list\n",
    "        self.experiment_prefix = self.loaded_results.experiment_prefix\n",
    "        self.method_arr = self.loaded_results.method_arr\n",
    "        self.baseline_arr = self.loaded_results.baseline_arr\n",
    "        self.perturb_arr = self.loaded_results.perturb_arr\n",
    "        self.rate_arr = self.loaded_results.rate_arr\n",
    "        self.game_arr = self.loaded_results.game_arr\n",
    "        self.dataset = self.loaded_results.dataset\n",
    "        self.basedir = self.loaded_results.basedir\n",
    "        self.exmethod_dict_raw = self.loaded_results.exmethod_dict_raw\n",
    "        self.baseline_dict_raw = self.loaded_results.baseline_dict_raw\n",
    "        self.exmethod_dict = self.loaded_results.exmethod_dict\n",
    "        self.baseline_dict = self.loaded_results.baseline_dict\n",
    "        self.sample_index = self.loaded_results.sample_index\n",
    "        self.score_arr = self.loaded_results.score_arr\n",
    "        self.method_relevance = self.loaded_results.method_relevance\n",
    "        self.data_shape = self.loaded_results.data_shape\n",
    "    \n",
    "    def get_class(self, idx):\n",
    "        \"\"\"\n",
    "        Translate from class ID to label\n",
    "        \"\"\"\n",
    "        return self.class_map[idx]\n",
    "\n",
    "    def get_method_name(self, method):\n",
    "        return self.explanation_names[method]\n",
    "    \n",
    "    def cols_to_plot(self, df, return_colvals=False):\n",
    "        \"\"\"\n",
    "        Returns the columns to plot from Dan's results CSVs, once loaded as dataframes\n",
    "        \"\"\"\n",
    "        first = 0\n",
    "        for idx, col in enumerate(df.columns):\n",
    "            try:\n",
    "                test = float(col)\n",
    "            except ValueError:\n",
    "                pass\n",
    "            else:\n",
    "                if test == 0:\n",
    "                    first = idx\n",
    "                    break\n",
    "        if return_colvals:\n",
    "            return (list(df.columns[first:]), \\\n",
    "                    np.array([float(val) for val in df.columns[first:]]), \\\n",
    "                    df[df.columns[first:]].to_numpy())\n",
    "        return (list(df.columns[first:]), np.array([float(val) for val in df.columns[first:]]))\n",
    "\n",
    "    def reconstruct_heatmap(self, method, idx=0):\n",
    "        px_ids = np.array([[p[0], p[1]] for p in self.pixel_lists.pixel_list_dict[method][idx]]).flatten()\n",
    "        wh = np.max(px_ids) + 1\n",
    "        hm = np.zeros((wh,wh))\n",
    "        for p in self.pixel_lists.pixel_list_dict[method][idx]:\n",
    "            hm[p[0],p[1]] = p[2]\n",
    "        return hm\n",
    "    \n",
    "    def get_experiment_name(self, rate, game, dataset, perturb, method, baseline=\"\"):\n",
    "        if baseline != \"\":\n",
    "            baseline = \"_\" + baseline\n",
    "        if rate != \"\":\n",
    "            rate = \"_\" + rate\n",
    "        return self.experiment_prefix + rate + \"_\" + game + \"_\" + dataset + \"_\" + perturb + \"_\" + method + baseline\n",
    "        \n",
    "    def filter_by_class(self, df, class_label=None):\n",
    "        if class_label is not None:\n",
    "            return df.loc[df[\"ground_truth\"] == class_label]\n",
    "        return df\n",
    "    \n",
    "    def link_pixel_lists(self, pixel_lists):\n",
    "        self.pixel_lists = pixel_lists\n",
    "        \n",
    "    def aopc_rank(self,ex_dict, deletion_steps=100):\n",
    "        \n",
    "        scores = np.zeros((self.data_shape,len(self.method_arr)-2))\n",
    "        for idx, exmethod in enumerate(self.method_arr[:-2]):\n",
    "            experiment = self.get_experiment_name(ex_dict[\"rate\"], ex_dict[\"game\"], \"CIFAR-10-original\", \\\n",
    "                                                  ex_dict[\"perturb\"], exmethod)\n",
    "            _,_,aopc = self.cols_to_plot(self.exmethod_dict[experiment], return_colvals=True)\n",
    "            scores[:,idx] = aopc[:,deletion_steps]\n",
    "        ranks = scores.copy()\n",
    "        for idx in range(scores.shape[0]):\n",
    "            ranks[idx,:] = sps.rankdata(scores[idx,:])\n",
    "        return ranks, scores\n",
    "        \n",
    "    \n",
    "    def get_all_scores_random_orderings(self, perturbation_type=\"mean\"):\n",
    "        # method_scores = dict()\n",
    "        method_relevance = dict()\n",
    "        scores_done = False\n",
    "        for exmethod in self.method_arr[:-1]:\n",
    "            print(exmethod)\n",
    "            experiment = self.get_experiment_name(\"1px\", \"deletion_game\", \"CIFAR-10-original\", \\\n",
    "                                                  perturbation_type, exmethod)\n",
    "            method_pixel_weights = self.pixel_lists.pixel_list_dict[exmethod]\n",
    "            _, _, temp = self.cols_to_plot(self.exmethod_dict_raw[experiment], return_colvals=True)\n",
    "            if not scores_done:\n",
    "                score_arr = np.zeros((len(self.baseline_arr[\"1px\"]), temp.shape[0], temp.shape[1]-1))\n",
    "            relev_arr = np.zeros((len(self.baseline_arr[\"1px\"]), temp.shape[0], temp.shape[1]-1))\n",
    "            for ridx, rseed in enumerate(self.baseline_arr[\"1px\"]):\n",
    "                print(rseed)\n",
    "                randname = \"random_\" + str(rseed)\n",
    "                experiment_rand = self.get_experiment_name(\"1px\", \"deletion_game\", \"CIFAR-10-original\", \\\n",
    "                                                           perturbation_type, \"random\", baseline=rseed)\n",
    "                rand_pixel_ordering = self.pixel_lists.pixel_list_dict[randname]\n",
    "                if not scores_done:\n",
    "                    _, _, colvals = self.cols_to_plot(self.baseline_dict_raw[experiment_rand], \\\n",
    "                                                      return_colvals=True)\n",
    "                    score_diffs = colvals[:,0].reshape((colvals.shape[0],1)) - colvals[:,1:]\n",
    "                relevance_sums = np.zeros(score_diffs.shape)\n",
    "                for image_id in range(colvals.shape[0]):\n",
    "                    #print(image_id, len(method_pixel_weights), len(rand_pixel_ordering))\n",
    "                    rand_mpw = np.cumsum(np.array([method_pixel_weights[image_id][k] \\\n",
    "                                                   for k in rand_pixel_ordering[0].keys()]))\n",
    "                    relevance_sums[image_id,:] = rand_mpw[:score_diffs.shape[1]]\n",
    "                #print(score_arr.shape, score_diffs.shape)\n",
    "                if not scores_done:\n",
    "                    score_arr[ridx,:,:] = score_diffs\n",
    "                relev_arr[ridx,:,:] = relevance_sums\n",
    "            method_relevance[exmethod] = relev_arr\n",
    "            scores_done = True\n",
    "        #method_scores = score_arr\n",
    "        self.loaded_results.score_arr[perturbation_type] = score_arr\n",
    "        self.loaded_results.method_relevance[perturbation_type] = method_relevance\n",
    "    \n",
    "    def aix_fidelity(self, exmethod, perturbation_type=\"mean\", sensitivity_n=0):\n",
    "        if perturbation_type not in self.score_arr.keys() or \\\n",
    "           perturbation_type not in self.method_relevance.keys():\n",
    "            self.get_all_scores_random_orderings(perturbation_type=perturbation_type)\n",
    "        score_rel_corr = np.zeros((10000,1))\n",
    "        omitted = []\n",
    "        for idx in range(self.data_shape):\n",
    "            if idx % 100 == 0:\n",
    "                print(idx)\n",
    "            if(np.std(self.score_arr[perturbation_type][:,idx,sensitivity_n]) != 0 and \\\n",
    "                np.std(self.method_relevance[perturbation_type][exmethod][:,idx,sensitivity_n]) != 0):\n",
    "                score_rel_corr[idx] = np.corrcoef(self.score_arr[perturbation_type][:,idx,sensitivity_n],\\\n",
    "                                                  self.method_relevance[perturbation_type][exmethod][:,idx,sensitivity_n])[1,0]\n",
    "            else:\n",
    "                omitted.append(idx)\n",
    "        #score_rel_corr_d = np.delete(score_rel_corr, omitted, 0)\n",
    "        return score_rel_corr, omitted\n",
    "    \n",
    "    def aix_fidelity_rank(self, perturbation_type=\"mean\", sensitivity_n=0):\n",
    "        scores = np.zeros((self.data_shape,len(self.method_arr)-2))\n",
    "        to_delete = []\n",
    "        for idx, exmethod in enumerate(self.method_arr[:-2]):\n",
    "            score_rel_corr, omitted = self.aix_fidelity(exmethod,perturbation_type=perturbation_type,\\\n",
    "                                                       sensitivity_n=sensitivity_n)\n",
    "            scores[:,idx] = score_rel_corr.squeeze()\n",
    "            to_delete = to_delete + omitted\n",
    "        scores = np.delete(scores, to_delete, axis=0)\n",
    "        ranks = scores.copy()\n",
    "        for idx in range(scores.shape[0]):\n",
    "            ranks[idx,:] = sps.rankdata(scores[idx,:])\n",
    "        return ranks, scores, to_delete\n",
    "    \n",
    "    def expected_rank(self,n=None,levels=None,samples=10000):\n",
    "        if n is None:\n",
    "            n=self.data_shape\n",
    "        if levels is None:\n",
    "            levels = len(self.method_arr)-2\n",
    "        counts = np.zeros((samples, levels))\n",
    "        for idx in range(samples):\n",
    "            if idx % 100 == 0:\n",
    "                print(idx)\n",
    "            ranks = np.zeros((n,levels))\n",
    "            for jdx in range(n):\n",
    "                ranks[jdx,:] = np.random.permutation(levels)+1\n",
    "            for ldx in range(levels):\n",
    "                counts[idx,ldx] = np.sum(ranks[:,ldx]==1)\n",
    "        return counts\n",
    "    \n",
    "    \n",
    "    def plot_aix_fidelity(self, perturbation_type=\"mean\",ax=None,size_inches=(10,10),bpc=\"#116611\",sensitivity_n=0):\n",
    "        if ax is None:\n",
    "            fig, ax = plt.subplots(len(self.method_arr)-1,1,sharex=True)\n",
    "        for idx, exmethod in enumerate(self.method_arr[:-1]):\n",
    "            score_rel_corr, omitted = self.aix_fidelity(exmethod,perturbation_type=perturbation_type,\\\n",
    "                                                       sensitivity_n=sensitivity_n)\n",
    "            score_rel_corr_d = np.delete(score_rel_corr, omitted, axis=0)\n",
    "            method_lq = np.percentile(score_rel_corr_d,25)\n",
    "            method_uq = np.percentile(score_rel_corr_d,75)\n",
    "            method_median = np.median(score_rel_corr_d)\n",
    "            method_mean = np.mean(score_rel_corr_d)\n",
    "            plt.sca(ax[idx])\n",
    "            sns.distplot(score_rel_corr_d, kde=True, rug=False, color=self.colors[idx])\n",
    "            ax[idx].axvline(method_mean, color=bpc, linestyle='-')\n",
    "            ax[idx].axvline(method_median, color=bpc, linestyle='--')\n",
    "            ax[idx].axvline(method_lq, color=bpc, linestyle=':')\n",
    "            ax[idx].axvline(method_uq, color=bpc, linestyle=':')\n",
    "            print(exmethod, str(len(omitted)))\n",
    "        plt.gcf().set_size_inches(size_inches[0], size_inches[1])\n",
    "        return ax\n",
    " \n",
    "    def score_relevance_correlation(self, ex_dict):\n",
    "        experiment = self.get_experiment_name(ex_dict[\"rate\"], ex_dict[\"game\"], ex_dict[\"dataset\"], \\\n",
    "                                              ex_dict[\"perturb\"], ex_dict[\"method\"])\n",
    "        \n",
    "        _, _, colvals_ex = self.cols_to_plot(self.exmethod_dict_raw[experiment], return_colvals=True)\n",
    "        colvals_ex[colvals_ex==0] = 1e-8\n",
    "        \n",
    "        if ex_dict[\"method\"] not in self.relevance_score_dict.keys():\n",
    "            print(\"Converting relevance scores to correct format...\")\n",
    "            relmat = np.zeros((self.sample_index.shape[0], \\\n",
    "                               len(self.pixel_lists.pixel_list_dict[ex_dict[\"method\"]][0])))\n",
    "            for idx in self.sample_index:\n",
    "                relmat[idx,:] = np.array([v[2] for v in self.pixel_lists.pixel_list_dict[ex_dict[\"method\"]][idx]])\n",
    "            relmat[relmat==0] = 1e-8\n",
    "            self.relevance_score_dict[ex_dict[\"method\"]] = relmat \n",
    "        \n",
    "        r = np.zeros((colvals_ex.shape[0], 1))\n",
    "        for idx in range(len(r)):\n",
    "            r[idx] = np.corrcoef(colvals_ex[idx,1:], \\\n",
    "                                 self.relevance_score_dict[ex_dict[\"method\"]][idx,0:colvals_ex.shape[1]-1])[0,1]\n",
    "        return r\n",
    "    \n",
    "    def correlation_metric_ranking(self, ex_dict):\n",
    "        metrank = dict()\n",
    "        metrank_scores = dict()\n",
    "        metrank[\"r\"] = np.zeros((self.sample_index.shape[0], len(list(self.explanation_names.keys()))))\n",
    "        metrank[\"m\"] = np.zeros((self.sample_index.shape[0], len(list(self.explanation_names.keys()))))\n",
    "        metrank_scores[\"r\"] = np.zeros((self.sample_index.shape[0], len(list(self.explanation_names.keys()))))\n",
    "        metrank_scores[\"m\"] = np.zeros((self.sample_index.shape[0], len(list(self.explanation_names.keys()))))\n",
    "        for idx, exmethod in enumerate(self.explanation_names.keys()):\n",
    "            ex_dict[\"method\"] = exmethod\n",
    "            experiment = self.get_experiment_name(ex_dict[\"rate\"], ex_dict[\"game\"], ex_dict[\"dataset\"], \\\n",
    "                                              ex_dict[\"perturb\"], exmethod)\n",
    "            _, _, colvals_ex = self.cols_to_plot(self.exmethod_dict[experiment], return_colvals=True)\n",
    "            metrank[\"r\"][:,idx] = np.squeeze(self.score_relevance_correlation(ex_dict))\n",
    "            metrank[\"m\"][:,idx] = np.squeeze(np.sum(colvals_ex,axis=1))\n",
    "        for k,v in metrank.items():\n",
    "            for row in range(v.shape[0]):\n",
    "                metrank_scores[k][row,:] = sps.rankdata(v[row,:])\n",
    "        return metrank, metrank_scores\n",
    "    \n",
    "    def get_baseline_adjusted_scores(self, exmethod_df, experiment, class_label=None):\n",
    "        _, xvals, colvals_ex = self.cols_to_plot(exmethod_df, return_colvals=True)\n",
    "        colvals_normed = []\n",
    "        rate = \"\"\n",
    "        if \"1px\" in experiment:\n",
    "            rate = \"1px\"\n",
    "        print(\"Number of random baseline measures: \" + str(len(self.baseline_arr[rate])))\n",
    "        for b in self.baseline_arr[rate]:\n",
    "            temp = experiment.split(\"_\")\n",
    "            temp[-1] = \"random\"\n",
    "            temp.append(b)\n",
    "            experiment_bl = \"_\".join(temp) ###\n",
    "            experiment_bl = experiment_bl.replace(\"preservation\", \"deletion\")\n",
    "            baseline = self.filter_by_class(self.baseline_dict[experiment_bl], class_label=class_label)\n",
    "            _, _, colvals_bl = self.cols_to_plot(baseline, return_colvals=True)\n",
    "            colvals_normed.append(colvals_ex - colvals_bl)\n",
    "        return (np.mean(np.array(colvals_normed),axis=0), xvals)\n",
    "    \n",
    "    def plot_experiment_against_baseline(self, experiment, class_label=None, axis_label=None, \\\n",
    "                                         ax=None, legendloc='upper right', n_comparison_lines=50,ylim=[-1,1],\\\n",
    "                                         color=\"black\", plot_id=(0,0), plot_id_legend=(-1,-1),linestyle=\"-\",\\\n",
    "                                         size_inches=(10,15)):\n",
    "        if ax is None:\n",
    "            fig, ax = plt.subplots()\n",
    "        exmethod = self.filter_by_class(self.exmethod_dict[experiment], class_label=class_label)\n",
    "        colvals_mean, xvals = \\\n",
    "            self.get_baseline_adjusted_scores(exmethod, experiment, class_label=class_label)\n",
    "\n",
    "        display_legend_all_axes = False\n",
    "        if axis_label is None:\n",
    "            if class_label is not None:\n",
    "                axis_label = self.get_class(class_label)\n",
    "                display_legend_all_axes = True\n",
    "            else:\n",
    "                axis_label = \"\"\n",
    "        ax.plot(xvals, np.zeros(colvals_mean.shape[1]), color=\"black\", \\\n",
    "                linewidth=3, alpha=0.85, linestyle=\":\", label='_nolegend_')\n",
    "        if n_comparison_lines <= 100:\n",
    "            alpha=0.15\n",
    "        elif n_comparison_lines <= 500:\n",
    "            alpha=0.1\n",
    "        else:\n",
    "            alpha=0.05\n",
    "        for row in range(n_comparison_lines):\n",
    "            ax.plot(xvals, colvals_mean[row,:], color=\"gray\", \\\n",
    "                    linewidth=1, alpha=alpha, linestyle=\"-\", label='_nolegend_')\n",
    "        ax.plot(xvals, np.mean(colvals_mean,axis=0), color=color, \\\n",
    "                linewidth=3, alpha=1, linestyle=linestyle, label=axis_label)\n",
    "        if display_legend_all_axes or plot_id_legend is None or plot_id_legend == plot_id:\n",
    "            ax.legend(loc=legendloc)\n",
    "        ax.set_ylim(ylim)\n",
    "        plt.gcf().set_size_inches(size_inches[0], size_inches[1])\n",
    "        return ax\n",
    "    \n",
    "    def plot_experiment_against_baseline_by_class(self, ex_dict, ax=None,ylim=[-1,1],size_inches=(10,15),\\\n",
    "                                                  n_comparison_lines=0,legendloc=\"upper right\"):\n",
    "        if ax is None:\n",
    "            fig, ax = plt.subplots(5,2)\n",
    "            if not isinstance(ax, np.ndarray):\n",
    "                ax = np.array([[ax]])\n",
    "        experiment = self.get_experiment_name(ex_dict[\"rate\"], ex_dict[\"game\"], ex_dict[\"dataset\"], \\\n",
    "                                              ex_dict[\"perturb\"], ex_dict[\"method\"])\n",
    "        count=0\n",
    "        for ii in range(5):\n",
    "            for jj in range(2):\n",
    "                ax[ii,jj] = self.plot_experiment_against_baseline( \\\n",
    "                              experiment, class_label=count, ax=ax[ii,jj],ylim=ylim, \\\n",
    "                              n_comparison_lines=n_comparison_lines,legendloc=legendloc)\n",
    "                count += 1\n",
    "        plt.gcf().set_size_inches(size_inches[0], size_inches[1])\n",
    "        return ax\n",
    "    \n",
    "    def plot_all_methods_experiment_against_baseline_by_class(self, ex_dict, ax=None,ylim=[-1,1],\\\n",
    "                                                              size_inches=(10,15),n_comparison_lines=0,\\\n",
    "                                                              include_edgedetect=False,legendloc=\"upper right\"):\n",
    "        if ax is None:\n",
    "            fig, ax = plt.subplots(5,2)\n",
    "            if not isinstance(ax, np.ndarray):\n",
    "                ax = np.array([[ax]])\n",
    "        count=0\n",
    "        num_methods = len(self.method_arr)-1\n",
    "        if not include_edgedetect:\n",
    "            num_methods -= 1\n",
    "        for ii in range(5):\n",
    "            for jj in range(2):\n",
    "                for kk in range(num_methods):\n",
    "                    experiment = self.get_experiment_name(ex_dict[\"rate\"], ex_dict[\"game\"], ex_dict[\"dataset\"], \\\n",
    "                                                          ex_dict[\"perturb\"], self.method_arr[kk])\n",
    "                    \n",
    "                    ax[ii,jj] = self.plot_experiment_against_baseline( \\\n",
    "                                    experiment, class_label=count, \\\n",
    "                                    axis_label=self.get_method_name(self.method_arr[kk]),\\\n",
    "                                    ax=ax[ii,jj],ylim=ylim,color=self.colors[kk],\\\n",
    "                                    plot_id=(ii,jj), plot_id_legend=(0,1),linestyle=self.linestyle[kk],\\\n",
    "                                    n_comparison_lines=n_comparison_lines,legendloc=legendloc)\n",
    "                count += 1\n",
    "        plt.gcf().set_size_inches(size_inches[0], size_inches[1])\n",
    "        return ax\n",
    "    \n",
    "    def plot_all_methods_experiment_against_baseline_by_deletion(self, ex_dict, ax=None,ylim=[-1,1],\\\n",
    "                                                                 perturb=None,size_inches=(10,15),\\\n",
    "                                                                 n_comparison_lines=0,include_edgedetect=True,\\\n",
    "                                                                legendloc=\"upper right\"):\n",
    "        if perturb is None:\n",
    "            perturb = self.perturb_arr\n",
    "        if ax is None:\n",
    "            fig, ax = plt.subplots(1,len(perturb))\n",
    "            if not isinstance(ax, np.ndarray):\n",
    "                ax = np.array([[ax]])\n",
    "        count=0\n",
    "        num_methods = len(self.method_arr)-1\n",
    "        if not include_edgedetect:\n",
    "            num_methods -= 1\n",
    "        for ii in range(1):\n",
    "            for jj in range(len(perturb)):\n",
    "                for kk in range(num_methods):\n",
    "                    experiment = self.get_experiment_name(ex_dict[\"rate\"], ex_dict[\"game\"], ex_dict[\"dataset\"], \\\n",
    "                                                     perturb[jj], self.method_arr[kk])\n",
    "                    ax[jj] = self.plot_experiment_against_baseline( \\\n",
    "                                    experiment, class_label=count, \\\n",
    "                                    axis_label=self.get_method_name(self.method_arr[kk]),\\\n",
    "                                    ax=ax[jj],ylim=ylim,color=self.colors[kk],\\\n",
    "                                    plot_id=(ii,jj), plot_id_legend=(0,2),linestyle=self.linestyle[kk],\\\n",
    "                                    n_comparison_lines=n_comparison_lines,legendloc=legendloc)\n",
    "                count += 1\n",
    "        plt.gcf().set_size_inches(size_inches[0], size_inches[1])\n",
    "        return ax\n",
    "    \n",
    "    def plot_experiment_against_baseline_by_method(self, ex_dict, ax=None, ylim=[-1,1],size_inches=(10,15),\\\n",
    "                                                   n_comparison_lines=0,legendloc=\"upper right\"):\n",
    "        if ax is None:\n",
    "            fig, ax = plt.subplots(3,2)\n",
    "            if not isinstance(ax, np.ndarray):\n",
    "                ax = np.array([[ax]])\n",
    "        count=0\n",
    "        for ii in range(3):\n",
    "            for jj in range(2):\n",
    "                print(ii,jj)\n",
    "                if ii == 2 and jj == 1:\n",
    "                    break\n",
    "                experiment = self.get_experiment_name(ex_dict[\"rate\"], ex_dict[\"game\"], ex_dict[\"dataset\"], \\\n",
    "                                                 ex_dict[\"perturb\"], self.method_arr[count])\n",
    "                print(experiment)\n",
    "                ax[ii,jj] = self.plot_experiment_against_baseline( \\\n",
    "                                experiment, ax=ax[ii,jj], \\\n",
    "                                axis_label=self.get_method_name(self.method_arr[count]),ylim=ylim,\\\n",
    "                                plot_id=(ii,jj), plot_id_legend=None,n_comparison_lines=n_comparison_lines,\\\n",
    "                                                                 legendloc=legendloc)\n",
    "                count += 1\n",
    "        plt.gcf().set_size_inches(size_inches[0], size_inches[1])\n",
    "        return ax\n",
    "    \n",
    "    def plot_aopc_distribution(self, ex_dict, size_inches=(10,15),ind=-1,ylim=(0,2500),xlim=(-1,1)):\n",
    "        fig, ax = plt.subplots(2,2)\n",
    "        count=0\n",
    "        for jj in range(2):\n",
    "            for ii in range(2):\n",
    "                experiment = self.get_experiment_name(ex_dict[\"rate\"], ex_dict[\"game\"], ex_dict[\"dataset\"], \\\n",
    "                                                 ex_dict[\"perturb\"], self.method_arr[count])\n",
    "                print(experiment)\n",
    "                #_, _, colvals_ex = self.cols_to_plot(self.exmethod_dict_raw[experiment], return_colvals=True)\n",
    "                colvals_ex, _ = self.get_baseline_adjusted_scores(self.exmethod_dict[experiment], experiment)\n",
    "                colvals_ex[colvals_ex==0] = 1e-8\n",
    "                sns.distplot(colvals_ex[:,ind],ax=ax[ii,jj], kde=False, \\\n",
    "                             label=self.explanation_names[self.method_arr[count]], \\\n",
    "                                   color=self.colors[count])\n",
    "                ax[ii,jj].legend(loc=\"upper left\")\n",
    "                ax[ii,jj].set_xlim(xlim)\n",
    "                ax[ii,jj].set_ylim(ylim)\n",
    "                count += 1\n",
    "        plt.gcf().set_size_inches(size_inches[0], size_inches[1])\n",
    "        return ax"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "pl = PixelLists()\n",
    "pl.load_pixel_list(\"Gradient\")\n",
    "pl.load_pixel_list(\"InputTimesGradient\")\n",
    "pl.load_pixel_list(\"Shap\")\n",
    "pl.load_pixel_list(\"DeepTaylor\")\n",
    "pl.load_pixel_list(\"EdgeDetection\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Loading perturbation type mean\n",
      "Loading deletion rate type \n",
      "Loading method Gradient\n",
      "Loading game deletion_game\n",
      "Loading game preservation_game\n",
      "Loading method Shap\n",
      "Loading game deletion_game\n",
      "Loading game preservation_game\n",
      "Loading method InputTimesGradient\n",
      "Loading game deletion_game\n",
      "Loading game preservation_game\n",
      "Loading method DeepTaylor\n",
      "Loading game deletion_game\n",
      "Loading game preservation_game\n",
      "Loading method EdgeDetection\n",
      "Loading game deletion_game\n",
      "Loading game preservation_game\n",
      "Loading method random\n",
      "Baseline: 123\n",
      "Loading game deletion_game\n",
      "Baseline: 234\n",
      "Loading game deletion_game\n",
      "Baseline: 345\n",
      "Loading game deletion_game\n",
      "Baseline: 456\n",
      "Loading game deletion_game\n",
      "Baseline: 567\n",
      "Loading game deletion_game\n",
      "Baseline: 678\n",
      "Loading game deletion_game\n",
      "Baseline: 789\n",
      "Loading game deletion_game\n",
      "Baseline: 890\n",
      "Loading game deletion_game\n",
      "Baseline: 901\n",
      "Loading game deletion_game\n",
      "Baseline: 120\n",
      "Loading game deletion_game\n",
      "Loading deletion rate type 1px\n",
      "Loading method Gradient\n",
      "Loading game deletion_game\n",
      "Loading game preservation_game\n",
      "Loading method Shap\n",
      "Loading game deletion_game\n",
      "Loading game preservation_game\n",
      "Loading method InputTimesGradient\n",
      "Loading game deletion_game\n",
      "Loading game preservation_game\n",
      "Loading method DeepTaylor\n",
      "Loading game deletion_game\n",
      "Loading game preservation_game\n",
      "Loading method EdgeDetection\n",
      "Loading game deletion_game\n",
      "Loading game preservation_game\n",
      "Loading method random\n",
      "Baseline: 123\n",
      "Loading game deletion_game\n",
      "Baseline: 234\n",
      "Loading game deletion_game\n",
      "Baseline: 345\n",
      "Loading game deletion_game\n",
      "Baseline: 456\n",
      "Loading game deletion_game\n",
      "Baseline: 567\n",
      "Loading game deletion_game\n",
      "Baseline: 678\n",
      "Loading game deletion_game\n",
      "Baseline: 789\n",
      "Loading game deletion_game\n",
      "Baseline: 890\n",
      "Loading game deletion_game\n",
      "Baseline: 901\n",
      "Loading game deletion_game\n",
      "Baseline: 120\n",
      "Loading game deletion_game\n",
      "Baseline: 1234\n",
      "Loading game deletion_game\n",
      "Baseline: 2468\n",
      "Loading game deletion_game\n",
      "Baseline: 3702\n",
      "Loading game deletion_game\n",
      "Baseline: 4936\n",
      "Loading game deletion_game\n",
      "Baseline: 6170\n",
      "Loading game deletion_game\n",
      "Baseline: 7404\n",
      "Loading game deletion_game\n",
      "Baseline: 8638\n",
      "Loading game deletion_game\n",
      "Baseline: 9872\n",
      "Loading game deletion_game\n",
      "Baseline: 11106\n",
      "Loading game deletion_game\n",
      "Baseline: 12340\n",
      "Loading game deletion_game\n",
      "Baseline: 13574\n",
      "Loading game deletion_game\n",
      "Baseline: 14808\n",
      "Loading game deletion_game\n",
      "Baseline: 16042\n",
      "Loading game deletion_game\n",
      "Baseline: 17276\n",
      "Loading game deletion_game\n",
      "Baseline: 18510\n",
      "Loading game deletion_game\n",
      "Baseline: 19744\n",
      "Loading game deletion_game\n",
      "Baseline: 20978\n",
      "Loading game deletion_game\n",
      "Baseline: 22212\n",
      "Loading game deletion_game\n",
      "Baseline: 23446\n",
      "Loading game deletion_game\n",
      "Baseline: 24680\n",
      "Loading game deletion_game\n",
      "Baseline: 25914\n",
      "Loading game deletion_game\n",
      "Baseline: 27148\n",
      "Loading game deletion_game\n",
      "Baseline: 28382\n",
      "Loading game deletion_game\n",
      "Baseline: 29616\n",
      "Loading game deletion_game\n",
      "Baseline: 30850\n",
      "Loading game deletion_game\n",
      "Baseline: 32084\n",
      "Loading game deletion_game\n",
      "Baseline: 33318\n",
      "Loading game deletion_game\n",
      "Baseline: 34552\n",
      "Loading game deletion_game\n",
      "Baseline: 35786\n",
      "Loading game deletion_game\n",
      "Baseline: 37020\n",
      "Loading game deletion_game\n",
      "Baseline: 38254\n",
      "Loading game deletion_game\n",
      "Baseline: 39488\n",
      "Loading game deletion_game\n",
      "Baseline: 40722\n",
      "Loading game deletion_game\n",
      "Baseline: 41956\n",
      "Loading game deletion_game\n",
      "Baseline: 43190\n",
      "Loading game deletion_game\n",
      "Baseline: 44424\n",
      "Loading game deletion_game\n",
      "Baseline: 45658\n",
      "Loading game deletion_game\n",
      "Baseline: 46892\n",
      "Loading game deletion_game\n",
      "Baseline: 48126\n",
      "Loading game deletion_game\n",
      "Baseline: 49360\n",
      "Loading game deletion_game\n",
      "Baseline: 50594\n",
      "Loading game deletion_game\n",
      "Baseline: 51828\n",
      "Loading game deletion_game\n",
      "Baseline: 53062\n",
      "Loading game deletion_game\n",
      "Baseline: 54296\n",
      "Loading game deletion_game\n",
      "Baseline: 55530\n",
      "Loading game deletion_game\n",
      "Baseline: 56764\n",
      "Loading game deletion_game\n",
      "Baseline: 57998\n",
      "Loading game deletion_game\n",
      "Baseline: 59232\n",
      "Loading game deletion_game\n",
      "Baseline: 60466\n",
      "Loading game deletion_game\n",
      "Baseline: 61700\n",
      "Loading game deletion_game\n",
      "Baseline: 62934\n",
      "Loading game deletion_game\n",
      "Baseline: 64168\n",
      "Loading game deletion_game\n",
      "Baseline: 65402\n",
      "Loading game deletion_game\n",
      "Baseline: 66636\n",
      "Loading game deletion_game\n",
      "Baseline: 67870\n",
      "Loading game deletion_game\n",
      "Baseline: 69104\n",
      "Loading game deletion_game\n",
      "Baseline: 70338\n",
      "Loading game deletion_game\n",
      "Baseline: 71572\n",
      "Loading game deletion_game\n",
      "Baseline: 72806\n",
      "Loading game deletion_game\n",
      "Baseline: 74040\n",
      "Loading game deletion_game\n",
      "Baseline: 75274\n",
      "Loading game deletion_game\n",
      "Baseline: 76508\n",
      "Loading game deletion_game\n",
      "Baseline: 77742\n",
      "Loading game deletion_game\n",
      "Baseline: 78976\n",
      "Loading game deletion_game\n",
      "Baseline: 80210\n",
      "Loading game deletion_game\n",
      "Baseline: 81444\n",
      "Loading game deletion_game\n",
      "Baseline: 82678\n",
      "Loading game deletion_game\n",
      "Baseline: 83912\n",
      "Loading game deletion_game\n",
      "Baseline: 85146\n",
      "Loading game deletion_game\n",
      "Baseline: 86380\n",
      "Loading game deletion_game\n",
      "Baseline: 87614\n",
      "Loading game deletion_game\n",
      "Baseline: 88848\n",
      "Loading game deletion_game\n",
      "Baseline: 90082\n",
      "Loading game deletion_game\n",
      "Baseline: 91316\n",
      "Loading game deletion_game\n",
      "Baseline: 92550\n",
      "Loading game deletion_game\n",
      "Baseline: 93784\n",
      "Loading game deletion_game\n",
      "Baseline: 95018\n",
      "Loading game deletion_game\n",
      "Baseline: 96252\n",
      "Loading game deletion_game\n",
      "Baseline: 97486\n",
      "Loading game deletion_game\n",
      "Baseline: 98720\n",
      "Loading game deletion_game\n",
      "Baseline: 99954\n",
      "Loading game deletion_game\n",
      "Baseline: 101188\n",
      "Loading game deletion_game\n",
      "Baseline: 102422\n",
      "Loading game deletion_game\n",
      "Baseline: 103656\n",
      "Loading game deletion_game\n",
      "Baseline: 104890\n",
      "Loading game deletion_game\n",
      "Baseline: 106124\n",
      "Loading game deletion_game\n",
      "Baseline: 107358\n",
      "Loading game deletion_game\n",
      "Baseline: 108592\n",
      "Loading game deletion_game\n",
      "Baseline: 109826\n",
      "Loading game deletion_game\n",
      "Baseline: 111060\n",
      "Loading game deletion_game\n",
      "Loading perturbation type random\n",
      "Loading deletion rate type \n",
      "Loading method Gradient\n",
      "Loading game deletion_game\n",
      "Loading game preservation_game\n",
      "Loading method Shap\n",
      "Loading game deletion_game\n",
      "Loading game preservation_game\n",
      "Loading method InputTimesGradient\n",
      "Loading game deletion_game\n",
      "Loading game preservation_game\n",
      "Loading method DeepTaylor\n",
      "Loading game deletion_game\n",
      "Loading game preservation_game\n",
      "Loading method EdgeDetection\n",
      "Loading game deletion_game\n",
      "Loading game preservation_game\n",
      "Loading method random\n",
      "Baseline: 123\n",
      "Loading game deletion_game\n",
      "Baseline: 234\n",
      "Loading game deletion_game\n",
      "Baseline: 345\n",
      "Loading game deletion_game\n",
      "Baseline: 456\n",
      "Loading game deletion_game\n",
      "Baseline: 567\n",
      "Loading game deletion_game\n",
      "Baseline: 678\n",
      "Loading game deletion_game\n",
      "Baseline: 789\n",
      "Loading game deletion_game\n",
      "Baseline: 890\n",
      "Loading game deletion_game\n",
      "Baseline: 901\n",
      "Loading game deletion_game\n",
      "Baseline: 120\n",
      "Loading game deletion_game\n",
      "Loading deletion rate type 1px\n",
      "Loading method Gradient\n",
      "Loading game deletion_game\n",
      "Loading game preservation_game\n",
      "Loading method Shap\n",
      "Loading game deletion_game\n",
      "Loading game preservation_game\n",
      "Loading method InputTimesGradient\n",
      "Loading game deletion_game\n",
      "Loading game preservation_game\n",
      "Loading method DeepTaylor\n",
      "Loading game deletion_game\n",
      "Loading game preservation_game\n",
      "Loading method EdgeDetection\n",
      "Loading game deletion_game\n",
      "Loading game preservation_game\n",
      "Loading method random\n",
      "Baseline: 123\n",
      "Loading game deletion_game\n",
      "Baseline: 234\n",
      "Loading game deletion_game\n",
      "Baseline: 345\n",
      "Loading game deletion_game\n",
      "Baseline: 456\n",
      "Loading game deletion_game\n",
      "Baseline: 567\n",
      "Loading game deletion_game\n",
      "Baseline: 678\n",
      "Loading game deletion_game\n",
      "Baseline: 789\n",
      "Loading game deletion_game\n",
      "Baseline: 890\n",
      "Loading game deletion_game\n",
      "Baseline: 901\n",
      "Loading game deletion_game\n",
      "Baseline: 120\n",
      "Loading game deletion_game\n",
      "Baseline: 1234\n",
      "Loading game deletion_game\n",
      "Baseline: 2468\n",
      "Loading game deletion_game\n",
      "Baseline: 3702\n",
      "Loading game deletion_game\n",
      "Baseline: 4936\n",
      "Loading game deletion_game\n",
      "Baseline: 6170\n",
      "Loading game deletion_game\n",
      "Baseline: 7404\n",
      "Loading game deletion_game\n",
      "Baseline: 8638\n",
      "Loading game deletion_game\n",
      "Baseline: 9872\n",
      "Loading game deletion_game\n",
      "Baseline: 11106\n",
      "Loading game deletion_game\n",
      "Baseline: 12340\n",
      "Loading game deletion_game\n",
      "Baseline: 13574\n",
      "Loading game deletion_game\n",
      "Baseline: 14808\n",
      "Loading game deletion_game\n",
      "Baseline: 16042\n",
      "Loading game deletion_game\n",
      "Baseline: 17276\n",
      "Loading game deletion_game\n",
      "Baseline: 18510\n",
      "Loading game deletion_game\n",
      "Baseline: 19744\n",
      "Loading game deletion_game\n",
      "Baseline: 20978\n",
      "Loading game deletion_game\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Baseline: 22212\n",
      "Loading game deletion_game\n",
      "Baseline: 23446\n",
      "Loading game deletion_game\n",
      "Baseline: 24680\n",
      "Loading game deletion_game\n",
      "Baseline: 25914\n",
      "Loading game deletion_game\n",
      "Baseline: 27148\n",
      "Loading game deletion_game\n",
      "Baseline: 28382\n",
      "Loading game deletion_game\n",
      "Baseline: 29616\n",
      "Loading game deletion_game\n",
      "Baseline: 30850\n",
      "Loading game deletion_game\n",
      "Baseline: 32084\n",
      "Loading game deletion_game\n",
      "Baseline: 33318\n",
      "Loading game deletion_game\n",
      "Baseline: 34552\n",
      "Loading game deletion_game\n",
      "Baseline: 35786\n",
      "Loading game deletion_game\n",
      "Baseline: 37020\n",
      "Loading game deletion_game\n",
      "Baseline: 38254\n",
      "Loading game deletion_game\n",
      "Baseline: 39488\n",
      "Loading game deletion_game\n",
      "Baseline: 40722\n",
      "Loading game deletion_game\n",
      "Baseline: 41956\n",
      "Loading game deletion_game\n",
      "Baseline: 43190\n",
      "Loading game deletion_game\n",
      "Baseline: 44424\n",
      "Loading game deletion_game\n",
      "Baseline: 45658\n",
      "Loading game deletion_game\n",
      "Baseline: 46892\n",
      "Loading game deletion_game\n",
      "Baseline: 48126\n",
      "Loading game deletion_game\n",
      "Baseline: 49360\n",
      "Loading game deletion_game\n",
      "Baseline: 50594\n",
      "Loading game deletion_game\n",
      "Baseline: 51828\n",
      "Loading game deletion_game\n",
      "Baseline: 53062\n",
      "Loading game deletion_game\n",
      "Baseline: 54296\n",
      "Loading game deletion_game\n",
      "Baseline: 55530\n",
      "Loading game deletion_game\n",
      "Baseline: 56764\n",
      "Loading game deletion_game\n",
      "Baseline: 57998\n",
      "Loading game deletion_game\n",
      "Baseline: 59232\n",
      "Loading game deletion_game\n",
      "Baseline: 60466\n",
      "Loading game deletion_game\n",
      "Baseline: 61700\n",
      "Loading game deletion_game\n",
      "Baseline: 62934\n",
      "Loading game deletion_game\n",
      "Baseline: 64168\n",
      "Loading game deletion_game\n",
      "Baseline: 65402\n",
      "Loading game deletion_game\n",
      "Baseline: 66636\n",
      "Loading game deletion_game\n",
      "Baseline: 67870\n",
      "Loading game deletion_game\n",
      "Baseline: 69104\n",
      "Loading game deletion_game\n",
      "Baseline: 70338\n",
      "Loading game deletion_game\n",
      "Baseline: 71572\n",
      "Loading game deletion_game\n",
      "Baseline: 72806\n",
      "Loading game deletion_game\n",
      "Baseline: 74040\n",
      "Loading game deletion_game\n",
      "Baseline: 75274\n",
      "Loading game deletion_game\n",
      "Baseline: 76508\n",
      "Loading game deletion_game\n",
      "Baseline: 77742\n",
      "Loading game deletion_game\n",
      "Baseline: 78976\n",
      "Loading game deletion_game\n",
      "Baseline: 80210\n",
      "Loading game deletion_game\n",
      "Baseline: 81444\n",
      "Loading game deletion_game\n",
      "Baseline: 82678\n",
      "Loading game deletion_game\n",
      "Baseline: 83912\n",
      "Loading game deletion_game\n",
      "Baseline: 85146\n",
      "Loading game deletion_game\n",
      "Baseline: 86380\n",
      "Loading game deletion_game\n",
      "Baseline: 87614\n",
      "Loading game deletion_game\n",
      "Baseline: 88848\n",
      "Loading game deletion_game\n",
      "Baseline: 90082\n",
      "Loading game deletion_game\n",
      "Baseline: 91316\n",
      "Loading game deletion_game\n",
      "Baseline: 92550\n",
      "Loading game deletion_game\n",
      "Baseline: 93784\n",
      "Loading game deletion_game\n",
      "Baseline: 95018\n",
      "Loading game deletion_game\n",
      "Baseline: 96252\n",
      "Loading game deletion_game\n",
      "Baseline: 97486\n",
      "Loading game deletion_game\n",
      "Baseline: 98720\n",
      "Loading game deletion_game\n",
      "Baseline: 99954\n",
      "Loading game deletion_game\n",
      "Baseline: 101188\n",
      "Loading game deletion_game\n",
      "Baseline: 102422\n",
      "Loading game deletion_game\n",
      "Baseline: 103656\n",
      "Loading game deletion_game\n",
      "Baseline: 104890\n",
      "Loading game deletion_game\n",
      "Baseline: 106124\n",
      "Loading game deletion_game\n",
      "Baseline: 107358\n",
      "Loading game deletion_game\n",
      "Baseline: 108592\n",
      "Loading game deletion_game\n",
      "Baseline: 109826\n",
      "Loading game deletion_game\n",
      "Baseline: 111060\n",
      "Loading game deletion_game\n"
     ]
    }
   ],
   "source": [
    "lr = DeletionResults()\n",
    "lr.load_results(aopc=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Generating Random Pixel List for:0\n",
      "Generating Random Pixel List for:0\n",
      "Generating Random Pixel List for:0\n",
      "Generating Random Pixel List for:0\n",
      "Generating Random Pixel List for:0\n",
      "Generating Random Pixel List for:0\n",
      "Generating Random Pixel List for:0\n",
      "Generating Random Pixel List for:0\n",
      "Generating Random Pixel List for:0\n",
      "Generating Random Pixel List for:0\n",
      "Generating Random Pixel List for:0\n",
      "Generating Random Pixel List for:0\n",
      "Generating Random Pixel List for:0\n",
      "Generating Random Pixel List for:0\n",
      "Generating Random Pixel List for:0\n",
      "Generating Random Pixel List for:0\n",
      "Generating Random Pixel List for:0\n",
      "Generating Random Pixel List for:0\n",
      "Generating Random Pixel List for:0\n",
      "Generating Random Pixel List for:0\n",
      "Generating Random Pixel List for:0\n",
      "Generating Random Pixel List for:0\n",
      "Generating Random Pixel List for:0\n",
      "Generating Random Pixel List for:0\n",
      "Generating Random Pixel List for:0\n",
      "Generating Random Pixel List for:0\n",
      "Generating Random Pixel List for:0\n",
      "Generating Random Pixel List for:0\n",
      "Generating Random Pixel List for:0\n",
      "Generating Random Pixel List for:0\n",
      "Generating Random Pixel List for:0\n",
      "Generating Random Pixel List for:0\n",
      "Generating Random Pixel List for:0\n",
      "Generating Random Pixel List for:0\n",
      "Generating Random Pixel List for:0\n",
      "Generating Random Pixel List for:0\n",
      "Generating Random Pixel List for:0\n",
      "Generating Random Pixel List for:0\n",
      "Generating Random Pixel List for:0\n",
      "Generating Random Pixel List for:0\n",
      "Generating Random Pixel List for:0\n",
      "Generating Random Pixel List for:0\n",
      "Generating Random Pixel List for:0\n",
      "Generating Random Pixel List for:0\n",
      "Generating Random Pixel List for:0\n",
      "Generating Random Pixel List for:0\n",
      "Generating Random Pixel List for:0\n",
      "Generating Random Pixel List for:0\n",
      "Generating Random Pixel List for:0\n",
      "Generating Random Pixel List for:0\n",
      "Generating Random Pixel List for:0\n",
      "Generating Random Pixel List for:0\n",
      "Generating Random Pixel List for:0\n",
      "Generating Random Pixel List for:0\n",
      "Generating Random Pixel List for:0\n",
      "Generating Random Pixel List for:0\n",
      "Generating Random Pixel List for:0\n",
      "Generating Random Pixel List for:0\n",
      "Generating Random Pixel List for:0\n",
      "Generating Random Pixel List for:0\n",
      "Generating Random Pixel List for:0\n",
      "Generating Random Pixel List for:0\n",
      "Generating Random Pixel List for:0\n",
      "Generating Random Pixel List for:0\n",
      "Generating Random Pixel List for:0\n",
      "Generating Random Pixel List for:0\n",
      "Generating Random Pixel List for:0\n",
      "Generating Random Pixel List for:0\n",
      "Generating Random Pixel List for:0\n",
      "Generating Random Pixel List for:0\n",
      "Generating Random Pixel List for:0\n",
      "Generating Random Pixel List for:0\n",
      "Generating Random Pixel List for:0\n",
      "Generating Random Pixel List for:0\n",
      "Generating Random Pixel List for:0\n",
      "Generating Random Pixel List for:0\n",
      "Generating Random Pixel List for:0\n",
      "Generating Random Pixel List for:0\n",
      "Generating Random Pixel List for:0\n",
      "Generating Random Pixel List for:0\n",
      "Generating Random Pixel List for:0\n",
      "Generating Random Pixel List for:0\n",
      "Generating Random Pixel List for:0\n",
      "Generating Random Pixel List for:0\n",
      "Generating Random Pixel List for:0\n",
      "Generating Random Pixel List for:0\n",
      "Generating Random Pixel List for:0\n",
      "Generating Random Pixel List for:0\n",
      "Generating Random Pixel List for:0\n",
      "Generating Random Pixel List for:0\n",
      "Generating Random Pixel List for:0\n",
      "Generating Random Pixel List for:0\n",
      "Generating Random Pixel List for:0\n",
      "Generating Random Pixel List for:0\n",
      "Generating Random Pixel List for:0\n",
      "Generating Random Pixel List for:0\n",
      "Generating Random Pixel List for:0\n",
      "Generating Random Pixel List for:0\n",
      "Generating Random Pixel List for:0\n",
      "Generating Random Pixel List for:0\n"
     ]
    }
   ],
   "source": [
    "pl.generate_random_pixel_lists(lr.baseline_arr[\"1px\"], images_shape=[1,32,32,3])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 195,
   "metadata": {},
   "outputs": [],
   "source": [
    "results_aopc = Results(pixel_lists=pl, loaded_results=lr)\n",
    "#results_corr = Results(pixel_lists=pl)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "method_scores, method_relevance = results_aopc.get_all_scores_random_orderings()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0\n",
      "100\n",
      "200\n",
      "300\n",
      "400\n",
      "500\n",
      "600\n",
      "700\n",
      "800\n",
      "900\n",
      "1000\n",
      "1100\n",
      "1200\n",
      "1300\n",
      "1400\n",
      "1500\n",
      "1600\n",
      "1700\n",
      "1800\n",
      "1900\n",
      "2000\n",
      "2100\n",
      "2200\n",
      "2300\n",
      "2400\n",
      "2500\n",
      "2600\n",
      "2700\n",
      "2800\n",
      "2900\n",
      "3000\n",
      "3100\n",
      "3200\n",
      "3300\n",
      "3400\n",
      "3500\n",
      "3600\n",
      "3700\n",
      "3800\n",
      "3900\n",
      "4000\n",
      "4100\n",
      "4200\n",
      "4300\n",
      "4400\n",
      "4500\n",
      "4600\n",
      "4700\n",
      "4800\n",
      "4900\n",
      "5000\n",
      "5100\n",
      "5200\n",
      "5300\n",
      "5400\n",
      "5500\n",
      "5600\n",
      "5700\n",
      "5800\n",
      "5900\n",
      "6000\n",
      "6100\n",
      "6200\n",
      "6300\n",
      "6400\n",
      "6500\n",
      "6600\n",
      "6700\n",
      "6800\n",
      "6900\n",
      "7000\n",
      "7100\n",
      "7200\n",
      "7300\n",
      "7400\n",
      "7500\n",
      "7600\n",
      "7700\n",
      "7800\n",
      "7900\n",
      "8000\n",
      "8100\n",
      "8200\n",
      "8300\n",
      "8400\n",
      "8500\n",
      "8600\n",
      "8700\n",
      "8800\n",
      "8900\n",
      "9000\n",
      "9100\n",
      "9200\n",
      "9300\n",
      "9400\n",
      "9500\n",
      "9600\n",
      "9700\n",
      "9800\n",
      "9900\n",
      "Gradient 221\n",
      "0\n",
      "100\n",
      "200\n",
      "300\n",
      "400\n",
      "500\n",
      "600\n",
      "700\n",
      "800\n",
      "900\n",
      "1000\n",
      "1100\n",
      "1200\n",
      "1300\n",
      "1400\n",
      "1500\n",
      "1600\n",
      "1700\n",
      "1800\n",
      "1900\n",
      "2000\n",
      "2100\n",
      "2200\n",
      "2300\n",
      "2400\n",
      "2500\n",
      "2600\n",
      "2700\n",
      "2800\n",
      "2900\n",
      "3000\n",
      "3100\n",
      "3200\n",
      "3300\n",
      "3400\n",
      "3500\n",
      "3600\n",
      "3700\n",
      "3800\n",
      "3900\n",
      "4000\n",
      "4100\n",
      "4200\n",
      "4300\n",
      "4400\n",
      "4500\n",
      "4600\n",
      "4700\n",
      "4800\n",
      "4900\n",
      "5000\n",
      "5100\n",
      "5200\n",
      "5300\n",
      "5400\n",
      "5500\n",
      "5600\n",
      "5700\n",
      "5800\n",
      "5900\n",
      "6000\n",
      "6100\n",
      "6200\n",
      "6300\n",
      "6400\n",
      "6500\n",
      "6600\n",
      "6700\n",
      "6800\n",
      "6900\n",
      "7000\n",
      "7100\n",
      "7200\n",
      "7300\n",
      "7400\n",
      "7500\n",
      "7600\n",
      "7700\n",
      "7800\n",
      "7900\n",
      "8000\n",
      "8100\n",
      "8200\n",
      "8300\n",
      "8400\n",
      "8500\n",
      "8600\n",
      "8700\n",
      "8800\n",
      "8900\n",
      "9000\n",
      "9100\n",
      "9200\n",
      "9300\n",
      "9400\n",
      "9500\n",
      "9600\n",
      "9700\n",
      "9800\n",
      "9900\n",
      "Shap 221\n",
      "0\n",
      "100\n",
      "200\n",
      "300\n",
      "400\n",
      "500\n",
      "600\n",
      "700\n",
      "800\n",
      "900\n",
      "1000\n",
      "1100\n",
      "1200\n",
      "1300\n",
      "1400\n",
      "1500\n",
      "1600\n",
      "1700\n",
      "1800\n",
      "1900\n",
      "2000\n",
      "2100\n",
      "2200\n",
      "2300\n",
      "2400\n",
      "2500\n",
      "2600\n",
      "2700\n",
      "2800\n",
      "2900\n",
      "3000\n",
      "3100\n",
      "3200\n",
      "3300\n",
      "3400\n",
      "3500\n",
      "3600\n",
      "3700\n",
      "3800\n",
      "3900\n",
      "4000\n",
      "4100\n",
      "4200\n",
      "4300\n",
      "4400\n",
      "4500\n",
      "4600\n",
      "4700\n",
      "4800\n",
      "4900\n",
      "5000\n",
      "5100\n",
      "5200\n",
      "5300\n",
      "5400\n",
      "5500\n",
      "5600\n",
      "5700\n",
      "5800\n",
      "5900\n",
      "6000\n",
      "6100\n",
      "6200\n",
      "6300\n",
      "6400\n",
      "6500\n",
      "6600\n",
      "6700\n",
      "6800\n",
      "6900\n",
      "7000\n",
      "7100\n",
      "7200\n",
      "7300\n",
      "7400\n",
      "7500\n",
      "7600\n",
      "7700\n",
      "7800\n",
      "7900\n",
      "8000\n",
      "8100\n",
      "8200\n",
      "8300\n",
      "8400\n",
      "8500\n",
      "8600\n",
      "8700\n",
      "8800\n",
      "8900\n",
      "9000\n",
      "9100\n",
      "9200\n",
      "9300\n",
      "9400\n",
      "9500\n",
      "9600\n",
      "9700\n",
      "9800\n",
      "9900\n",
      "InputTimesGradient 221\n",
      "0\n",
      "100\n",
      "200\n",
      "300\n",
      "400\n",
      "500\n",
      "600\n",
      "700\n",
      "800\n",
      "900\n",
      "1000\n",
      "1100\n",
      "1200\n",
      "1300\n",
      "1400\n",
      "1500\n",
      "1600\n",
      "1700\n",
      "1800\n",
      "1900\n",
      "2000\n",
      "2100\n",
      "2200\n",
      "2300\n",
      "2400\n",
      "2500\n",
      "2600\n",
      "2700\n",
      "2800\n",
      "2900\n",
      "3000\n",
      "3100\n",
      "3200\n",
      "3300\n",
      "3400\n",
      "3500\n",
      "3600\n",
      "3700\n",
      "3800\n",
      "3900\n",
      "4000\n",
      "4100\n",
      "4200\n",
      "4300\n",
      "4400\n",
      "4500\n",
      "4600\n",
      "4700\n",
      "4800\n",
      "4900\n",
      "5000\n",
      "5100\n",
      "5200\n",
      "5300\n",
      "5400\n",
      "5500\n",
      "5600\n",
      "5700\n",
      "5800\n",
      "5900\n",
      "6000\n",
      "6100\n",
      "6200\n",
      "6300\n",
      "6400\n",
      "6500\n",
      "6600\n",
      "6700\n",
      "6800\n",
      "6900\n",
      "7000\n",
      "7100\n",
      "7200\n",
      "7300\n",
      "7400\n",
      "7500\n",
      "7600\n",
      "7700\n",
      "7800\n",
      "7900\n",
      "8000\n",
      "8100\n",
      "8200\n",
      "8300\n",
      "8400\n",
      "8500\n",
      "8600\n",
      "8700\n",
      "8800\n",
      "8900\n",
      "9000\n",
      "9100\n",
      "9200\n",
      "9300\n",
      "9400\n",
      "9500\n",
      "9600\n",
      "9700\n",
      "9800\n",
      "9900\n",
      "DeepTaylor 221\n",
      "0\n",
      "100\n",
      "200\n",
      "300\n",
      "400\n",
      "500\n",
      "600\n",
      "700\n",
      "800\n",
      "900\n",
      "1000\n",
      "1100\n",
      "1200\n",
      "1300\n",
      "1400\n",
      "1500\n",
      "1600\n",
      "1700\n",
      "1800\n",
      "1900\n",
      "2000\n",
      "2100\n",
      "2200\n",
      "2300\n",
      "2400\n",
      "2500\n",
      "2600\n",
      "2700\n",
      "2800\n",
      "2900\n",
      "3000\n",
      "3100\n",
      "3200\n",
      "3300\n",
      "3400\n",
      "3500\n",
      "3600\n",
      "3700\n",
      "3800\n",
      "3900\n",
      "4000\n",
      "4100\n",
      "4200\n",
      "4300\n",
      "4400\n",
      "4500\n",
      "4600\n",
      "4700\n",
      "4800\n",
      "4900\n",
      "5000\n",
      "5100\n",
      "5200\n",
      "5300\n",
      "5400\n",
      "5500\n",
      "5600\n",
      "5700\n",
      "5800\n",
      "5900\n",
      "6000\n",
      "6100\n",
      "6200\n",
      "6300\n",
      "6400\n",
      "6500\n",
      "6600\n",
      "6700\n",
      "6800\n",
      "6900\n",
      "7000\n",
      "7100\n",
      "7200\n",
      "7300\n",
      "7400\n",
      "7500\n",
      "7600\n",
      "7700\n",
      "7800\n",
      "7900\n",
      "8000\n",
      "8100\n",
      "8200\n",
      "8300\n",
      "8400\n",
      "8500\n",
      "8600\n",
      "8700\n",
      "8800\n",
      "8900\n",
      "9000\n",
      "9100\n",
      "9200\n",
      "9300\n",
      "9400\n",
      "9500\n",
      "9600\n",
      "9700\n",
      "9800\n",
      "9900\n",
      "EdgeDetection 221\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "array([<matplotlib.axes._subplots.AxesSubplot object at 0x7f9e9e2038d0>,\n",
       "       <matplotlib.axes._subplots.AxesSubplot object at 0x7f9e869bd390>,\n",
       "       <matplotlib.axes._subplots.AxesSubplot object at 0x7f9e9c740dd0>,\n",
       "       <matplotlib.axes._subplots.AxesSubplot object at 0x7f9e8c30c850>,\n",
       "       <matplotlib.axes._subplots.AxesSubplot object at 0x7f9e004322d0>],\n",
       "      dtype=object)"
      ]
     },
     "execution_count": 37,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAkkAAAI/CAYAAABj+03oAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAAgAElEQVR4nOzdd3xUVf7/8dedlskkmRTSIBBCrwpSpFmwwqqooIigrrquqFssa9ldt/y2qdtcdV3XFduqqPi1N9aGBRFFutJ7CYT03mfm/v4IICUhAWbmTjLv5+ORx8zcufd8Puc+ZPzMuXfOMUzTREREREQOZrM6AREREZFIpCJJREREpBkqkkRERESaoSJJREREpBkqkkRERESaoSJJREREpBmOUDSamppq5uTkhKJpEelgNuVvAqB3Rm+LMxGRaLV06dIi0zTTDt0ekiIpJyeHJUuWhKJpEQmSGY/MAOCFH78Q1riT7p8EwNu3v93s63Cxqv8iEnkMw9je3PaQFEkiEvlO63+a1SlYKtr7LyKtU5EkEqVuPOtGq1OwVLT3X0Rapxu3RURERJqhkSSRKDX1n1MBePnml4+7rVnvrjji+zPPH3rcMYItmP0XkY5JRZJIlJp44kRL4l48/GJL4h7Kqv6LSPuhIkkkSl03/ro279vcSFFpZR3b8supqm2gqraR6vpGPDEOOnlj6ZQQS3qSB29czHHFDaVIyUNEIpeKJBFps/zSapZtzGfF5ny25Vcc9J7TbqPRHzhoW5dO8Qzs3onu6V7GD8kmxuWgpqEGAI/LE7a8RUSOhYokkSg1+YHJALx+2+ut7ltaWccbCzeyaO1uTCAnw8vFY/swILsTiXExxMU6cTnsNPj8lFTUUVxRS25RJWu2F/Hpyh18tGw7KQlurjxrEN+U/o34WFfY50U61NH0X0Sik4okkSh18YjW7w2qrKnnLy8t4q8vLcIEzhmew5lDu5Oc4G52f5fDTmZKHJkpcQzKSWXCiB40NPpZn1vCV2t38++3l+FO34Pb5eCH989leN9MdhdX0aVTfJB717q29F9EopthmmbQGx0xYoSpGbdF2relG/Yw7Z432by7jJH9Mrl4bF9SE2OPq82q2gYemX8LZVX1lOycRqcEN4ld59Cvawpz73o3SJmLiBwdwzCWmqY54tDtmidJRA5imiYPv7GUsbfOpr7Rz2f3z+CH3xty3AUSQHysi6R4NzmZifzowpNISnBTUFbDonV5PPTaEuoafEHogYhIcKhIEolSk+6ftH/dtH0qa+q55PdvcPMjH3Hu8BxW/OdaTjuxW0jiD+mZzl2XjaJbmheP28mtj86jzzWzeHzuCnyH3AAeCs31X0TkQLonSSRKTR8zff/zWe+uoKSiln+9tYy84mouPbUfZw/rzqufrw963OHdJhz0OjbGQa8uSTxz/eX86un5zHzgff75+lIeuOkszh6WE/T4+xzYfxGR5uieJBHh7ic/45G3ltHgC3DD+UMY2D01bLEf++I2AG4Y9wCmabJ8cwGvzl9PUUUtQ3qmcelp/fj1FePClo+IRJ+W7knSSJJIlGr0NwLwzpdb+fsrX+P1xHDbJSND/kuz6vpyAOJiEg97zzAMhvXO4IScVOYt387cr7fwh+cWYrfZuGPqyTgd9qDlsa//TrszaG2KSMeiIkkkSk15cAq7iipZsfR75GQ03Ujt9Rw+Q3awzV7yO6Bp5KglToediSN7MmZgFnM+XcvdT83nxU/W8vhtExk1oEtQ8pjy4BQAy+drEpHI1WqRZBhGN+BZIAMwgVmmaT4U6sREJHRM08TtH8nqdZuYNLo35wzLweUM3ihNsCTGxXDD+UPJTI7jRw9/wJhbnuMX00bz+6tPOe5RpavGXRWkLEWko2rLSJIPuN00zWWGYSQASw3D+NA0zTUhzk1EQsDnD3DDg+/xyocxXH/eDP5987k89d43Vqd1RHtKq7lz6ihenr+O++Z8xYufrOG6751IelIcADPPH3rUbV42+rJgpykiHUyrUwCYpplnmuayvc8rgbVAVqgTE5Hga2j0c/k9b/LUe9/yy+nDeeBHp+Kwt4+ZQGJjHHz/nMHMPH8IBWU1/On5L1m4ehfH+uOTmoaa/evIiYg056g+HQ3DyAFOAhaFIhkRCZ26Bh9Tfv86r36+gQduOpNvSx/i8n9dbnVaR214n0x+c+VYuqd7eebDVTw+9xtKK+uOup1pD09j2sPTQpChiHQUbb5x2zCMeOBV4FbTNCuaeX8mMBMgOzs7aAmKyPGrrm3g4t+9xrzl23ns1gnMPH8o2YuvtSSX0TkXHncbKQmx3HbJSN5fspW3vtrEkBueYvYvJh3VxJfXnmZN/0Wk/WjTPEmGYTiBd4D3TdP8R2v7a54kkchRW9/I0BueZuPuUq459wRGB+nXYcFy4DxJx2LbnnJenr+eLXvKuHv6aH575bigThUgIh3fMa/dZhiGATwJrG1LgSQikcPnD3D5PW+xcVcpP5hwcIFU11hFXWNV2HMqqy2grLYgaO3lZCay7NGr+f7Zg/jT819y2s9eYEteWavHVdRWUFF72KC4iMh+bbncNg64CvjWMIwVe7fdbZrm3NClJSJHY9a7Kw7bZpomz364moVrdnH5GQM4uf/BI0jPfP0b4NhHcI7VS8vuC3rcBE8MT995PhNG9ODGhz5g6I1P859bJjDjzIEtHnPFv68ANE+SiLSs1SLJNM0FgBGGXEQkiF5bsIGFa3ZxwahenDHk8PsEx/WYbEFWoXX5GQMZPSCLK//8Nlfc9zbvLd7Cv35yDt64wyfJnHnGTAsyFJH2RDNui3RAn63cwQdLtzF+SDcuGN2r2X0GdzktzFmFzqEjaVecNZDkBDez563mf4u38M4fLz1spu5JwyaFM0URaYfaxwQpItJmm3aXMuezdZzQI41p4wfQdFvh4arry/evo9bR2G02Jo3uzR1TT8YMmIy7dTZ/nP0FPn9g/z7FVcUUVxVbmKWIRDoVSSIdSFlVHY+9s4JUbyw/mHACthYKJGhaQ23fOmodVe8uyfz6irFcdnp/fvvMAk65dTYbc0sAuOaxa7jmsWusTVBEIpout4l0ED5/gMfeXUl9o59bp4zA4z7y6van9poapsysjetxOxk/JJs4t5MXPl7D4Ouf4tLT+tErbSIYBrPeXXFMy5qISMenIkmkg/i/z9axJa+MmecNISs1odX9B2aODUNWkRN3ZL/O9O6SzDMfruKFj9cwrHcGV509yJJcRKR90OU2kQ5g+aZ8PvtmJ+cMy2F438w2HVNZV0JlXUmIMztcYdUOCqt2hD0uQHKCm5snD2fKKX1ZuX0Lf3zxA7a2YU4lEYlOKpJE2rldRZU899FqstO9XDyuT5uPe2HpH3lh6R9DmFnzXlv5AK+tDO/cTAeyGQYTRvSg38BPML2v8NeXv+bhN5Ye80K5ItJx6XKbSDsWCJhc/dd3afQF+OH3TsRhb/v3nvF9pocws8g3cdBV1Df4WLDMzc2PfMTqbUU8/JOztaSJiOynIkmkHbv/la+Zt3w7V509iIzkuKM6tl/6ySHKqn3Y1//BWSbb8yv485yv2LirlJd/cxEp3liLsxORSKDLbSLt1LKNe/jV0/O55NS+jBuUddTHB3sNtfZmX/9thsF9153OM3edz4LVuYy++Tm253fM+aNE5OioSBJph+oafFz1l3dIS/Qw69aJLU4YeSQvLbtv/zpq0ejA/s96dwV1DT5umTycXUWVnHTjf/nj7C+Y9e6KZtfFE5HooMttIu3Qr5+ez5rtxbx379RjvjR0Zt8rg5xVZMc9VHN59O6SzM8uGclDry/l7y9/zS2TR9A1rfXpFESkY9JIkkg7M/+bnfzj1cXcNOkkJozseczt9EkbTp+04UHMLLLjtjWPbule7ph6Mnabjftf+Zpte3TpTSRaqUgSaUcqa+q55m/v0rNzEn+7fvxxtVVcvZvi6t3BSewo7C7fxO7yTWGPe6gj9T8zJY47pp6Mx+3kwdeWsHTDnjBnJyKRQEWSSDty+2OfsC2/nGfuPJ+4WNdxtfXKir/xyoq/BSmztnt71SO8veqRsMc9VGv9T02M5fZLRuJxOznnFy+xYlN+GLMTkUige5JE2oFZ765g5ZYCHp+7knOH57B6exGrtxcdV5vn9LsmOMm1U23pf4o3lp9dMpL/vLOcs3/+Ep/8fTon9EgLfXIiEhE0kiTSDlRU1/Pch6vplpbAhWPaPqv2kfRMHULP1CFBaas9amv/UxNj+eTv03G77Jx11xzWHGdxKiLth0aSRCKcaZo8+9Fq6hp8/GDiSJyO4Hy32bd+Wlp8dlDaa2+Opv/zlm/nhvOHcv8rixlz83PcfulIMlPi978/8/yhIctTRKyjkSSRCPfYOyv4dmshU07tS5dO8a0f0EZWr6FmtaPtf0ZyHD+7ZAQm8I9Xl5BfWh265EQkImgkSSSCrd9ZzM8e+5iB3TsxfkhwR3wmDrguqO1FetxDHUsemSnx/OySkdz/ytc88OoSbp86krRETwiyE5FIoJEkkQjV6PNzxX3v4IlxcvU5g7Edw6zaR9I9ZTDdUwYHtc1IjhusPLp0iufWKSNo8Pn5xyuLKSqvDUF2IhIJVCSJRKjfP/cFSzfuYdatE0iKdwe9/T0VW9lTsTXo7bZme8kqtpesCnvcQx1P/7ulebl1ygjqGnz849XF7CioCHJ2IhIJVCSJRKAFq3K5b85XXDvhBKac2i8kMd789p+8+e0/Q9L2kby39kneW/tk2OMe6nj7n53u5ZbJI6iua+TMO19kV1FlELMTkUigIkkkwlRU13PVX94hJyORh350VsjinDfwBs4beEPI2o90weh/TmYit0weTkFZDWfc8SI7NaIk0qGoSBKJMDc/8hE7Cip47ufnk+CJCVmcbsn96ZbcP2TtR7pg9b9n5yT+d89U8kurGXvrbM2jJNKBqEgSiSDPfbiKZz5cxa+mj2HsoK4hjRUpa6hZJZj9Hze4K5/dPwOfP8Aptz3PwtW5QWlXRKylKQBEIsCsd1eQV1LFvS98RZ+sZDp3imPWuytCGnPf+mk3jIvOuZKC3f+hvTNY+OCVnPuL/+Psn7/EnLsv5MKxwZkdXUSs0WqRZBjGU8AFQIFpmtb/blekA2po9DPr3ZXEOO388HsnYreFfpB30uAfhzxGJMU9VCjy6NE5iS8evILzf/0KF//uNe79wen8fNoojCBP3yAi4dGWkaT/Av8Cng1tKiLRa86na8krruLmycND8nP/5nRJ7B2WOJES91DBzOPQUb+rzxmMAfzyyc94bcF6Pv37DDxuZ9DiiUh4tPp11TTN+UBJGHIRiUrPfPAtX6zexfdO7snA7qlhi7uzdB07S9eFLd4+GwuXsrFwadjjHiqU/Xc57Vz3vRO5eGwflqzfw2k/e4HcQv3yTaS90Y3bIhZasj6PGx58n37dUpg0OrwjLHPXPMbcNY+FNSbAxxtm8/GG2WGPe6hQ998wDL53ck9uuvAkNuwqYcSPn+XLNbtCFk9Egi9oRZJhGDMNw1hiGMaSwsLCYDUr0mEVlFYz+fevk5kSx8zzhmCzhfe+lYtOuJmLTrg5rDEjSbj6P6RnOl/98yriY52Mv+NFnn7vm5DHFJHgCNqv20zTnAXMAhgxYoQZrHZFOqJGn5+pf3yTovJavnjwCpZs2BP2HDK9PcIeM5KEs/8LVuXy4wtP4vG53/CD+//H7HmrufTUftjtTd9TZ54/NGy5iEjb6XKbiAVuf+wT5n+7kyd+NpFhfTItySFS1lCzSrj7H+d28dOLh3Hm0Gw+XrGDB19fQkVNfdjii8jRa7VIMgzjReBLoJ9hGLmGYVwX+rREOq5H3lzGw28s5bZLRnDFWYMsyyNS1lCzihX9t9tsTBs/gGsnnMDWvHLuffErtuWXhzUHEWm7Vi+3maY5PRyJiESDtxZu5OZ/f8Sk0b352/VnWJrLlCG3RVXcQ1mZx+gBXejSKZ5H317O3/7va/pmpXDNhBMsy0dEmqfLbSJhsnh9Hpff+xbD+2Ty4t2T9t+PYpW0+GzS4rOjJm6k5ZGd7uVXM8bQq0sS1/59Lj95+EMaGv2W5SMih9OyJCJhcM8LC/nLS4uIi3Fy2Wn9eP7jNVanxJailQD0TB0S1rhr9iwEYGDm2LDGPZRV/T9QfKyLWyYPZ/PuMu5/ZTErtxTw8m8uIjMl3rKcROQ7GkkSCbEdBRU8+OoSAgGTn148HG9cjNUpAfDh+v/y4fr/hj3u55tf5vPNL4c97qGs6v+h7DYbf7/hTF68exJLN+5h+I+eYdHa3VanJSJoJEkkpHYVVXLmnS9SXdfIrZeMIDMlzuqU9rt06J1Wp2CpSOr/vmVNbr90JP95ewWn3Dab6WcM5JTBXQFNESBiFY0kiYRIXnEVZ9zxIgVlNdwyeTg5GYlWp3SQTnFd6BTXxeo0LBOJ/e+W5uXuGaPpm5XCcx+t5vl5q2n0BaxOSyRqqUgSCYHcwgrOvPNF8kqqee/eqfTonGR1SoeJlDXUrBKp/W+aT2k4E0f0YP63ufz1/xaxaVep1WmJRCUVSSJBtnZ7EWNvmc2u4irm3nMpYwd1tTqlZkXKGmpWieT+22wGk0/py02TTqKovIaTbvovz89bbXVaIlFH9ySJBNHC1blc8JtXcTnsfHb/DE7qnWF1Si2aNuyXURX3UJGSx5EM7ZVO9pVjmfv1Fq788zt8sHQbD9x4JineWKtTE4kKGkkSCZK3Fm7k7J+/RKeEWBY+dGVEF0gASbHpJMWmR03cSM2jNSkJsXz8t+n89sqxPD9vNQOue4I5n6zBNLVEpkioaSRJ5DiZpsmU373Omws3kp3h5YYLhvDRsm1Wp9Wq9QVfA9Av/eSwxl256xMAhmRZO+O4Vf0/Fg67jd9ffSpTTunH9Q+8x/R73+aZD1fx0I/Opm/XFKvTE+mwNJIkchxq6hqZce/bvLFwIyP6ZnLHpSfj9UTGPEit+XTji3y68cWwx/1q21t8te2tsMc9lFX9Px5DeqXz5UNX8tCPzuLzb3MZcN0TXP3Xd9mYW2J1aiIdkkaSRI7R9vxyLvn9GyzbtIfJ4/owYUQPDMOwOq02mzH8N1anYKn22n+73cbNk0cwbfwA/vrSIh59ZznPz1vN9DMGcsMFQxk3KKtd/XcoEslUJIkcg3cXbeaqv7yD3x/grT9cwu7iKqtTOmoJ7ui+TNOe+r9vsslD9euWwu+/fwp5JVU8PvcbZs9bTd+uKfxgwglMP3Mg2eneMGcq0rHocpvIUfD5A/zyyc+44NevkJORyLJHr+GC0b2tTuuYrNmzcP86atGoo/Q/MS6Gf9x4Fnkv/Zin7ziPjGQPv3jyM7pf8Sgjf/wM977wJet2FFudpki7pJEkkTaY9e4Kiitqeer9b9m0q5RTB3dl2vj+zFu+nXnLt1ud3jHZt36a1QvNWqUj9f/AkaYrzxrEucNzWLYxn+WbCvjV0/P51dPzyUyO46TeGQztnU73dO9Bl+S07IlI81QkibTB4vV5PD9vDSYm1044gdEDIms5i2Nx5YjfRVXcQ0VKHqGQnhTHxJE9mTiyJ6WVdazYnM/yzQW8v2Qr/1u8heQENyf1Smdorwx6Z0XebPAikUJFksgRlFXVcfMjH/HcR6vp2TmJH0w8gbREj9VpBUVcjDVryVkV91CRkkeoJSe4OWNod84Y2p2q2ga+2VLI8s35zP82l49X7CAh1sXaHSVcfc5ghvXJ0E3fIgdQkSTSgrcWbuSmf37AntJqzh/Vi/NH9cRu6zi38a3aPR+AwV1OC2vcJTveA2BE9sSwxj2UVf23Unysi7GDshg7KIu6Bh+rtxexZMMeHnt3BQ+/sZRBOalcc+5grjhzEJ07xVudrojlVCSJHKKgtJqb//0RL326jhN6pPHm76ewbFO+1WkF3RdbXwfCXyQs3fk+YH2RZFX/I4Xb5WB4n0yG98mkuq6BJev38OXa3dw561PuevxTBnVPZfSALgztlc6PLxpudboillCRJLJXfYOPf725jD8+v5DaBh9/vOZU7rpsFC6nvUMWSVef/EerU7BUtPf/QHFuF6cPyeb0IdnsKanmy7W7WLQ2jyf+9w2xLgffbC3k6nMGM2ag5mCS6KIiSaKeaZq8vmADdz3xKZt3l/G9kT25/4YzGNA91erUQsrtjO7LKdHe/5ZkpsQxeVxfLhrTh/W5JXy1djez561h1rsr6ZOVzPfPGcxVZw+ie0Z03NMl0U1FkkQtnz/AK/PX8ec5i1i5pYDBOam8f99lnDuih9WphUWkrKFmlWjvf2tsNoMB2Z0YkN2J6WcM4NXPN/DMh6v4zX8/5zf//ZwzhmZz6an9uHBMb7qmadJK6ZhUJEnUKa2s4/l5q3ngtSVsySujf7cUrj5nMKMGdGZbfnmLsxt3NPvWT4vWIiHa+380EjwxXDPhBK6ZcALb9pTz3EermD1vDT9++EN+/PCHDOuTwQWjenHG0O6MHtAFt0v/a5GOwTBNM+iNjhgxwlyyZEnQ2xU5Vn5/gI9XbOep977l9S82UN/o5+T+nfnl5aO5cEwfnvjfSqtTDLsGXx0ALofb0riPfXEbADeMe8DSPOTo7SmpYsXmAvaUVrNoXR6BgEmM087YQVmcMSSb8UOyGdW/Cy6n3epURY7IMIylpmmOOHS7yn3psCqq6/lg6Vbe/mozr36+nuq6RjwxDsYMzGLcoCy6pSVQUFYTlQUSWFccREpREil5tGeZKfFMTGm6t+uy0/qzcXcp63eWsCG3hE9X7MAEnA4bpwzuyoi+mft/TderS5JuAJd2QUWSdAimabI9v4KlG/ewYFUuC1blsnxTPv6ASXKCm8E5qQzplc6JPdJwOvStFmD5zg8BOKnbOWGN++XWNwEY0+OisMY9lFX976g8bidDeqYzpGc6ANV1DWzMLWV9bgkVNQ089PpSGhr9QNN6c8P6ZDCibyZDe2XQr2sKfbKS8cbFWNkFkcOoSJJ2o6HRT25RJTsKKvb/bc8vZ/X2IlZtK6KypgGA2BgHo/p34ReXj+bc4T0YOyiLp977xuLsI8/XO+YC4S8Svtn9KWB9kWRV/6NFnNvF0N4ZDO2dATT9UGJ3cdX+f7db8sr4/Nud+Pzf3fKRkRxH367J9M1qKpp6ZyXTPd1Lt3Qv6UkejT5J2LWpSDIMYyLwEGAHnjBN888hzUqiRn2Dj8LyGgrKaigsa3osKKv5blt5DfmlNeQWVrKntIpDb6FLT/LQv1snhvfJICs1gW6pCXRL9+KwN82MvW5nMet2agX05vxwzN+sTsFS0d7/cHPYbWSne8lO93LK4K5AU+GUX1pNflkNBaXVFJTVkFdcxcrNBVTs/dKzT4zTTte0BLLTvXRL85KdnrD30bt3WwIJHo1ESXC1WiQZhmEHHgHOAXKBxYZhvGWa5ppQJyftR0Ojn7KqOkqr6iirqqe0qo7SyjrKquv3PtZRUlFHYXnNQUXRoR+E+9htBgkeFwmxTX89OycyvG8GKQmxpCS4SUlwk5zgxqVLZ8fMbovugeRo738kcNhtZKUmkJWacNh7tfWNFJbXUlJZR2nlvsc6duRXsGJzAWVVdYd9aUqKj6FbWlPBlJ7kITXRQ6o3lrSk7x47eWOJczuJdTlw7/2z2TRCJc1ry6fEycAm0zS3ABiGMQe4CFCR1E6ZponPH6C+0U9dg2/vn5+q2gYqahqoqKlveqxueqysraei+oDth7xfVl1PTV3jEWM67TY8bicJsU4SPDGkJLjpnu49qBBK8LiIj3Xh9bhwuxwaWg+xSFlDzSrR3v9IFxvjJDvdSXZ683Mw+QMByqvr9xdPJXv/Sitrmy6/1zZQVdNAoz/QaiyX006sy0FszN7CyWnHfdDrfQWVfX9h5XbZD9h+5PecDht2m4HNMLDbD3huO+T13uetvbYZBjaboc/IMGhLkZQF7DzgdS4wKjTptD9/mfMVz81bjWma+7/VmKbJvi84B23b95yD923adsBz8+Dthx1vNrVxWPvN7ntw+w2+AHUNPgKBtk/9YBgc9K1r3/M4t4tO3lg8MU5iYxx43E7iYhzExjjxxDjxuB1NjzEO3SwdgSJlDTWrRHv/2zu7zbZ3ZDn2iPvVN/qoqm1sKppqG6iqbaTR56fRF6Bh72OjP0Cjz0+DL4Bv7/NGf4Cyqnoa/bX49r5u9O3983933NF8lgabrQ0Flf2AYsqA757vra8Mw2BfqWUYxnfb+W6/A485cN8D33/hl5M4ce9N+x1Jq/MkGYZxKTDRNM0f7n19FTDKNM2fHLLfTGDm3pf9gPXBT7fNUoEiC+NHMp2blunctEznpnk6Ly3TuWmZzk3LrDo33U3TTDt0Y1tGknYB3Q543XXvtoOYpjkLmHXM6QWRYRhLmpsUSnRujkTnpmU6N83TeWmZzk3LdG5aFmnnxtaGfRYDfQzD6GEYhgu4HHgrtGmJiIiIWKvVkSTTNH2GYfwEeJ+mKQCeMk1zdcgzExEREbFQm34Da5rmXGBuiHMJpoi47BehdG5apnPTMp2b5um8tEznpmU6Ny2LqHMTkgVuRURERNq7ttyTJCIiIhJ1VCSJiIiINENFkoiIiEgzVCSJiIiINENFkoiIiEgzVCSJiIiINENFkoiIiEgzVCSJiIiINENFkoiIiEgzVCSJiIiINENFkoiIiEgzVCSJiIiINENFkoiIiEgzHKFoNDU11czJyQlF0yISZlsLtgLQI71HWOJtyt8EQO+M3gc9D4dw91VEIsPSpUuLTNNMO3R7SIqknJwclixZEoqmRSTM/jPvPwDceNaNYYk36f5JALx9+9sHPQ+HcPdVRCKDYRjbm9sekiJJRDqOaCoYoqmvItI63ZMkImFl+hvw1RYS8NVimqbV6YiItEgjSSJyRFP/ORWAl29++ZiOry9bT9X2/1GT9zn++jICDeXfvWnYMexu7O5OxGWdgbvTCZiBRgybMxipH7Xj7auIdCwqkkTkiCaeOLFN+1VseW3/84C/nvqildQVf4O/tgAMOw5PBs74rthiTsDmjMf012P66gj4a/HX5FO+/jnKCXCqvRx7TAr5i36NryYfDBsVW17D23NKqLq4X1BMniAAACAASURBVFv7KiLRQUWSiBzRdeOva/O+pmlSX7Kaml0fE2iswuHpTFy3c4lJGYjN4TnysYFGfDX5fL/zFhpK11K94z0aq3Zjc3qaiqUwOJq+ikjHpyJJRILCV5NP1Y738VXn4vB0JqHnJTjjs9p8vGFz4ozvii8mjZjUk3H5yrF/8xCBxkrK1j5JQ9l6kgb8EHenwSHshYjId1QkicgRTX5gMgCv3/Z6s++bpknlllcpW/c0ht1NfPfziOk0BMMwjineNS8/BMBLM+7CFpOEzeXF0+UU6opWsvvja0jocREpJ96M3ZV4bB06gtb6KiLRRUWSiBzRxSMuBg6+52gfM9BI1fb/UV+yCqe3Fwk9JrV6We2oGTY8nU8hfdSfKF3zBOUbnqd693w6DfkZ8dkTj7kYa86+voqIgIokEWnF1adeDRxeJPnry6nY/DL+2gI8nU8ltvMpQS1YDlW14z2c8V1J6n8NVTv+R+HXv6F8w3PEd78Am8MdlBu79/VVRAQ0T5KIHAN/XQnl658j0FCBt/c0PF1ODWmBdCCHJ4PEft/H0/UsGso2Ubb2KXzVeWGJLSLRRUWSiBzRpPsn7V8eBMBXW0DZ+ucwzUYS+16BK7FX2HMyDBuejFEk9rsKzABl65+lYvMrx93uoX0Vkeimy20ickTTx0zf/9xXnUf5xjkYNjvePlfhiE0NerxLTxjX5n2d8VkkDfwBlVvfpmjZn2ko30ynobdj2I7to+3AvoqIqEgSkSOaMXYGACWrZ1G+8UUMewyJfWdgj0kOSbypR1EkAdgcHry9L8NXlUv5hudorN5Fxuh7sTnjjzr2vr6KiIAut4lIKxr9jdRW7aZi0xwwbCEtkABKaiopqak8qmMMw6DTkFtIHX43tfmL2PXxdfhq9hx17EZ/I43+xqM+TkQ6Jo0kicgRTf7HRTSUrec/w9JJ6ntlSAskgJveeBRomifpaOz79Z2392VUbnmN3A+m4+0zff8lwbb8+m3Kg037vH3720cVW0Q6JhVJItIiM9DIhWnQ4Hbh7TkFR1xnq1Nqlcvbg8S+V1K+cQ7l65/D2+cynHFtm/n7qnFXhTg7EWlPVCSJSIuKlv+ds+MKSBv/V8xA+7kM5fBkkNT/+5RveJHyDS/g7XVJm467bPRlIc5MRNoT3ZMkIs2q2PoGlVtexdXzcuxdzrI6naNmj0kmqf/3scckU7HpZap3f9bqMTUNNdQ01IQhOxFpD1Qkichh6kpWUbTsL8RmjOKmTz5l2sPTrE7pmNic8ST2vQJHbDr5C++iOvfjI+4/7eFp7bavIhJ8rV5uMwyjG/AskAGYwCzTNB8KdWIiYg1fXTH5C+/C4U4jfdQ9XBt75MIi2K48aXxQ27M5YvH2nU71zo/I/+qXpI/6I/Hdzm1232tPuzaosUWkfWvLPUk+4HbTNJcZhpEALDUM40PTNNeEODcRCaOKLa9hmgHKN7yAv66EpP5XU73rY87uFN48Jg04Oeht2uxuOp/6MHsW3ErBV7/GDPhJ6P69w/abMvL4138TkY6j1cttpmnmmaa5bO/zSmAt0LafiohIu1Kb9wW+qh3Ed5+Iw5MBQEV9DRX14btPZ3dFCbsrSoLers0ZR+ap/8SdNozCr39L5bZ3DtunoraCitqKoMcWkfbpqO5JMgwjBzgJWBSKZETEOo2V26nJW0BMymDcnU7cv/36V//F9a/+K2x53PbOE9z2zhMhadvmiCXzlAeJzTiZwsW/p2LLGwe9f8W/r+CKf18Rktgi0v60eQoAwzDigVeBW03TPOyrlmEYM4GZANnZ2UFLUERCz19fRuXWN7HFJBOfPfGg964d3v5+2dacfZNNAni6nI6vtoiipX+itnAxsWnD8facwswzZlqYoYhEmjYVSYZhOGkqkJ43TfO15vYxTXMWMAtgxIgRZtAyFJGQMk2TwsW/I+CrJan/NAy766D3J/YbblFmoWPYHHh7XULlltep3vE+hmHH23MKk4ZNsjo1EYkgrV5uMwzDAJ4E1pqm+Y/QpyQi4VS+8QVq8hYQ1/Ws/fchHehY1lJrDwybg4Sek3F6e1K1fS6V2/9HcVUxxVXFVqcmIhGiLfckjQOuAs40DGPF3r/zQpyXiIRBfckaSr55GE+X8bjTmh8xuumNR/evp9bR7BtRciZ0p3Dx77jqnxdxzWPXWJ2WiESIVi+3maa5ADDCkIuIhFGgsYr8r36JIzaVtJG/oTp3XrP7XX9y83MKhUq44xk2J95eUynf+CKXJW/CkznwoPuX9mnLArki0rFo7TaRKHHg//hN06Ry65v4qneT2O/KFgskgLN7Dw1HepbFAzDsLrx9pnFG4Hn89d/SWH1CmxfFFZGOS8uSiESh+uKVNJSuwdPlNJzx3Y64b0FVOQVV5WHKDDYX72Fz8Z6wxdvHZnfT0OV7lPhjqNj4f/hqi8Keg4hEFhVJIlHGV1tI1Y4PcCbkEJs5ttX9f/rWY/z0rcfCkFmTu99/lrvffzZs8Q5089zZ/GZdFRg2KjbOwd+giSVFopkut4lEETPQSOXWNzDsLhJ6XEjTj1eP7KbR0fM7jX19TcxMo3z9bCo2ziGx3/exOdwWZyYiVlCRJBJFqnPn4a8txNv7cmzO+DYdM77n4BBnFTkO7GtC70uo2DiHyi2v4u19uYVZiYhVdLlNJErUl66jrnAZsRmjcCX2bPNxoVpLLRId2FdXQg7x3c+nsXI7VdvfxTQ1R65ItFGRJBIFGqvzqNo+F4enM54u44/q2FCupRZpDu2ru9MJeLqcRn3JKkrXzLIwMxGxgi63iXRwZsBHwaJfgRkgoefFGDb7UR3/07EXhCizyIjXWuzYzHH468spW/M4Tk9nEnpcaEFmImIFFUkiHVzpmlnUF39DQo+LsMckH/Xxp+QMDEFWkROvtdiGYRDffSI2h5vCpfdg96TjyRhtQXYiEm663CbSgdUWLKZs7dMk5FxITMqgY2pjR1khO8oKg5xZy1bn72B1/o6wxTtQS301DDsZY/6Cy9uD/IU/p75sowXZiUi4qUgS6aD89aUULPoNzoTudDrpzmNu5865T3Pn3KeDmNmR/WHeHP4wb07Y4h3oSH21OePJPOUhbA4Pexbcgq+2IMzZiUi46XKbSAdkmn4KFv2WQEMFmac+jM0Re8xt3XbKRUHMLLK11leHJ4PMUx9k9yfXs+fzm+k8/nHsroQwZSci4aaRJJEOqHT1LGrzv6TTSXcQk9TnuNoand2P0dn9gpRZZGtLX2OS+pEx9q80VGwj/4vbCPjqwpSdiISbRpJEOpjq3Z9RtvZJEnIuJKHH5ONub986ar06ZR53W5HuSH09cIFggIScC6jc+ga7Pr4Gb69LSOw1NSw5ikj4qEgS6SAqtryGv66EsrVP4/Bk4kruT+XW14+73X3rqL00467jbivSHU1fY1IGEvDXUb3jPaq2vYu35yUYhgbnRToSFUkiHYTpb6Bi86tg2EjoeQmGLTj/vO86bUpQ2onUeMcTOzZtGKavhprd8ylaeh+pw3+pQkmkA1GRJNIBmKZJ5fa5+OuK8Pa5HHtMYtDaHt61d9DaisR4xxs7NnMcZsC3d9TOJHX43SqURDoIFUkiHUDFxhdpKF2Dp8t4XN4eQW17feEuAPqlZQW13ZYszd0EWFMsHUtfDcPA0+V0YpIHULb2SZoKpV+pUBLpAFQkibRztYXLKP7mIVxJfYnNHBP09n/74fNA+O5J+uv818Ia70DH2lfDMEgedCNgULb2CcyAj7QRvwnaJU8RsYb+BYu0Y77aQgq+/CXOuCzicyZhGEbQY9x9RvT8aut4+moYBimDb8SwOShd/R8CDRWkj7kPm90dxAxFJJxUJIm0U2agkfwvf07AV0Pn8Y9SV7QiJHGGdA7u5btIdjx93TdFgN2dQly3CVTvfJ/cDy7H2+sybA433p7W3ZAuIsdGF81F2qniFf+gvvgb0kb+P1zeniGLY+VaauEWrL7Gpg8nocfF+Kp3U77hOfwNFUHITkTCTUWSSDtUuf1dKja/TGLfK4nvdnZIY1m5llq4BbOvMSkD8faeRqC+nPJ1/6W+ZE1Q2hWR8NHlNpF2Yt/lHF/NHsrWPYszPhtHfNfDZoIOtt+edXlI27c6Xihju7w9SOz/fSo2vczuT68n7eQ/EN/1rKDGEJHQUZEk0o4EfLVUbH4NmyOWhJ6Tw/Iz80EZ2SGPYWW8UMd2xKaT1P8aqnd9QsGXP6dx8I9J6n9NSG6yF5HgavUT1jCMpwzDKDAMY1U4EhKR5plmgMqtbxJorCCh5xRszriwxF2Zt5WVeVvDEgtgwbY1LNhmzaWpUPXV5oyj8+mPEtdtAqWrHqFw8e8x/Q1BjyMiwdWWkaT/Av8Cng1tKiJyJDV5n9NYsYW47Ik448MzsSPAvZ+8DIRv3qKHF74DwCk5A8MS70Ch7GvV9ndx713GpGr7O9QVr8Tb6xJsDg+Afv0mEoFaLZJM05xvGEZO6FMRkZZU755Pbd4XxHQ6EXfqSWGN/YdzrghrPCuFuq9Ns3Ofit2dQuW2dyhb9wzeXpfgiE0PaVwROTa6J0kkwjVW7aTw699i92QSnz0h7PeyhGs5kkgQrr7GpAzC5kqiYvMrlK17hoTu54NGkkQiTtDu+jQMY6ZhGEsMw1hSWFgYrGZFolrAV8ueL+4Aw4635xQMmzPsOSzN3bR/PbWOLpx9dcZnkTzwOhyxGVRufYPilQ9hBnxhiS0ibRO0Isk0zVmmaY4wTXNEWlpasJoViVqmaVK45E80VmwhfdQ92GOSLMnjr/Nf27+eWkcX7r7anPEk9r0Cd9pwyjc8R978H+GryQ9bfBE5Ml1uE4lQFZvmUL3zfZIH34Qnc3TI50Nqyb0Tvt+h41kd27DZic+eQEKPiyhaeh+5H84gbeT/I67LaWHPRUQO1pYpAF4EvgT6GYaRaxjGdaFPSyS61RYup3jlg3i6nEZS/2stzaVXp0x6dcrssPEiJXZC9/PIOmc2Dk8m+V/8jKLlf9c0ASIWa8uv26aHIxERaZpV299QQdnap7G5EnGnj6By6xuW5vTVjvUAjM7uF5Z4H21qWqj37N5DwxLvQOHu64H2jRQm9LiI6l2fNI0k7voEb8+Lsbs7aYoAEQto7TaRCGIGGqnY/AoEGpvm0LG7rU6JBxa8yQML3gxbvMe//oDHv/4gbPEOFO6+NsewOYjvdg4JvaYSaKigdO1T1BV/a2lOItFK9ySJRAjTNKnc9jb+mj14e03FERsZP4D423nWXu4Lp0jqa0xSHxwDr6Ny65tUbXsb099A6rCfY3d5rU5NJGqoSBKJEGVrn6ChdB2erDNxJfWxOp39spMio1gLh0jrq93lJbHvFdTuWUh17kfUFa0g/eTfEZs+0urURKKCLreJRICqnR9QuvoxYlJOIDZjlNXpHMTKtdTCLRL7ahg2PJ1PocuZT2Gzx5D32U0Ur/gHAX+91amJdHgaSRKxWM2eLylY9FvcqUOJ63ZuxK0Ob+VaauEWyX1tKFuPt880qnPnUb7xBap2fkBCj4tweDJ0U7dIiKhIErFQXdFK8hfeiSuxJxnjHqA690OrUzrMAxf8sEPHi5TYbWHYnMRnT8SV2Gfv2m9P4+lyOgk9LsIw7FanJ9Lh6HKbiEXqyzawZ8EtOGLTyTz1YeyuBKtTalYXbwpdvCkdNl6kxD4arsReJA+6HldiH2p2fcLuT66noWKr1WmJdDgqkkQsUF+6jj3zf4LhiKPz6f/G4e5kdUot+nTLKj7dsips8d5e+zVvr/06bPEOFO6+Hg+bw0NCzynE50yisXI7uR/OoHTtU1r/TSSIdLlNJMxq8heRv/BO7C4vmaf9C4fHmhme2+rRr+YCML7n4LDEm738UwAmDTg5LPEOFO6+Hi/DMHB3OoGUE35C8fK/U7rq31Tv/JBOJ91FbNpJVqcn0u6pSBIJo6od71Hw9e9weXPIPPXhiJkL6UgevvAGq1MIm/baV4e7Exlj7qN61wSKlv+NvE+vJ67buXQ68eaIL8JFIpmKJJEwME0/ZWuepHTNLBzx2cTnXEhN3udWp9Um6fGJVqcQNu29r3FZ44nNGE3Z+mcoX/csNbvnk9h3Bol9r9QklCLHQEWSSIj5avZQsOi31BUtIyZlMPHdz8OwtZ9/elaupRZu7bWv+9Z928cRm0bSwOuo3vUJZWufomLT/5HY9woS+0zH5oy3KEuR9qf9fFKLtDOmaVK9ax5FS+7FNH2knfx7zHY4AeC+ddTaW+FwLDpSX+0xSXh7TsZXM46G8k2Urn6M8o1zSOp3Fd7el2FzeKxOUSTiqUgSCYGGim0Ur7if2vwviUkeSProe3DGdzvsG3978OjFN3XoeJESO1QcnnQcnnRc3h5U531Oybf/onTtU8RmjiE2bRiGzanJKEVaoCJJJIjKNjxPTd4X1BUswbA7iet6Nu704dQWLKa2YLHV6R2TFE94528Kd7xIiR1qjrjOJPa+jMaqXdTsnk9N7jxq93xJbMZo4rMnamRJpBkqkkSCwF9XQtmG56nY+CJmoIGY1KHEdTkdmzPO6tSO23vrlwIwsd/wsMR7+dsvAJh6wriwxDtQuPtqBWd8Fol9p9NYtZOavAXU7PqYHe8uJbHvlST2nqp7lkQOoCJJ5Dg0Vu2kfOMcKre+gelvJCa5P7Gdx+KITbc6taB5euk8IHyFwysWFknh7quVnPHdSOwzncaqXU33LK16hPL1z5HYdzre3pdH7AzwIuGkIknkKJlmgNr8RVRseomavC/AsBOfPZGkAddSV7jU6vSC7vFLfmJ1CmETTX3dxxmfRacTf0p96VpK1zxB6erHKFs/m8TeU/H2uRyHO9XqFEUsoyJJpI0CjdUULr2HuoIl+OtLMBxxxHYeR2zaMGzO+A5ZIAF4Y6LnXpVo6uuhYpIHkDnufurLNlC29inK1j1L2YbnSeh+AYn9rsCVkGN1iiJhpyJJpBUNFVup2PIalVvfwvRV4/B0Jj7nQmKS+7er+Y6O1b511KxYJiTcoqmvBzr0V5exGSfjSuxNbf4iKre9ReXW14nNGI2391Q8nU/BMOwWZSoSXh3/E17kGAT89VTnfkzllteoK1oOhoO4rmfhiMvEGZdldXphZeVaauEWTX1tjd2dQnz37+Hpchqmv56KLa+R/8Xte78kXEBC9/Nxxne1Ok2RkDJM0wx6oyNGjDCXLFkS9HZFQq2hchuVW16ncts7BBrKccR1xdtzMgk5k7C7U9rlPEfHq7axaQLMWGdM2ONNe+GvALw0466wx5aDmaafhrKN1BUuo7FyGwCO+G64UwaTMuRWHO5O1iYochwMw1hqmuaIQ7drJEminr+uhKrceVTv/KBp1AgbrqQ+uLufhzMhB8MwqN79qcVZWifcBYOVBYqKo5YZhp2Y5P7EJPfH31BOffEq6ou/pWrH/6ja8R7u1CF4ssbjyRiD09sTwzCsTlnkuKlIkqhjmiaNFZupyV9Ebd4X1BYsAQI4E3rg6XI67tQhmivmAK+v/hKAyYPGhCXec8s+AeCqYWeEJd6Bwt3X9sruSsTTeRyxmWPx1xZiGDaqd31KycoHKeFB7DEpuNNHEJs2jJjkgbiS+mDYnFanLXLUVCRJhxdorKK+bD31pWupL1lLXeFS/HVFADgTupM04Briu52L09uLyq2vW5xt5Jmz8nMgfIXDO+uaZia3okgKd1/bO8MwcHjS8facQvKgmTRW51G3d3b52oLFVO9sWgsPm5OYxD44vT1xeXNwenvgjOuKw5OB4YjTqJNErDYVSYZhTAQeAuzAE6Zp/jmkWYkcBdMM4K8rxle9i8aqXBqrd+193vS4ryACsDm9OOKziM04Gae3B3ZXIgB1xd9QV/yNVV2IaLOn/czqFMImmvoaTIfeq+dOO4mY1KEEGsrx1eRhcybQULqO2vyvqNr+zkH7GvZYHLFp2GOb1phzxKZji0nG7krE5krA5vRidyVgc3mxOb0Y9hgVVRI2rRZJRtNvPR8BzgFygcWGYbxlmuaaUCcnsk/AV4uvehflm18hUF+Gv74Mf0MZgfpS/PXlYPoO2NvA4cnAEZeFJ3MsjviuxCT1IyZ5QFTfW3SsnPboGXCOpr6GmmEY2GOSsMckAeCM70ocEPDV4a8rJtBQhr+hkkBjJXZXIr7aAuoKl+GrLQTT33K7Ntf+4snmSsDu8mJzJmDbW1R999rb9HxvcWVzxWPYVGDJ0WnLJ8LJwCbTNLcAGIYxB7gIUJEkQWGaJqavBl9dEf66InzVu/ePAjVW78JXtQt/ffFBxxg2V9O3TXcqrsTe2GKSsLuSscckYXN5D5u/yFebj682P5zd6jCsXEst3KKpr1axOdzY4rOA5qfSME0T01+H6a8j4Gt6NH21BPx1mHtfB/ZtayjHV5O/d586zED9kYMbDmzO+MP/HLFNI1T2GAy7C8Puxrbvtc21d7t7//u2Q14b9hhsdjeGzQU2pwqxDqQtRVIWsPOA17nAqNCkIwA1+YuaLhEdND2Duf+1ibn39QHvHfD+vjfMg16bR9FeAMwApunf+xho+mZ33NuanptmoOkDrrGagK+GQEM5pr/ukLNg+240qMspOOKycMZlUV++EXtMEoY9Vh9EYWLlWmrhFk19jVSGYWA4YsERi/0of2xomoH9BdN3RdW+Aqse01+/twCrJ+Cvx1ezp2lboBFMH2bAt/d5yyNZbejBAQVXzN4Cal/Bte/RDoYdDBuG4Wh63LvNMGx7Hw94bfvuddMxzb/XdKwDjvOz0eA4P1v3xzda2N70XkzKwIifyb3VeZIMw7gUmGia5g/3vr4KGGWa5k8O2W8mMHPvy37A+uCn22apQFGre0UnnZuW6dy0TOemeTovLdO5aZnOTcusOjfdTdNMO3RjW0aSdgHdDnjdde+2g5imOQuYdczpBZFhGEuamxRKdG6OROemZTo3zdN5aZnOTct0bloWaefG1oZ9FgN9DMPoYRiGC7gceCu0aYmIiIhYq9WRJNM0fYZh/AR4n6YpAJ4yTXN1yDMTERERsVCbfu9qmuZcYG6IcwmmiLjsF6F0blqmc9MynZvm6by0TOemZTo3LYuocxOSBW5FRERE2ru23JMkIiIiEnVUJImIiIg0Q0WSiIiISDNUJImIiIg0Q0WSiIiISDNUJImIiIg0Q0WSiIiISDNUJImIiIg0Q0WSiIiISDNUJImIiIg0Q0WSiIiISDNUJImIiIg0Q0WSiIiISDMcoWg0NTXVzMnJCUXTItLBbMrfBEDvjN4WZyIi0Wrp0qVFpmmmHbo9JEVSTk4OS5YsCUXTIhJkMx6ZAcALP37Bkvg5t+YAsORB6z4zrD4HImItwzC2N7c9JEWSiLQfp/U/zeoULKdzICLNUZEkEuVuPOtGq1OwnM6BiDRHN26LiIiINEMjSSJRbuo/pwLw8s0vW5xJy3I/mt/qPl3PPvZLZu3hHIhI+KlIEolyE0+cGPIYvppaqnPzqC0ooq6opOmvuJTGyiri6gwImHx85U8w7HbsLhf2GBeOOA/e3jkk9e9NY2UVjvg4DMMISX7hOAci0v6oSBKJcteNv+6Yj21uhCfg91NXUET1rnzqCoupLy3DV1Vz0D62GBfOOA/2WDeptgQMh42k7N6YgQABnx/T58NXW0fepwvZ9WFTDFdiAilDBpHYtweGLbh3ChzPORCRjktFkogcN399PRWbt1O1LZeavAJMvx8Mg5jkRDyZ6cQkJ+JKTsKVmIAz3oPN6dx/rOOTjwBIHz3ssHbNQID60nJq8woo37CFPfO/onj5t3QaOripWLLbw9ZHEYk+KpJEotzkByYD8Pptrx/VcWYgQNXO3ZSv30LV9p2Y/gCuJC9J/XvjycrA0zkDe4yr1XY2ljc7PQkAhs2Gu1My7k7JJA3qS/XO3RQt/ZY9ny+i5Nu1dDnrFNydko8q7+Yc6zkQkY5NRZJIlLt4xMVHtb8ZCFDw9XK2vPwO1Tt3Y4txkdS/D4n9ehLTKTlk9w0ZhkF8dhZx3bpQvWMXez7/mu1vvEfaqGEkD+p7XG0f7TkQkeigIkkkyl196tVt2s80TQoXr2TLy29TtT0XT1Ymnc8cS0KPbGxhvOxlGAbx3buSk55K3mdfUrBwCTW5eWSMGo7LG39Mbbb1HIhIdNE8SSLSqrqiElbc+0+++fujBBoaGfTTHzDm/v9HYu8eYS2QDuSIddN1wnjSxw6nOjePxXffR/WuPZbkIiIdk0aSRKLcpPsnAfD27W8f9p4ZCLDro8/ZOPtVMKHvtdPoeu7plhVGhzIMg5TB/YlNTyXvk4Us/vVfOPGOG0kZ1O+o2jnSORCR6NVqkWQYRjfgWSADMIFZpmk+FOrERCQ8po+Z3uz2bW+9z+55X1CTl48nK5PM00ZhczrY/ckXQY2f6k467jZi01MZec8vWHHfv1j+p4cYcMNVdBk/ps3Ht3QORCS6tWUkyQfcbprmMsMwEoClhmF8aJrmmhDnJiJhMGPsjMO2Ve3YxbY33sNfW0/maaNI7NcrZDdkp7qP/9dp0FQojfjTXXxz/39Y8+//UltQSM+pk9qUd3PnQESk1SLJNM08IG/v80rDMNYCWYCKJJF2bN9EkI0BHwBOW9PHQdWOXeyetwCb00n3C8/BndYppHnsix8MzjgPJ919M2tnzWbrK+9Su6eQgTd9/6B5mZrNwd/YdLz9yPuJSHQ5qnuSDMPIAU4CFoUiGREJvxvm/z8Anhp/D6WrN5C/cAkxKUl0nTAeZ7wn5PE3V+wMans2h4OBN12NJzOdzXPepK64hBPvuAlXQsu/fJvy4BRA9ySJyMHaXCQZhhEPvArcappmRTPvzwRmAmRnZwctQREJrSk9zgGgbO0m8r9YTHx2Fl3OGtfq6EskMwyDHlPOIzY9ldX/foYlv/4LQ395M57MtGb3v2rcVWHOUETaA8M0MjEwUwAAIABJREFUzdZ3Mgwn8A7wvmma/2ht/xEjRphLliwJQnoiEioHrrtWuXUHuz5aQFzXznSdcHrQ10Y7krFvNN0PtPDiF0LSfs2eAnLf/wzDMBj229tI7N0jJHFEpP0yDGOpaZojDt3e6ieh0XTX45PA2rYUSCLSvhTv3MHWeZ8Rm96JrLNPDWuBFA6ezHS6XzQBm9PJ0t/dT+GSlYftU9NQQ01DTTNHi0g0a8un4TjgKuBMwzBW7P07L8R5iUgY1BWV8OMv/sR9iV/QdcJ4bM6OOXVaTJKX7hedS1zXzqz826Pkfjj/oPenPTyNaQ9Psyg7EYlUbfl12wIgNL/9FRHL+OrqyH3/Uyba+pM2dAh2d4wleaS7U8ISx+GJpfPpowk0+lj3+PMULl5B6sghGIbBRSlNcyrlfjSfrmefFpZ8RCTydcyvjSJyRKZpsufTr/DV1nP5xdfgTg1PodKcFHdi2GLZnE66TjidPQsWU7xiNY3VNXQ+bRQTu50SthxEpP1QkSQShXb+7xOqduwifcxwGhNjaGysJsEZZ0kuDXvnKAoXw2Yj89STccbHUbRkJb6aWrzjh2N3OS07ByISmVQkiUSZiq072Dj7VeKys0ge3I/rPvs10DRPkhW2VOaGPaZhGKQOG4wz3kPeZ1/xi7m/JCYliafPui/suYhI5FKRJBJFfHV1rHrwcVwJ8XQePxrDMJjR+wKr07JMYt+eODyxTJy3C7vpprFav3ATke90rN/6isgRrX9yDjV7Chl08w9wuN3A/2fvvuOjKrMGjv/uTDLpvRBCGqEESIAAoQtSpSgqoCugoC4uqAhrede1s6tuQ10X+7KiLLigNAVUUEAQkJZAAiIk9JoE0nsySea+f4REygQSmJk7k5zv55PNzJ07zzmcNcmZW54HhoX1ZVhYwxeDbWo8wlpyz20P0rMkmNNr1lOenat1SkIIOyFNkhDNRMbWXWT8uIPW40bjHxtTtz2vopC8iqsm0W9WKvxd8Rrei+ryCpJmv0lpZpbWKQkh7IA0SUI0A6WZF0j9z//w7dCW1vfcftlrz+z4B8/s+IdGmdmHZ3b8gxePfkzEHcOoLitnz5/foiwrR+u0hBAakyZJiCbOVFXFz//6D4peT9ysqej0+sten9L+Lqa0v0uj7CDELZAQt0DN4sOvNXAN9Kf7K09RXV7B3lffpiKvQNO8hBDakgu3hWjiji7+kqLjp+nyf4+ZnQ9pUGgvDbL6la+Ll6bx4fIaFBw9QathAzj9zUZ2P/cXIsYMv2yiTZlsUojmQ5okIZqg2sVri0+f4+y6zfh2ao+xsPCyRW1rZZfnARDo6mfTHGuVV1VoEvdSV9bArUUgYSNu5ey6TZxZu4nw24eiNzhrmaIQQgNyuk2IJqqypJSMzTtw8fcluE/3evd7duebPLvzTRtmdrmTxemcLE7XLD6Yr4FHqxBChw2gPDuXs99txlRVpVF2QgityJEkIZog1WQiY9N2TFVVhA69BZ2Tvt59f9thvA0zs0/11cArMozQwf1I/+Enzq3fSthtcqpNiOZEmiQhmqCcfQcpTT9PyMA+uPhde220W0LqP8rUXFyrBt5tozBVVZG5ZRfpP/xEq2EDr7r4XQjRNMnpNiGamPy0Y2Qn7cerTSQ+MdHX3T+zNIvM0uY9L9D1auDboS3BfbpTdOIMh/69CNVksmF2QgityJEkIZqQyuISDsz9GGdPD0IG9EJRlOu+54Xd/wK0W7vNHjSkBv5dOmKqrCRj8w70ri7EPDyhQfUVQjguaZKEaCJUk4kD78ynIq+AiDHD0BsMDXrftI6/sXJm1xbqHqRpfGh4DQK6d8a9ZQinv16P3sWFtpPGSqMkRBMmTZIQTcSxL1aTk/ILHabd36j39WnR1UoZNYy3wVPT+NDwGiiKQrvJ4zEZjZxa9R16Fxeir5jBXAjRdMg1SUI0Aed37uXkl2tpNfSWRk92eLY4k7PFmVbK7PpKq8oorSrTLD40rgaKohDz2wm0HNiH40tXc+rr9VbOTgihFTmSJISDKz6TzsH3F+DTPpqY305o9PtfSXoX0O6apNMaNmi1GlsDRaej42NTqDZWcmThcvQGA2G33WrNFIUQGpAmSQgHZiwqZt8bH6J3c6HL09PROTd+VujHYydaITPHciM10On1xM36LfuNRlLnL0Hv6kLLgX2skJ0QQivSJAnhoKqNlex/40MqcnLp/srTuPj73tA4CUFxFs7M8dxoDXROTnR+ejr7/vEev7y/AJ3BmRZ9elg4OyGEVuSaJCEckGoycfD9T8lPPUrsE7/FN6bNDY91ougcJ4rOWTA7x3MzNdAbnOn6h8fxaR/Ngbkfk733ZwtnJ4TQihxJEsIBHV38Jed37KHd5Hto0ffmjly8tucDoHnPk9SYGphbJBgguG93KnLy2PfGB3R9dgaB3eQInRCOTpokIRzMz3P/w/mfkvDt1B6dq6HeP9oNNStusoUyuzFhHi00jQ+WqYHeYCB89BBOf7ORfXM+oPNT0wjuFW+B7IQQWpHTbUI4kMztiZz/KQnPiFa06NfDIhMZxgd2ID6wgwWyuzGezu54OrtrFh8sVwO9qwsRdwzDKzqCn//5bzK3J1ogOyGEVqRJEsJBZCcf4Jd3P8EtJIjQYbeg6Czz43uk4BRHCk5ZZKwbUVxZSnFlqWbxwbI10LsY6P7Sk/jEtOHA3Pmkb95hkXGFELYnTZIQDiDv0BH2v/URnhFhhI0chM7JcmfK/5Y8j78lz7PYeI11tuQ8Z0vOaxYfLF8DJzdXur0wC/+4Dhz8YMFNnxIVQmhDrkkSws4VnjhNyt/fwzXQn24vzuLC7mSLjv90l4csOp4jsnQNapuiwJ5dqSgoIHXe/8g9kIp/3K+n9Bo7M7oQwvakSRLCjpWkZ5L8l3dw9nCn+0tPYvD2sniMOP92Fh/T0VirBjonPWHDB3Ju409c2L4HtaqagPhYq8QSQljedU+3KYryiaIoFxRFOWCLhIQQNcqzc9n72r9QFOj20pO4BvpbJU5q/nFS849bZWxHYc0aKHo9rYbdgnebSLJ2p5CVtB9VVa0SSwhhWQ05krQAeA9YaN1UhBC1p2mqyso5vfp7qsrKibhjGHkH08g7mGaVmHNS5gPNe54ka9dA0eloObgfipMTOXt/xlRhJGzoAItdfC+EsI7rNkmqqm5RFCXK+qkIIQCqjUbOfPsDlcWlhN8+xGpHkGo9Gz/VquNfT4RniKbxwTY1UHQ6Qgb2RmdwJu/nVH5571M6Pf4QOie91WMLIW6MXJMkhB2pNlZydu0mKvIKCBtxK+4hwVaP2cE32uoxrsXdyU3T+GC7GiiKQnCf7ji5upC5bTdVpWV0fmoaeheDTeILIRrHYsd6FUWZpihKkqIoSVlZWZYaVohmo6qsnLNrN1F2IYfQIf3xDA+1SdwDuUc4kHvEJrHMKTQWU2gs1iw+2LYGiqIQ0C2ODr+7n+zkA+x9/V8Yi7T99wshzLNYk6Sq6jxVVRNUVU0ICgqy1LBCNAtV5eWk/P1dyi5kEzq0P97RETaL/c/9C/jn/gU2i3el9NIs0ku1/WClRQ3Chg+k81O/o+j4KZJenkPZhWybxhdCXJ+cbhNCY9XlFaT8/X3yU48ROqQf3tGRNo3/fLdpNo1nj7SqQYs+PTD4eLNvzgckvvgP4p9/wub//wsh6teQKQCWADuAGEVRziqKou1VnkI0IVWlZST/9R3yDx0hbuZv8W4TZfMc2vlE0s6nef9h1rIGfh3b0fO1Z9EZnNkz+y2y9/6sSR5CiKtdt0lSVXWiqqotVVV1VlU1TFXV+bZITIimzlhYzJ4//5OCI8fp/PtHCLmllyZ5pGSnkpKdqklse6F1DTzCWtLz9T/iHhpMyj/e59TX62UuJSHsgJxuE8JGLl2/q7KklDPfbKSyqIRWwwdSWVqq2fpe7xxYBDTveZK0qIG5/79bDuqHumk7RxYu58KuZHrMftqi6/QJIRpHfvqEsDFjQRFnvv2B6vJywkcNxj20hab5vNzjcU3jR3na5i6+a9G6BrV0zk60Gj6A7D37ydl7gL2vvU2Xp6dj8PHWOjUhmiWZ7lUIGyrNzOLUV99hMlYSfvswzRskgNZerWjt1Uqz+K5OLrg6uWgWH7SvwaUURSEooSuhQ/pTePQUu577KwVHTmidlhDNkjRJQthI4fHTnPlmAzoXA5F3j8AtOEDrlABIyjpAUpZ2SzPmVxSRX1GkWXzQvgbmeLeNIuH1Z1F0OpJmv8nZ9VvkOiUhbEyaJCGsTFVVTq3+nvQNW3EN9Cfy7tsw+HhpnVadD35Zwge/LNEsfmZZNpll2s4RpHUN6uPdOoLe/3gR/7gYUv/zPw5++F+qjUat0xKi2ZBrkoSwompjJan/+R8ZP+7AKzqCloP62d1aXa8mzNQ6Bc3Zaw1qL+4OTOgCOh0Zm3eQ+3MqrYYPxODtCUDYsIFapihEkyZNkhBWUp6Tx/43P6Lw2Ela33MHBj9vFEXROq2rhNnBArNas/caKDodQQldcAvyJ33TDk5+udamS9cI0VzJ6TYhrCDv0BF2P/cXSs5l0OUPj9HmN2PsskEC2Hl+HzvP79M6DU05Sg08I8OIGjcSZw93zq7dRPben1FNJq3TEqLJkiNJQliQajJxas16ji35CrcWgXSf/TSeYfb9aX/eoaUA9GnRVeNMtONINTB4exF59wgyt+wiO2k/yX97l7iZv8XgbT/XuQnRVCjWuFsiISFBTUpKsvi4Qtizk19/T8amHZScScerdTghA/ugdzFondZ1ZV5cXDbEXZuFqR/Y+CwAnw2do0l80L4GN0JVVQpSj3Jh516cPT2J+/1U/Dq11zotIRySoih7VFVNuHK7nG4TwgLyDh7m5IpvKT2XSYv+PQkdNsAhGiSoaQy0bA4MemcMemfN4oP2NbgRiqLg27EdPf/yHHpXA3v+/E9OrPxWTr8JYUFyuk2Im1BdYeTokq84s/YHnL08ibx7EK6B/lqn1SjbMvcCcEtId03i55YXaBL3UlrX4GZ4RYXT6+8vcmjeZxz7fBU5+34hdsbDuAUHap2aEA5PmiQhblB+6lF++eC/lGVeIOy2W3Fv1QKds7ZHRG7EJ6krAO0ahAvluZrEvZTWNbgZtdME+HZqh87ZifM/JbLjqdm06N8T73atCR9+q8YZCuG4pEkSopGMBYUcX/Y1Z9dvwS0ogO6vPI1/XIxmC9TerDl9/k/rFDTXFGqgKAo+7aNxaxlMxqbtZGzeQdHJswR174JrgJ/W6QnhkKRJEqKBqo1GTn+zkZNfrcNUYSR85GDaTLwLJ1dXrVO7KYGu8ge0KdXA4OVJxB3DyN1/iOw9P7PjqdlE/2YM4aOGoNPb10SmQtg7aZKEuI7q8grSf9zBya/WUZGTR1BCV9o+MA6PUPuegLChNqfvBmBQaC+NM9FOU6uBotMREB+LV3QkBWnHOLJwORk/7qT9g7/BPy5G6/SEcBjSJAlRj/LsXM6s28S5jduoKinFu11r4mb+tsndZr3w8Cqg6TQIN6Kp1sDg7Un8c0+QtTuZtAVL2fvqP/GLjSH6N2Pw69hO6/SEsHvSJAlxiWpjJdlJ+8jYspOclF9QVZXg3t1wDQ7ErUUgJemZlKRnap2mRb3V94+axm/jHa5pfNC+BtakKArBvbsT0K0z5zZs5eRXa9kz+038O3ckcsxw/Lt0RNHJbDBCmCNNkmj2zqz/kfKsHPJTj1F0/BQmYyVOHm74de6Ab6d2GLw8tU7RqvxcvDWN76zT/teQ1jWwBb3BmYjRQ2g19BbOrv+RU6u+J/mv7+DWIoiw4QNpObhfk/9vXYjG0v63kxAaMRYWk7l1JydXf48xrwDFSY9X6wh82rXGPbRFs/l0veHsDgCGhfXVJH52eZ4mcS+ldQ2sydxdl3pXF6LuGU3xiTPkHTzMkc9WcPTzVQR2iyOkf08Ce3RxmMlQhbAmaZJEs6KaTOQeSCV94zYuJO5DrarCNSiAkAG98GoThd7gePMc3azFR78GtGyS8jWJeymta6AFnV6Pd9sovNtGUZ6bR0HqMXIPpJKVmILO2QnPyDA8o8LwCAtFb3AmbNhArVMWwuakSRJNWu2n6MriEgoOH6cg7RiVRSXoXAz4dmiDT0ybZj+HzNz+L2idguaaew1c/f1w7ZdAcJ/ulGZeoPDoKYpPnKHw6EnQ6fAIbYFabSKoRxeHm1FeiJshTZJoskxVVRSdOE1+6jFKzmaAquLeKoSgnvF4RoWjc5I5YwC8nD20TkFzUoMaik6HR2gIHqEhqLf0pOxCNsUnz1J06ixp85eQNn8JXq3DCezRlaCELnhFhTeb09KieZImSTQ5JecySf9hG+k/7qSysAgnDzcCusXi074NBm+5MPVK685sA2Bk+C0aZ6IdqcHVFJ0O95Bg3EOCCe7THb9O7clK2kd20n5OrPiGE8u/xuDjhX/XWALjY/Hv2kku/BZNjjRJokmoyC8kKzGFzK27yE89iqLXEdijKy7+PniEtZRPu9ew9NhaoHk3CFKD68s7eBgndzdCBvYmsGdXSs5mUHImnaxdyWRu2QmKgk/bKAK6xREQH4d3dIT83AmHp6iqavFBExIS1KSkJIuPK0QtVVUpzThPzr6DXNiVTP6hI3Wn00IH96PlwL64+Ho77HpqtlRWVQGAm5OLJvEf3PQ8AP8d/DdN4oP2NXBkqslEeXYuxafTqSwsovDYKVBVnDzc8e3YDr9O7fGLbY9XZJg0TcJuKYqyR1XVhCu3y5Ek4RCqSssoPn2OopNnyD90hLxDRzDmFwLgER5K6/G306JPdzzCQ1EUReNsHYvWjYFe0f4Pp9Y1cGSKTodbcCBuwYEABPftQcm5DErTL1Bw+DjZSfsA0LkYCOjcEb+4GPw6tccjPFTWkhN2T5okYTeqSssoz86l7EI2ZeezKD2fRdn5bIpPn6Mi59e5dFwC/PDv3BG/ju2oKCjA4FMzEWD+4WPkHz6mVfoO6+tTmwG4I3KQJvEvlOVqEvdSWtegKXFyc8WnbWt82rYGoLK4lNKM85Smn6f4zDmyLmmavKMj8GnbGu+2rfFp1xqXAD/5kCPsijRJwupUVaWqtAxjXgEV+QVU5OZTnp1LeXYe5Tm5Fx/nUl1Wftn7dM5OOHt74eLng3ebSFz8fXHx98XJwx1FUVBR6xokceNWnlgPaNcg5FYUaBL3UlrXoClz9nTHp11NEwQ103GUZl6g/EIOZRdyKDi8EbXaBIDezRW/2Bi8W4fjEd4Kz4hQ3FoEyREnoZkGXZOkKMpIYC6gBz5WVfXv19pfrklq+lRVpbqsnMriEiqLSzDmF9Y0QHkFF5uhQoy1z/MLMVVWXjWG3tUFJ08PnD3ccfb0wMnTveaxtyfO3p7oXVzkU6UNVJqqAO2WB+n31SQAtt+9WJP4oH0NmjO1upry3HzKL2RTdiGHqpJSSjMuwMW/TTpnJ9xDQ/AMD8UzohUeYS1xDwnGNShAZgUXFnPD1yQpiqIH3geGA2eBREVRVquqetDyaQpbUlUVU2UV1WVlVF3S8FQVl9Y9rvkqpcrMc9VkMjuuzsWAk7sbTu5uGHy8cG8ZXPdc7+6Gs7sbTp7u6JzkD5I9kMZAaqAlRa/HLSgAt6AA/GJrtpmqqmo+bOXVHHmuyCsgO/kAmdt2X/Zeg48XrkEBGHy8MXh74ezt+et3H6+ax54e6AzO6A0GdAaDzI8mGqUhvxl6AUdVVT0OoCjK58BdgMM3Saqq1n1aUU0qqKaabSa17jXVZLrk8cXvqunyfWqPxqkX/0dV+fUAnVqzXVVRuWS/S47gXZ6HCbWqGtVkwlRVjWqqRq02oVZXX/wyYaq+Ytul+158r1pVTbXRSFVpOdXl5VSVlVNVWkZ1eUXN97KabWp19TVrpHN2Ru9qQOfigt7FgN7VBYOPN3oXAzrXi9tcDDi5ueHk7ore3U0OjTuYVSc3AnBX1FCNM9GO1MC+6JyccA0KwDUo4LLt1UYjxrwCjEXFVBaVUHnxe0VeQc3vtPIKqOfDWy1Fp0PnYkBvcL7YNDmhOOnROelR9Pqa53p9zbbLvjtdfP2SbbX7X/JenbNTzTbnXx/XbXd2Rql77ISi09Xc8acodY8VnQJ1j698XZE7BG2sIU1SK+DMJc/PAr2tk07D7P3LXEpOp9c0HZc0K3UNi5nvdU0Otd+5rFFpimp/GGt+GTijc675Mvh6owvyr9umd655Xe9ysRlyNaC/2BTJD2TTt+rkD0DzbhCkBo5BbzDg1iIItxZBZl9XVRWTsfLiB8MKqsvLqa4wolZV13y4rKqqeVxVjVpddfG7qeaDZrUJKquoMpXVfNA0mWr+VlRXo5rUS7bVfHHpPjb+W1LbONU8Ufj1qgQFah8rCkrtEwUu2anmMoYrn9fuRyPGveTlS/V/7y9N5kzBda9JUhTlHmCkqqqPXHw+GeitquoTV+w3DZh28WkMkGb5dBssEMjWML49k9rUT2pTP6mNeVKX+klt6ie1qZ9WtYlUVfWq7rshrd45IPyS52EXt11GVdV5wLwbTs+CFEVJMncBlpDaXIvUpn5SG/OkLvWT2tRPalM/e6tNQ86lJALtFEVprSiKAZgArLZuWkIIIYQQ2rrukSRVVasURXkC+I6aKQA+UVX1F6tnJoQQQgihoQZdWaWq6rfAt1bOxZLs4rSfnZLa1E9qUz+pjXlSl/pJbeontamfXdXGKgvcCiGEEEI4Orm/WwghhBDCDGmShBBCCCHMkCZJCCGEEMIMaZKEEEIIIcyQJkkIIYQQwgxpkoQQQgghzJAmSQghhBDCDGmShBBCCCHMkCZJCCGEEMIMaZKEEEIIIcyQJkkIIYQQwgxpkoQQQgghzJAmSQghhBDCDCdrDBoYGKhGRUVZY2ghhBWcuHACgNbBrTWJf/T8UQDatmjboO22oHVNhBC2s2fPnmxVVYOu3G6VJikqKoqkpCRrDC2EsIKPNn4EwKNDH9Uk/pi3xgCw5pk1DdpuC1rXRAhhO4qinDK33SpNkhDCsUgjcDWpiRBCrkkSQgghhDBDjiQJIbj3nXsBWDZr2U2NM2/evOvuM23atJuKYSuWqokQwnFJkySEYGSXkZrGv7vH3ZrGN0frmgghtCdNkhCCqYOm3vB7y8rKOHjwIJmZmfz0008UFBRgMpnw8vLCy8sLb29vQkJC8PT0tEp8a7HHnIQQtiVNkhCiUUwmE4mJiWzYsIGNGzeyfft2Kioqrvu+li1b0q5dOzw8PLjtttsICvr1bttSYykA7gZ3q+UthBCNJU2SEIKxb48F4Munvqx3n8zMTBYsWMDHH3/MsWPHAIiPj2fGjBn069ePsLAwfvjhB7y9vdHpdBQVFVFYWEhhYSGnT5/m6NGj7N69my1btqDT6Rg4cCDjxo3j7rvv5vGljwPa3Opfn4bURAjRtEmTJITg7oS72bplq9kLr8+cOcO3337L/v37qaqq4tZbb2X27NmMGjWKwMDAy/bdt29f3WNfX198fX0BiIuLA2qOQvXs2ZNVq1axcuVKZs2axaxZswidFEpgYCBHjhxh06ZNdWOkp6cDV18QbouLv+9OsL/rpIQQtiVNkhCCBwc8SMWhy0+ZZWZmsnr1avbs2YO7uzuDBw/mlltuISQkhLKyMlauXNnoODqdjh49etCjRw9effVV0tLS+PLLL3ln/zscP36c9u3bExoaWrePlh4c8KCm8YUQ2rtuk6QoSjiwEGgBqMA8VVXnWjsxIYQ2iouLWblyJdu3b8dgMDB69GiGDx+Ou7tlrhe68qiQv78/LVu2JCgoiI73dWTPnj18/fXXrFmzhoDxAXh5eZGbm4u/v79F4gshREM15EhSFfCMqqp7FUXxAvYoirJeVdWDVs5NCGEjY94aQ3p6Ot0Lu/PFF19QUlLCkCFDGDVqFF5eXjbJwcnJiSFDhjBkyBDy8/NJTk5mY/VGsrOzeeG/L9CxY0f69etHfHy8TfLRckkUIYR9uG6TpKpqBpBx8XGRoiiHgFaANElCNBEjO4zk7bVv8/HGj4mMjOTJJ58kLCzMZvF7BFx+as3X15fBgwdzOO0wlZWVRN4eyfbt2/n444/x9vamrKyMRx999JrTCtysiX0nWm1sIYRjUFRVbfjOihIFbAHiVFUtrG+/hIQEVRa4FcIx/PDDD9x3330UFBRw1113MWTIEPR6vdZpAfDvtH8DMD1mOiaTiUOHDvH999+TmppKQEAATz/9NDNnzrTZ0S4hRNOkKMoeVVUTrtze4LXbFEXxBFYAT5prkBRFmaYoSpKiKElZWVk3l60QwupUVeXNN99k+PDhBLUI4vkXn2f48OGaNEglVSWUVJVccx+dTkdsbCxPPfUU27dvp3fv3rz44ot06NCBlStX0pgPfA1RWV1JZXWlRccUQjiWBjVJiqI4U9Mg/U9VVbO3tKiqOk9V1QRVVRMunSROCGF/SkpKmDhxIn/4wx8YN24cUZOj+Lrwa83y+ezYZ3x27LMG79+3b1+++eYbduzYQVBQEOPHj+fOO+/k1KlTFstp3L/GMe5f4yw2nhDC8TTk7jYFmA8cUlX1n9ZPSQhhaZfeUVZSUsI777zDqVOnGDt2LMOGDSMlN4XIwEgNM7wxffr0ISkpiXfeeYeXX36ZTp06MXfuXKZOnUrNr64bN7n/ZAtlKYRwVA25u60/MBn4WVGUlIvbXlBV9VvrpSWEsIaioiLmzp1LRkYGjz76aN2dYt0CummcWeNcOY2Ap6cnL7/8MgsXLuR3v/sdn3zyCRs2bLipaQt+0+c3N5umEMLBXfd0m6qq21RVVVRV7aKqavzFL2mQhHAwBQUFvPXWW2RmZvLYY49ddiu90WTEaDJqmN3N8/f3Z9asWdx+++3s3LmTvn37cuTIkRser9RYWremnBCieWrwhdtCCMeVl5fHW2+9RU5ODk8UCU9xAAAgAElEQVQ88UTdMiG1Pj3yKZ8e+VSj7CxHp9Nx55138sQTT3D27FkSEhJYv379DY1137v3cd+791k4QyGEI5FlSYRo4oqKinjvvffIz8/n97//PW3btr1qnz5BfTTIzHrx4+Li+L//+z8++OADRo0axUMPPUSvXr0u2+d66789PPBhi+YkhHA80iQJ0YRVVVVx3333kZ6ezhNPPGG2QQLo6t/VxplZP35AQADPPPMMH374IfPnz6egoIDhw4c3+P3jesqdbUI0d3K6TYgmSlVVZs2axdq1a5k4cSKxsbH17lteXU55dbkNs7tcvjGffGO+xcd1d3dn1qxZdO/eneXLl7N8+fIGz6dUWFZIYVm9c+YKIZoBaZKEaKLefvttPvzwQ5599lkGDhx4zX3/e/S//Pfof22U2dW+OPEFX5z4wipjOzs787vf/Y5Bgwaxfv16Pv/88wY1Svd/cD/3f3C/VXISQjgGOd0mhIO78nZ4gAMHDvDee+/RvXt3Wrdufd0x+gf3t0ZqdkOn0zFhwgScnZ3rLuSeNm3aNedSmjb42tcsCSGaPmmShGhi8vPzWbBgAaGhoTz88MPodNc/YBznF3fdfRydoiiMHz8egPXr1zNz5kzefffdehulMd3H2DI9IYQdkiZJiCbEZDLxySefUFFRwe9+9zsMBkOD3le7bpqHk4c109PcpY3S+++/D1Bvo5RTnANAgGeA7RIUQtgVaZKEaELWrl1LWloaU6ZMoWXLlg1+X+26adNjplsrNbtR2yh17dqVN998E1dXV954442rGqWH/v0QAGueWaNBlkIIeyBNkhBNxJEjR1izZg29evWiX79+jXrvgBYDrJSVfcZXFIU5c+ZQXl7OW2+9hbu7O6+++upl+8wYNsOmOQkh7I80SUI0AaWlpcyfP5+goCDuv//+Ri/u2sm3k5Uys9/4iqIwd+5cysrKeO2113Bzc+P555+ve31k15E2z0kIYV+kSRKiCVixYgX5+fk899xzuLq6Nvr9RZVFAHg5e1k6tQbJKs8CIMg1yGYxa+8KTEhI4Oeff+aFF14gJSWFoUOHAjU1eeCBB2jh08JmOQkh7IvMkySEg0tNTWXbtm0MHz6cqKioGxpj8fHFLD6+2LKJNcLKUytZeWqlJrF1Oh0PPfQQ3bp1Y+nSpWzZsgWoqckjHz+iSU5CCPsgR5KEcGClpaUsWrSI4OBgxoy58VvWB4UMslxSDkiv1/PII4/w4YcfsnjxYgwGA4M6DmLUiFFapyaE0JAcSRLCgb3yyitkZ2czefLkBt/ub06MTwwxPjEWzMzxODk5MX36dNq3b8+CBQsoPlrMsLhhWqclhNCQNElCOKjdu3fz9ttvM3DgQNq3b39TY1lr7TRHYzAYmDFjBtHR0cxfMp8FSxdonZIQQkPSJAnhgIxGI1OnTqVly5aMG3fzq9Vbc+00R+Pi4sLMmTPxG+HH7//3+7plTIQQzY80SUI4oL///e8cOHCADz/8EDc3t5seb0jLIQxpOcQCmTlm/Cu5ubkxoesEQnJDuOuuu+ou5hZCNC/SJAnhYA4ePMjrr7/OxIkTb+pi7Uu1825HO+92FhnLEeOb0zm4M1tXbCUyMpLbb7+dXbt2aZ2SEMLGpEkSwoFUV1czdepUvL29mTt3rsXGzanIIacix2LjNVZ6aTrppemaxTcnpyKHUqWUjRs30qJFC0aOHEliYqLWaQkhbEiaJCEcyHvvvcfOnTt55513CAqy3MSLy08uZ/nJ5RYbr7HWnFnDmjP2tUba8pPLmblwJqGhoWzcuBFfX1+GDBnC5s2btU5NCGEj0iQJ4SBOnDjBCy+8wO23387EiRMtOvbw0OEMDx1u0TEd3fDQ4Tw35jkAIiMj2bZtGxEREYwcOZLVq1drnJ0QwhakSRLCAaiqyrRp09Dr9Xz44YeNXpvteqK9oon2irbomI4u2iua/u371z1v1aoVW7ZsoUuXLowbN47PPvtMw+yEELYgM24LYefmzZvHTz/9xIYNG5g0aRJr1661eAwt1k6zd1nlWfzlnb9cVZMHHniAwsJCJk+ezLJly/jqq68s3rQKIeyDHEkSws4VFBSwfPly2rZty4ABA6wSQ8u10+xVfTVxdXVl5syZ9O7dm9WrVzNx4kRKS0s1yFAIYW1yJEkIO7dkyRKMRiNTpkxBp7PO55qRrUZaZVxHiW/OtXJydnbm4YcfJjQ0lKVLl3L06FG++uorwsLCbJihEMLa5EiSEHZsxYoVJCcnM2bMGFq0aGG1OJGekUR6RlptfHuPb871clIUhZEjR7Jq1SrS0tJISEjg+++/t2GGQghrkyZJCDuVl5fHjBkziIiIYPhw6955llmWSWZZplVjXMup4lOcKj6lWXxzGlqTMWPGsHPnTgICAhgxYgTPPvssRqPRBhkKIaxNmiQh7NTMmTPJzs5mypQp6PV6q8ZadXoVq06vsmqMa1l3bh3rzq3TLL45jalJbGwsiYmJPProo7zxxhv079+fo0ePWjlDIYS1yTVJQtihJUuW8L///Y8///nPhISEWD3e6LDRVo/haBpak3nz5tU97tatG9OnT2fRokXExsZy1113sWTJEqs3uUII67jukSRFUT5RFOWCoigHbJGQEM3dqVOneOyxx+jXrx8vvPCCTWKGe4QT7hFuk1iO4kZr0r17d1555RViYmJYtmwZAwYM4NChQ1bIUAhhbQ053bYAsL9bT4Rogqqrq5kyZQomk4lFixbh5GSbg732uHaa1m6mJn5+fsyYMYPf/va3pKWlER8fz1//+lcqKystnKUQwpqu+xtYVdUtiqJEWT8VIcScOXPYsmULCxYsIDradjNg166bNj1mus1i2rubrYmiKPTu3ZuOHTvy+eef8+KLL/LRRx8xZcoUIiIiLtt32rRpN52vEMLy5JokIezE7t27eeWVV7j33nuZMmWKTWOPCR9j03j2Ft8cS+Xk7e3NtGnTSE5OZvHixfztb39jxIgR3H777Tg7O1skhhDCOizWJCmKMg2YBlz1KUkIcW0XLlxg/PjxtGrVio8++sjmy1yEuofaNJ69xTfH0jl169aN9u3bs2zZMtauXUtKSgpTpkyx6RFDIUTjWGwKAFVV56mqmqCqakJQkKz/JERDVVVVMWHCBLKzs1m5ciX+/v42z+FMyRnOlJyxedxaRwqPcKTwiGbxzbFGTTw8PHjooYeYOXMm5eXlzJkzh6VLl1JSUmLROEIIy5B5koTQ2PPPP8+mTZv46KOP6N69uyY5fHv2W749+60msQF+yPiBHzJ+0Cy+OdasSVxcHH/6058YOHAgGzdupEuXLvzwg339+4UQDTjdpijKEmAQEKgoyllgtqqq862dmBDNwbRp0/jPf/7DoEGDqKiouGzOHVu6K+IuTeLaM2vXxNXVlUmTJpGQkMCqVasYOnQoU6dO5Y033sDPz8+qsYUQDXPdI0mqqk5UVbWlqqrOqqqGSYMkhGWkpKSwcOFC2rRpw7333qtpLiFuIYS4WX/SSkdiq5q0b9+e/fv388c//pEFCxbQqVMnVqxYYfW4Qojrk7vbhNDAmTNnuP3223F3d2f69Ok2mw+pPrXrptnbIrNasmVNFi1aRHR0NM899xwLFy7knnvuIT4+nokTJ+Lr6wvINAFCaEGuSRLCxgoKChg9ejTFxcXMnDkTHx8frVOyy7XTtKZFTSIiInj++ecZO3Ysv/zyC3/605/Ytm0bqqraNA8hRA05kiSEDVVWVnLPPfeQmprKunXrOHbsmNYpATAuclyzjm+OVjnp9XpGjhxJt27d+Oyzz1i0aBG7du2if//+xMXFaZKTEM2VHEkSwkZUVWXatGls2LCBjz/+mKFDh2qdUp0g1yCCXLWbukPr+OZonVOLFi146qmneOCBBzh79izx8fE8/fTTFBQUaJaTEM2NHEkSwkouvVNNVVVWrFjB+vXrueOOOzS9k82c40XHAYj20mZiw4P5BwHo5NtJk/jmaF0TAJ1Ox4ABA+jWrRuHDx/mX//6F4sXL+avf/0rDz74IHq9XrPchGgO5EiSEDawdu1a1q9fz6BBg7jjjju0Tucq69PXsz59vWbxt57fytbzWzWLb47WNbmUp6cnH330EYmJiURHRzN16lTi4+NZu3atXK8khBVJkySElW3evJlVq1bRu3dv7rvvPpsvOdIQ90Tdwz1R92idhl2xx5r06NGDn376iWXLllFeXs7o0aMZOnQou3bt0jo1IZokOd0mhBXt2rWLJUuW0KVLFx588EF0Ovv8XBLgEqB1CnbH3mpy5enZp556iq1bt/LNN9/Qp08fRo0axezZs+ndu7dGGQrR9Njnb2whmoDExEQ+/fRT2rdvz7Rp0+z6+hF7XDtNa/ZeEycnJwYPHszrr7/O2LFj2b17d12ztHXrVjkNJ4QFSJMkhBV88cUXzJ8/n7Zt2zJjxgycnZ21Tuma7HHtNK05Sk1cXV0ZOXIkJ0+e5B//+AdJSUkMHDiQvn37snz5cqqrq7VOUQiHpVjj00ZCQoKalJRk8XGFcARffPEF999/P61bt2bmzJm4urpqndJ15RvzAfA1+NpV/H+n/RuA6THT7SYne2c0GtmxYwfr168nKyuLwMBABg0aRL9+/Xjqqae0Tk8Iu6Qoyh5VVROu3C7XJAlhQUuWLGHy5Mn069ePe+65xyEaJNC+EdA6vjn2mFNDGAwGbr31VgYMGEBKSgobNmxg+fLlrFq1igMHDvDYY4/Ro0cPu7yBQAh7I6fbhLCQd999l/vvv5/+/fvz7bffOkyDBJBWkEZaQZpm8ffl7mNf7j7N4pujdU1ulk6no3v37jz77LO89NJL9O3bly+++IKePXvSuXNn3nzzTTIyMrROUwi7Jk2SEDdJVVVeeuklZs2axZ133sm6devw9PTUOq1G2Zy5mc2ZmzWLvzNrJzuzdmoW3xyta2JJ4eHh3H///Zw7d46PPvoIb29v/vCHPxAWFsbo0aP54osvKC8v1zpNIeyOnG4T4iZUVVXx6KOPMn/+fB555BE+/PBDnJwc78dqUvQkrVOwO02xJj4+PkyfPp3p06eTlpbGwoULWbhwIRMmTMDHx4f77ruv7nSxvU5XIYQtOd5vcyHsRFZWFhMnTmTjxo289NJLvPrqqw57nYeXs5fWKdidpliTK+daioyM5MUXXyQtLY0dO3bw6aefMm/ePCIiIpgwYQITJ06ka9euDvvftRA3S5okIW7Azp07GT16NEVFRUyZMoXw8HD+85//aJ3WDbPHtdO01lxqotPp6NixIx07dmTSpEn4+/uzePFi3nrrLebMmUOHDh2YNGkSEydOpG3btlqnK4RNyfFUIRpBVVXef/99Bg4ciF6v549//CP9+/fXOq2bZo9rp2mtOdbE1dWV0tJS7r77bubMmcOkSZOorq7mlVdeoV27dkRFRfHPf/6Tc+fOaZ2qEDYh8yQJ0UBHjhzh8ccfZ8OGDdxxxx0MGTIEDw8PrdOyiJKqEgA8nLT599QXX8t5krSuiT3Jzc0lKSmJxMRETp8+jaIo3HrrrUycOJGxY8cSFBSkdYpC3JT65kmSI0lCXEdFRQWvvvoqnTt3Zvfu3bz//vusWrWqyTRIUNMIaNkMaB3fHHvMSSv+/v7cdtttvPjii6SmpjJ79mzS09OZPn06ISEhDBkyhA8++ECmFBBNjjRJQtSjsrKShQsX0rlzZ2bPns3YsWNJTU3l8ccfb3J3/hzIO8CBvAOaxU/KTiIp276OPmtdE3v1448/0rJlS55++mleeuklRo4cycGDB5kxYwatWrWiXbt2zJ07lzNnzmidqhA3TS7cFuIK5eXlTJ06le+++46cnBxatWrFrFmziI2NZc2aNVqnZxU/XfgJgDi/OE3i78nZA0BC4FVHuzWjdU3snaIohIeHEx4ezp133klGRgZ79uwhOTmZJ598kieffJI+ffowfvx4xo8fT+vWrbVOWYhGk2uShLjo0KFDzJ8/n4ULF5KVlUXr1q0ZNWoUXbp0afK3QJdX10wk6KrXZpbw+q490vKaJK1r4sgyMzPZu3cvycnJnD59GoCIiAh69OhBt27dePnllzXOUIjLydptQpiRk5PDihUrWLBgATt27MDJyYk777yTyMhIYmJimnxzVEsagatJTW5cSEgIo0ePZvTo0WRlZdU1TF9++SVffvkly5cv55577mH8+PF06tS0p1gQjk2aJNHsFBcXs3r1apYsWcK6deuoqqqiY8eOvPnmm0yePJng4OCrJt1r6mrXTevq31XjTOyH1MQygoKCGDFiBCNGjCA3N5e9e/eSkZHB7NmzeeWVV+jYsWNdw9QcjtoKxyJNkmgWjEYj69atY8mSJaxcuRKj0Yifnx+DBw+mV69ehIeHoygKX331ldapaqJ23TRpCH4lNbE8f39/hg0bBsCYMWNITk5m7969vP7667z22msEBATQuXNnOnfuzJw5cxxqkWjRNMk1SaLJqqqqYvPmzXz++eesXLmSvLw8AgICiI2NpWfPnrRt27bJ3aV2o4wmIwAGncGu4mt5TZLWNWlOCgsL2b9/P/v37+fQoUMYjUbc3Nzo378/Q4YMYfDgwSQkJDjkuojCMcg1SaJZMJlM/PGPfyQxMZG9e/dSVFSEi4sLXbt2pVevXnTq1Am9Xq91mnZH60ZA6/jm2GNOTZW3tze33HILt9xyC0ajkcOHD6MoCps2beKFF14AwM3NjR49etCzZ0969epFt27daNOmjTROwqrkvy7h8Kqrq0lMTGTp0qUsXbqUc+fO4ezsTJcuXUhISCAuLg6DQf7gXUtyTjIA3QK6aRJ/x4UdAPQN7qtJfHO0rklzZTAYiIuLY9q0aQBcuHCBzZs389NPP5GYmMgHH3zA22+/Xbdvhw4diI2NpU2bNkRGRhIZGUlUVBTh4eFyuk7cNGmShENKT09n48aNrF27lu+//56cnBycnZ0ZNWoUI0eOpEuXLvILshF2Z+8GtGsI9uftB+yrSdK6Js3dlTdPxMbGEhsby+TJkzl37hznzp0jPT2d9PR0vv/+e3Jzc7ny8pGQkBAiIyOJiIggKCiIgIAAAgMDCQwMvOyxv78/np6ectG4uEqDmiRFUUYCcwE98LGqqn+3alZCXKSqKpmZmaSmppKcnMzOnTvZtWtX3dwrwcHBjB49mpEjRzJq1Cj8/Pya3Z1plvBI+0e0TsHuSE3sk16vJyIigoiIiMu2V1dXk5eXR25uLjk5OeTm5hIUFMSpU6fYv38/2dnZZhupWk5OTvj6+uLr64ufn99VX7Xb/f39CQoKokWLFgQHB+Pr6yvNVRN23SZJURQ98D4wHDgLJCqKslpV1YPWTk40baqqkpOTQ0ZGBhkZGaSnp1/2+PTp06SlpVFYWFj3noCAAFq3bk3v3r1p164d4eHh6HQ6iouLWbZsmYb/GsemV+Q6rStJTRyLXq+vOzJ0qV69etU9NplMlJaWUlxcTHFxMSUlJRQVFVFSUkJZWRmlpaWUlpZSUlJCVlZW3fPy8nKqqqrMxnV2diY4OLjuq7Z5Mvc4KChITv07mIYcSeoFHFVV9TiAoiifA3cB0iQ1Q9XV1RiNRoxGIxUVFVRUVNT9wjH3lZ+fT15eHnl5eaSkpNT9Aqr9bjKZrorh7e1Ny5YtCQsLY/LkyXTo0IEOHToQFxfH6tWrNfhXN32166bZ07IgWpOaND06nQ5PT088PT0b9T5VVamoqKj7vVVcXExRURGFhYUUFRXVPU5LSyMpKYnCwkIqKyvNjuXq6oqHh8cNf7m7u9e73WAwyFEtC2tIk9QKuHSlwrNAb+uko63FixeTk5MDUHdI9tLvDd12o6/dzFgmk4mqqiqqq6vrvl/6+FrbKisr6xqf2ubH3GOj0Uh1dXWj61p7mLq6uhoPDw/8/Pzqfqi9vb3x8fG57MvFxeWqMY4fP87x48cbHVs0jD2unaY1qYmopSgKrq6uuLq64u/vf939a5uqSxup2u/l5eV1v1crKiooLy+noKDgsm213+s7elUfvV5/WdPk5OSEXq9v1JdOp6v73tjH19p2rdcubeyufOzk5MSjjz7aqDpY0nXnSVIU5R5gpKqqj1x8PhnorarqE1fsNw2YdvFpDJBm+XQbLBDI1jC+PZPa1E9qUz+pjXlSl/pJbeontamfVrWJVFU16MqNDTmSdA4Iv+R52MVtl1FVdR5gF1fMKoqSZG5SKCG1uRapTf2kNuZJXeontamf1KZ+9labhkw3nAi0UxSltaIoBmACIBeGCCGEEKJJu+6RJFVVqxRFeQL4jpopAD5RVfUXq2cmhBBCCKGhBs2TpKrqt8C3Vs7FkuzitJ+dktrUT2pTP6mNeVKX+klt6ie1qZ9d1cYqC9wKIYQQQjg6WQJdCCGEEMIMaZKEEEIIIcyQJkkIIYQQwgxpkoQQQgghzJAmSQghhBDCDGmShBBCCCHMkCZJCCGEEMIMaZKEEEIIIcyQJkkIIYQQwgxpkoQQQgghzJAmSQghhBDCDGmShBBCCCHMkCZJCCGEEMIMJ2sMGhgYqEZFRVljaCGEBRw9fxSAti3aapyJEEJob8+ePdmqqgZdud0qTVJUVBRJSUnWGFqIZmHS+5MAWDxjsUXHHfPWGABa0hKANc+ssej4YL3chRDCWhRFOWVuu1WaJCHEzRnYYaDWKdwwR85dCCEuJU2SEHbo0aGPap3CDXPk3IUQ4lJy4bYQQgghhBlyJEkIO3TvO/cCsGzWsmvul5KSctW2+Ph4q+TUUA3NXQgh7J00SULYoZFdRlpl3Lt73A3AV3u+ssr4YL3chRDC1qRJEsIOTR00tcH7mkwmSktLqaysxN3dHVVVURTlmuNas0lqTO5CCGHPpEkSwkGdO3eO3bt3U1BQgMlkqtu+Y8cOWrduTVxcHFFRUej1+rrXSo2lWqQqhBAOSZokIezQ2LfHAvDlU18Cl197VFlZyeHDhzl79iyurq5ERETg5eWFs7MzpaWluLi4cPjwYVJTU/H09KRXr1507doVg8HAfe/eZ/PchRDCUUmTJIQdujvhbrPbKysrSUxMpLi4mMjISNq2bYuT09U/xsHBwWRnZ3Pq1Cl++OEHtm3bxoABA655Ks7auQshhKORJkkIO/TggAev2lZVVcWePXsoLi6me/fuBAYG1vt+nU5HcHAwwcHB5OXlcfToUTZu3EjmhUx8/Xxxc3UDK/VK5nIXQghHJPMkCeEATCYTycnJFBYWEh8ff80G6Up+fn4kJCQwfvx4ALKysriQdYFKY6W10hVCiCbhuk2SoiiuiqLsVhRln6IovyiK8mdbJCZEczbmrTF166wBHDlyhNzcXOLi4ggODm70eIqi0LZtW0JahuDn64fRaCQjM4ONGzdiNBotmfpVuQshhKNqyOm2CmCIqqrFiqI4A9sURVmrqupOK+cmRLM1se/EuscnTpzg5MmThIeHExoaesNjpqSkcEvYLQCsPbSWsrIykpKSOHLkCLfddhvR0dE3nTdcnrsQQjiy6zZJqqqqQPHFp84Xv1RrJiVEczep3yQASkpK+Oabb/Dw8CAmJuamxx3ZoWaix+/SvsPDw4NevXrxyy+/sGzZMkJDQ4mJicFgMAA3PnN3be5CCOHoGnRNkqIoekVRUoALwHpVVXdZNy0hmrfK6koqqyv57rvvKC8vp2vXrpfNd3SjCsoKKCgrqHvu5+dH3759iY6OJiMjg23btpGZmXlTMWpzF0IIR9egu9tUVa0G4hVF8QW+VBQlTlXVA5fuoyjKNGAaQEREhMUTFaI5GfevcZSWlnKb4TZuvfVWXF1dLTLun77/01Xb9Ho97dq1IyQkhAMHDrBv376665/MTS9wPeP+NQ6ANc+sudl0hRBCU426u01V1XxgE3DV4kyqqs5TVTVBVdWEoKAgS+UnRLM0ofcE2ujaEBQURM+ePW0S08vLi969exMVFcWZM2f47LPPKCwsbPQ4k/tPZnL/yVbIUAghbKshd7cFXTyChKIobsBwINXaiQnRnIVUhNDGqQ0jRoywyGm2htLpdMTExNCtWzfy8vJYtGgRFy5caNQYv+nzG37T5zdWylAIIWynIcfSWwL/VRRFT01TtVRV1a+tm5YQzVdmZiY7E3fSpUsXWrVqpUkOwcHBuLm5sWfPHhYtWkR8fDwBAQENupi7dn04d4O7tdMUQgirasjdbfuBbjbIRYhmLzk5mZ07d7I8cznry9YTHhquWS5eXl706dOHPXv2sHfvXnr06NGg99WuDyfXJAkhHJ0sSyKEHTl9+jSFhYWMiR2Dn5+fxce/M/ZOAFb/srpB+7u6utKzZ092797N3r17ad++PWFhYdd8z8MDH77pPIUQwh7IsiRC2InCwkKOHDlCYGAgYxPGMrjtYIvHGNx2cKPHNRgM9OzZExcXF5YtW0Z6evo19x/Xcxzjeo67mTSFEMIuSJMkhJ3YuHEjqqrSsWNHSowlFFcUX/9NjXSh+AIXiht3ITaAi4sLPXv2xN3dnaVLl15zLqXCskIKyxp/V5wQQtgbaZKEsAOnTp3i8OHDREdH4+7uzsvrXubldS9bPM7fNv6Nv2382w2919XVlQkTJuDi4sIXX3xBVlaW2f3u/+B+7v/g/ptJUwgh7IJckySERlJSUgAwmUzs2LEDNzc3oqKiABjbeayGmdXPx8eHCRMmsHjxYj7//HMmTpxIYGDgZftMGzxNo+yEEMKypEkSQmNnz56luLiY+Pj4ujmRBkYP1Dgr82obu/j4eHbv3s2iRYvo1asX/fv3r9tnTPcxWqUnhBAWJafbhNCQ0Wjk6NGj+Pv7ExwcXLf9yjXW7I2Hh0fdTOCJiYnk5ubWvZZTnENOcY5WqQkhhMVIkySEho4dO0ZlZSUdOnRAUZS67X/6/k9m11mzJ56eniQkJKCqKosXL667Rumhfz/EQ/9+SNvkhBDCAuR0m03slJ4AABlTSURBVBAaKSoq4syZM4SHh+Pl5XXZa/d2vdcqMWvHXbZvmUXG8/LyomfPnuzbt48lS5Zw3333MWPYDIuMLYQQWpMjSUJoQFVVUlNT0ev1tG3b9qrX+0X1o19UP4vHtca4np6eTJo0CWdnZ5YsWUKXoC6M7HrVGthCCOFwpEkSQgNHjhwhNzeXtm3bYjAYrno9tzSX3NJcM++8OafzTnM677TFx/Xz82PSpEm4ubkx/3/zST6UbPEYQghha3K6TQgbq66uZtOmTXh6ehIebn5tttfWvwbA23e9bdHYb2+x7Hi1au9669q1K48vfZwVH63g7bvfZvBgy88aLoQQtiJNkhA2lpycTH5+Pt27d0enM38wd2K3iTbOyjJcXV2Z2n8qaWlpJCcnYzAY8PDwuGyf+Ph4jbITQojGkdNtQthQeXk527dvJzIy8qpJGC/VK6IXvSJ62TAzy+nfpj8TBk1AURT27NlDRUWF1ikJIcQNkSZJCBvauXMnZWVlDBo06LJb/q90o2us2YMLxRcoNhXTvXt3jEYjycnJmEwmrdMSQohGkyZJCBspLCwkKSmJ2NhYQkJCrrnvzayxprXa3H18fIiLi6OgoICjR49qnZYQQjSaXJMkhI1s3boVgAEDBlx33wd6PGCVHGrH/WzPZ1YZ/9IYACEhIeTk5HDixAkCAgIICAiwWlwhhLC06x5JUhQlXFGUTYqiHFQU5RdFUX5vi8SEaErOnz/PgQMH6NGjBz4+Ptfdv0dYD3qE9bB4HtYa91oxYmJi8PDw4Oeff8ZoNFo1thBCWFJDTrdVAc+oqtoJ6APMUBSlk3XTEqJp2bx5M66urvTt27dB+6cXppNemG7xPI5mH+VotnVPfV2Zu5OTE126dMFoNJKammrV2EIIYUnXPd2mqmoGkHHxcZGiKIeAVsBBK+cmRJOwYcMGTp48SUxMTIObhDc2vQFYfp6k939636LjmWMud29vb6Kjozl27BjHjh2jTZs2Vs9DCCFuVqOuSVIUJQroBuyyRjJCNDUmk4m0tDTc3NyIiIho8Pse6vmQ9ZKysvpyj46OJjMzk++++46pU6fi4uJi28SEEKKRGnx3m6IonsAK4ElVVQvNvD5NUZQkRVGSalcDF6K5+/nnnykuLqZ9+/b1ThxpTtfQrnQN7WrFzKynvtx1Oh1xcXEUFRWxZcsWDTITQojGadBvbUVRnKlpkP6nqupKc/uoqjpPVdUEVVUTgoKCLJmjEA6poqKCrVu34uvrS4sWLRr1XmutsWYL18rd19eXHj16sHfvXs6ePWvjzIQQonEacnebAswHDqmq+k/rpyRE07Br1y5KSkqIiYm55sSR5ry95W2rrbNmbdfLfeDAgXh7e7N27VqqqqpsmJkQQjROQ65J6g9MBn5WFCXl4rYXVFX91nppCeHYCgsLSUxMpFOnTvj6+jb6/VN7T7VCVr+OO3/XfKuMf2mM+hgMBkaMGMGyZcvYvn07AwcOtFouQghxMxpyd9s2oHEfg4Vo5n788UcAbr31Vo4fP97o98eFxFk6JauO29gY0dHRxMbGsmvXLjp06EBwcLDV8xJCiMaSZUmEsLD09HQOHjxIz5498fb2vqExTuSc4ETOCQtnBgcyD3Ag84DFx71UQ3MfOnQorq6urFu3TtZ2E0LYJVmWRAgLSUlJQVVVdu/ejcFgwN3dnZSUlOu/0Yx3tr0DWH6eJGueZqvV0Nzd3NwYOnQoa9asISkpiV69elk9NyGEaAxpkoSwoPPnz5Ofn09sbCxOTjf+4zW973QLZmVbjcm9Y8eOHDx4kK1bt9K+ffsbun5LCCGsRU63CWEh1dXVHD58GE9PT1q1anVTY3UI7kCH4A4Wysy2GpO7oij/3969x0Z13Qkc//48ZmzDBOMHYBtsYxfbxJhnjUMSlE2aR0myCTRJd6nU3WS3VbfdditltdU2iqqtWq3a3X+iXW2ktmqzJItEuyFJAcU8gwnKYpdXwNT4gbEJg8EFbB61GRtszv4xd9zBzOCxfe+8+H2k0dzXnPubn8/MHN/HOTz11FOkpKSwbds2Pe2mlIoreiRJKZt0dHTg8/lYsWLFuG/5Hy0wvtr83Pl2hBZVY8Ue6hRkWVkZTU1NNDQ08NBDDzkan1JKRUqPJCllg97eXjo7O8nPzyc7O3vS5b35f29GZZw1J0wk9jlz5lBZWcknn3zCmTOJ2YmmUir56JEkpSbJGMOuXbtwuVxUVFTYUua3H/62LeWEK9fJBthEYg+cduvu7mbr1q288sorTJs2zYHolFIqctpIUmqSWltbOX36NAsWLLBt0FanTrNF4/TdRPfR3NzMggULaGhoYMOGDaxYsYKUlBSWLl1qc4RKKRUZPd2m1CQMDAywe/duZs+eTWFhoW3ltlxooeVCi23lBRw+e5jDZw/bXm6wycR+3333UVVVxZUrVzhx4gTGGJujU0qpyOmRJKUmoa6ujuvXr/PlL3+Z8+fP21buz+t/DtjfT9KGwxtsLS+Uycaen59PX18fHR0deDweli1bZmd4SikVMW0kKTVBZ86cobGxkZqaGmbPnm1rI+m7q75rW1nRZkfs8+fPp6+vj9bWVjo6OigtLbUhMqWUGh893abUBNy4cYPt27czY8YMVq1aZXv5JTkllOSU2F5uNNgRu4iwaNEiPB4PW7Zsoaenx6bolFIqctpIUmoCNm3axOXLl0f695no8CPhRGOMNafYFXtqairLly/H5XLx3nvvMTAwYEN0SikVOW0kKTVOnZ2deL1eiouLbekTKZRf/e5XURlnzQl2xp6RkcGXvvQlrl27xvvvv8/Q0JAt5SqlVCT0miSlxmFgYIDa2lqmTZtGWVmZY/t59ZFXHS33jX32XhAeah92mTt3Ls888wxbt26ltraW5557btI9miulVCS0kaTUOOzatYvr169TU1ODy+VybD9FWUUJVa6T+wicyiwrK6O5uRmfz0dFRYX2n6SUctyYjSQReQv4c+CCMabK+ZCUik8tLS2cOHGCVatWOd4b9LFzxwBYUrDE1nL3n95va3mhOBV7SUkJAwMDnD59GrfbrY0kpZTjIrkmaT2w2uE4lIprfX197Ny5k7y8PFauXOn4/tYfXM/6g+ttL/fdY+/y7rF3bS83mFOxiwj3338/eXl5tLW12X6xvFJKjTbmkSRjzD4Rmed8KErFJ2MM27Zt4+bNmzz77LOOnmYL+N5j33N8H05xMvZA1wDDw8Ps2LGD1NRUqqr0ALdSyhl6TZJSYzh48CAdHR088cQT5ObmRmWfBdMLorIfJzgde0pKCkuWLOHkyZN8+OGHDA8Ps2SJvaf2lFIKbOwCQES+ISKHROTQxYsX7SpWqZjau3cve/fuZdasWaSkpHD06NGonOaJxhhrTolG7C6XixdffJGSkhK2b9/O4cOJmSulVHyzrZFkjPmFMabaGFM9c+ZMu4pVKmZ8Ph/Hjh0jPT2dqqqqqN52vuHwhqiMs+aEaMU+ZcoUXnjhBebPn8/u3bvZt2+fDoirlLKVnm5TKoTh4WE2b97M4OAgDzzwAFOmTInq/l97/DVHy/3JRz9xpPzgfTgtcEQvcNdbfX09V69e5emnnyY1Vb/alFKTF0kXABuBR4FcETkL/IsxJjG7AlYqQnv27OGzzz6jqqqKzMzMqO9/lmdWQpUb7X0ES0lJobKykoyMDE6cOMGVK1dYu3Yt9913X1TjUEoln0jubvtKNAJRKl4cPXqUI0eOsGLFCseGHRnLgTMHAKgpqrG13Lr2OlvLC8Wp2O9GRCgtLWXhwoXU1tayfv161qxZQ1GR851nKqWSlx6TVipIe3s7O3fupLS0lEcffZTGxsaYxLHx042A/Q2NLU1bbC0vFKdij4TP52PFihV8+umnbNy4kc997nOUlpayfPnyqMeilEp82khSynLmzBl++9vfkpeXx/PPP09KSuzGf/7Bkz+I2b4nK9axezweHnzwQZqbmzl16hSXLl2iuLiYnJycmMallEo82khSCvj44485ePAg6enpLFiwgObm5pjGkz01Nqf57BAPsaemprJo0SJmzpxJU1MTb731FjU1NTz44IO43e5Yh6eUShDaSFL3PK/Xy8GDB3G73VRXV8fFj2hgjLWH5j0U40jGL55iz8vLIysri0uXLtHQ0EBTUxMPP/wwVVVVUek5XSmV2LSRpO5pnZ2dvP/++6Snp1NdXU16enqsQwIYGV8tHhoa4xVvsaelpfHss8+yePFi6urq2L59Ow0NDdTU1FBZWUlaWlqsQ1RKxSlxovO16upqc+jQIdvLVcpOjY2N7Nixg9zc3Lj7sbzquwpAZoa93Q8Eyv3hzh8C8MaaN2wtP3gfdsc+GUuXLgX84/B1dHTwySef0N3djcvloqCggKKiIjwez23bKqXuHSJy2BhTPXq5HklS95xbt27x8ccfc+DAAYqLi1m7di0tLS2xDus2TjUwotFwiafGUcDooWQWL15McXExXq+Xrq4uvF4vWVlZFBYWUlVVpZ1RKqUAbSSpe8yBAwc4fvw4ly5dorCwkPLy8rhrIAHs69gHwCOlj9ha7vaW7baWF4pTsdtJRJgxYwYzZsygoqJipKHU2NhIW1sblZWVLFmyhFmzotsxplIqvmgjSd0zzp49y/79+7lx4waVlZUUFhbGOqSwPjj+AWB/Q2NH6w5bywvFqdid4na7KSkpYd68efT29tLf38+xY8c4cuQIeXl5LF68mPvvvz9urldTSkWPNpJU0hseHmb//v3U19eTnp7OypUrmT59eqzDuqsfr/5xrEOYsESNXUTIycnh8ccfx+fz0dTURGNjIzt37mTPnj1UVFSwePFiCgsLozrYsVIqdrSRpJLaxYsXqa2tpbu7m6qqKmbNmhX1wWonwpPmiXUIE5bIscOfrl9KTU1l2bJlXLt2ja6uLlpbW2lqamLq1KnMnTuXOXPm4Ha79UJvpZKYNpJUUhoaGqK+vp6GhgbS0tJYu3YtFRUVd1zAG68CY6w9Nv+xGEcyfokc+2giQmZmJpmZmVRUVNDd3U1XVxdtbW20t7eTl5fH7Nmzyc/Pj3WoSikHaCNJJZ2Ojg4++ugjent7KSgooKKiAp/PlzANJPjTGGuJ2NBI5NjvxuVyMWfOHObMmcMf//hHvF4v586d45133iE/P5/Pf/7zVFRU6J1xSiUR7SdJJY2enh7q6uo4deoUWVlZlJaWkpubG+uwJmTg5gAA6VPsvVg4UO5rta8BzvST5FTs8ejmzZukpKRw5MgRent7mTp1KkuWLGHp0qVxf92bUupPtJ8klbTq6+s5deoUXV1duFwuysvLKS4ujukAtZPlVAMjGg2Xe6FxFBC4vq26upqenh68Xi/19fXU19dTXl4+0h+THl1SKjHpJ1clrHPnznHo0CFaWloQEYqLiykpKYmrnrMnalfbLgCeLH/S1nI3/36zreWF4lTs8UxEyM3NJTc3F5/Ph9frxev10tbWhtvtZt68eRQVFTF37lxmzpyZ0A14pe4l2khSCeXy5cu0tLTQ0tLChQsXcLvdFBUVMW/evKTqx6a2uRawv6Gx99ReW8sLxanYE0VGRgbl5eW88MILfPbZZ7S3t9Pe3k5bWxvgv2suOzubnJwccnJyyM3NJTs7m+nTpydFA1+pZBLRNUkishr4D8AF/NIY89O7ba/XJKnJMsbQ399PT08Ply5d4vz583i9Xq5duwZAQUEBCxcuZOHChTQ3N8c4WvsNDQ8BkOqy9/+YVze/etu8E9ckORV7IjPGMDAwwOXLl7l27Rr9/f309/fj8/lu2y4tLY3MzEymT58+8jx9+nQ8Hg8ej4dp06YlRBcWSiWaCV+TJCIu4E3gSeAscFBEthhjTtgfpkp2xhiuX79OX1/fbY/+/v475m/dujXyumnTpuHxeCgoKGDWrFlkZGQAJGUDCRK7gZHIsTtFRMjIyCAjI4OCgoKR5UNDQ1y/fp3+/n4GBgbw+XwMDAzQ3d1NZ2cnw8PDd5TldrtHGkyBR/B8YDojI0NP6yk1SZF8m9UA7caYDgAR+TWwBtBGUhIIHEk0xow8Qs0PDw9z69atkUeo+eHhYQYGBkYeg4OD+Hw++vr6uHDhAoODg9y4cYNQRy8zMjJGvuBzcnLo7+8nLS1t5D9ot9t9T/VyHBhjbfWC1TGOZPwSOfZoS01NHTlaFMrNmzfx+Xwjn53BwcGR6b6+Pnp6ehgcHAzZmBIRpk6dSnp6Om63O+zD5XKRkpJy2/PdpoPnRYSUlJSR5+DpUM9KJZpIGklzAG/Q/FngAWfCUXbp6enh7bffBsI3gJwmIqSnp+PxeJgyZQoej2fkCzs9PZ20tLSRh/7He7vAGGuJ2NBI5NjjzZQpUyI6vTY0NHRHIyowHVjn8/kYGhpieHiYoaEhhoaGovI9EExE7tpYsntdWVkZzz33XOQBKjXKmNckichLwGpjzNet+b8CHjDGfGfUdt8AvmHNVgCt9ocbsVzgUgz3H880N+FpbsLT3ISmeQlPcxOe5ia8WOWm2Bgzc/TCSI4kdQHBw6XPtZbdxhjzC+AXEw7PRiJyKNQFWEpzczeam/A0N6FpXsLT3ISnuQkv3nITyTmOg0CZiJSIiBtYB2xxNiyllFJKqdga80iSMWZIRL4D7MDfBcBbxpgmxyNTSimllIqhiO7VNcbUArUOx2KnuDjtF6c0N+FpbsLT3ISmeQlPcxOe5ia8uMqNIwPcKqWUUkolOr3vWimllFIqhIRtJIlItojsEpGT1nNWiG0eE5GjQY8BEVlrrVsvIp1B65ZG/104I5LcWNsNB73/LUHLS0TkdyLSLiK/sS7YTwoR1pulIlIvIk0i0igifxm0LqnqjYisFpFW62/9/RDr06w60G7ViXlB616zlreKyBejGXc0RJCbfxSRE1Yd+UhEioPWhfxsJYsIcvOKiFwMysHXg9a9bH3+TorIy9GN3FkR5OWNoJy0iciVoHXJXmfeEpELIvL7MOtFRP7Tyl2jiCwPWhe7OhPc0WAiPYB/B75vTX8f+Lcxts8GeoGp1vx64KVYv49Y5gboC7P8f4F11vTPgG/F+j1FMzdAOVBmTRcA54EZyVZv8N+IcQooBdzAMaBy1DZ/D/zMml4H/MaarrS2TwNKrHJcsX5PUc7NY0HfJ98K5MaaD/nZSoZHhLl5BfivEK/NBjqs5yxrOivW7ylaeRm1/T/gvxEq6euM9f4eAZYDvw+z/hlgGyDASuB38VBnEvZIEv6hUd62pt8G1o6x/UvANmPMdUejig/jzc0IERHgC8Cmibw+AYyZG2NMmzHmpDV9DrgA3NHJWBIYGXLIGHMDCAw5FCw4X5uAx606sgb4tTFm0BjTCbRb5SWLMXNjjKkL+j5pwN+H3L0gknoTzheBXcaYXmPMZWAXkCxds483L18BNkYlsjhgjNmH/0BFOGuAd4xfAzBDRPKJcZ1J5EbSbGPMeWu6G5g9xvbruLNC/qt1WO8NEUmzPcLYiTQ36SJySEQaAqchgRzgijFmyJo/i39ommQxrnojIjX4/ys8FbQ4WepNqCGHRv+tR7ax6sRV/HUkktcmsvG+v6/h/y84INRnK1lEmpsXrc/JJhEJdEiczPUm4vdmnZotAfYELU7mOhOJcPmLaZ2J6+G6RWQ3kBdi1evBM8YYIyJhb9OzWqOL8Pf1FPAa/h9JN/5bDv8Z+NFkY44Wm3JTbIzpEpFSYI+IHMf/I5jQbK43/wO8bIy5ZS1O6Hqj7CciXwWqgT8LWnzHZ8sYcyp0CUlpK7DRGDMoIn+H/2jkF2IcUzxZB2wyxgSPTHyv15m4FNeNJGPME+HWicgfRCTfGHPe+jG7cJei/gL4wBhzM6jswNGEQRH5b+CfbAk6SuzIjTGmy3ruEJG9wDLgPfyHOVOtIwchh6GJZ3bkRkSmAx8Cr1uHfgNlJ3S9GSWSIYcC25wVkVQgE+iJ8LWJLKL3JyJP4G98/5kxZjCwPMxnK1l+8MbMjTGmJ2j2l/ivBQy89tFRr91re4SxMZ7PxDrg28ELkrzORCJc/mJaZxL5dNsWIHCV+8vA5rtse8e5X+sHMnANzlog5BX3CWrM3IhIVuBUkYjkAg8DJ4z/Srk6/NdwhX19AoskN27gA/znxzeNWpdM9SaSIYeC8/USsMeqI1uAdeK/+60EKAMORCnuaBgzNyKyDPg58Lwx5kLQ8pCfrahF7rxIcpMfNPs80GxN7wCesnKUBTzF7Uf4E1lEQ3iJyAL8FyDXBy1L9joTiS3AX1t3ua0Erlr/lMa2zkTrCnG7H/ivi/gIOAnsBrKt5dXAL4O2m4e/JZoy6vV7gOP4f+Q2AJ5Yv6do5gZ4yHr/x6znrwW9vhT/D1478C6QFuv3FOXcfBW4CRwNeixNxnqD/46SNvz/sb5uLfsR/h9+gHSrDrRbdaI06LWvW69rBZ6O9XuJQW52A38IqiNbrOVhP1vJ8oggNz8Bmqwc1AELgl77t1Z9agf+JtbvJZp5seZ/CPx01OvuhTqzEf+dwjfxX1f0NeCbwDet9QK8aeXuOFAdD3VGe9xWSimllAohkU+3KaWUUko5RhtJSimllFIhaCNJKaWUUioEbSQppZRSSoWgjSSllFJKqRC0kaSUUkopFYI2kpRSSimlQtBGklJKKaVUCP8P2DThliuiDW8AAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 720x720 with 5 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "results_aopc.plot_aix_fidelity()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 196,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0\n",
      "100\n",
      "200\n",
      "300\n",
      "400\n",
      "500\n",
      "600\n",
      "700\n",
      "800\n",
      "900\n",
      "1000\n",
      "1100\n",
      "1200\n",
      "1300\n",
      "1400\n",
      "1500\n",
      "1600\n",
      "1700\n",
      "1800\n",
      "1900\n",
      "2000\n",
      "2100\n",
      "2200\n",
      "2300\n",
      "2400\n",
      "2500\n",
      "2600\n",
      "2700\n",
      "2800\n",
      "2900\n",
      "3000\n",
      "3100\n",
      "3200\n",
      "3300\n",
      "3400\n",
      "3500\n",
      "3600\n",
      "3700\n",
      "3800\n",
      "3900\n",
      "4000\n",
      "4100\n",
      "4200\n",
      "4300\n",
      "4400\n",
      "4500\n",
      "4600\n",
      "4700\n",
      "4800\n",
      "4900\n",
      "5000\n",
      "5100\n",
      "5200\n",
      "5300\n",
      "5400\n",
      "5500\n",
      "5600\n",
      "5700\n",
      "5800\n",
      "5900\n",
      "6000\n",
      "6100\n",
      "6200\n",
      "6300\n",
      "6400\n",
      "6500\n",
      "6600\n",
      "6700\n",
      "6800\n",
      "6900\n",
      "7000\n",
      "7100\n",
      "7200\n",
      "7300\n",
      "7400\n",
      "7500\n",
      "7600\n",
      "7700\n",
      "7800\n",
      "7900\n",
      "8000\n",
      "8100\n",
      "8200\n",
      "8300\n",
      "8400\n",
      "8500\n",
      "8600\n",
      "8700\n",
      "8800\n",
      "8900\n",
      "9000\n",
      "9100\n",
      "9200\n",
      "9300\n",
      "9400\n",
      "9500\n",
      "9600\n",
      "9700\n",
      "9800\n",
      "9900\n",
      "0\n",
      "100\n",
      "200\n",
      "300\n",
      "400\n",
      "500\n",
      "600\n",
      "700\n",
      "800\n",
      "900\n",
      "1000\n",
      "1100\n",
      "1200\n",
      "1300\n",
      "1400\n",
      "1500\n",
      "1600\n",
      "1700\n",
      "1800\n",
      "1900\n",
      "2000\n",
      "2100\n",
      "2200\n",
      "2300\n",
      "2400\n",
      "2500\n",
      "2600\n",
      "2700\n",
      "2800\n",
      "2900\n",
      "3000\n",
      "3100\n",
      "3200\n",
      "3300\n",
      "3400\n",
      "3500\n",
      "3600\n",
      "3700\n",
      "3800\n",
      "3900\n",
      "4000\n",
      "4100\n",
      "4200\n",
      "4300\n",
      "4400\n",
      "4500\n",
      "4600\n",
      "4700\n",
      "4800\n",
      "4900\n",
      "5000\n",
      "5100\n",
      "5200\n",
      "5300\n",
      "5400\n",
      "5500\n",
      "5600\n",
      "5700\n",
      "5800\n",
      "5900\n",
      "6000\n",
      "6100\n",
      "6200\n",
      "6300\n",
      "6400\n",
      "6500\n",
      "6600\n",
      "6700\n",
      "6800\n",
      "6900\n",
      "7000\n",
      "7100\n",
      "7200\n",
      "7300\n",
      "7400\n",
      "7500\n",
      "7600\n",
      "7700\n",
      "7800\n",
      "7900\n",
      "8000\n",
      "8100\n",
      "8200\n",
      "8300\n",
      "8400\n",
      "8500\n",
      "8600\n",
      "8700\n",
      "8800\n",
      "8900\n",
      "9000\n",
      "9100\n",
      "9200\n",
      "9300\n",
      "9400\n",
      "9500\n",
      "9600\n",
      "9700\n",
      "9800\n",
      "9900\n",
      "0\n",
      "100\n",
      "200\n",
      "300\n",
      "400\n",
      "500\n",
      "600\n",
      "700\n",
      "800\n",
      "900\n",
      "1000\n",
      "1100\n",
      "1200\n",
      "1300\n",
      "1400\n",
      "1500\n",
      "1600\n",
      "1700\n",
      "1800\n",
      "1900\n",
      "2000\n",
      "2100\n",
      "2200\n",
      "2300\n",
      "2400\n",
      "2500\n",
      "2600\n",
      "2700\n",
      "2800\n",
      "2900\n",
      "3000\n",
      "3100\n",
      "3200\n",
      "3300\n",
      "3400\n",
      "3500\n",
      "3600\n",
      "3700\n",
      "3800\n",
      "3900\n",
      "4000\n",
      "4100\n",
      "4200\n",
      "4300\n",
      "4400\n",
      "4500\n",
      "4600\n",
      "4700\n",
      "4800\n",
      "4900\n",
      "5000\n",
      "5100\n",
      "5200\n",
      "5300\n",
      "5400\n",
      "5500\n",
      "5600\n",
      "5700\n",
      "5800\n",
      "5900\n",
      "6000\n",
      "6100\n",
      "6200\n",
      "6300\n",
      "6400\n",
      "6500\n",
      "6600\n",
      "6700\n",
      "6800\n",
      "6900\n",
      "7000\n",
      "7100\n",
      "7200\n",
      "7300\n",
      "7400\n",
      "7500\n",
      "7600\n",
      "7700\n",
      "7800\n",
      "7900\n",
      "8000\n",
      "8100\n",
      "8200\n",
      "8300\n",
      "8400\n",
      "8500\n",
      "8600\n",
      "8700\n",
      "8800\n",
      "8900\n",
      "9000\n",
      "9100\n",
      "9200\n",
      "9300\n",
      "9400\n",
      "9500\n",
      "9600\n",
      "9700\n",
      "9800\n",
      "9900\n",
      "0\n",
      "100\n",
      "200\n",
      "300\n",
      "400\n",
      "500\n",
      "600\n",
      "700\n",
      "800\n",
      "900\n",
      "1000\n",
      "1100\n",
      "1200\n",
      "1300\n",
      "1400\n",
      "1500\n",
      "1600\n",
      "1700\n",
      "1800\n",
      "1900\n",
      "2000\n",
      "2100\n",
      "2200\n",
      "2300\n",
      "2400\n",
      "2500\n",
      "2600\n",
      "2700\n",
      "2800\n",
      "2900\n",
      "3000\n",
      "3100\n",
      "3200\n",
      "3300\n",
      "3400\n",
      "3500\n",
      "3600\n",
      "3700\n",
      "3800\n",
      "3900\n",
      "4000\n",
      "4100\n",
      "4200\n",
      "4300\n",
      "4400\n",
      "4500\n",
      "4600\n",
      "4700\n",
      "4800\n",
      "4900\n",
      "5000\n",
      "5100\n",
      "5200\n",
      "5300\n",
      "5400\n",
      "5500\n",
      "5600\n",
      "5700\n",
      "5800\n",
      "5900\n",
      "6000\n",
      "6100\n",
      "6200\n",
      "6300\n",
      "6400\n",
      "6500\n",
      "6600\n",
      "6700\n",
      "6800\n",
      "6900\n",
      "7000\n",
      "7100\n",
      "7200\n",
      "7300\n",
      "7400\n",
      "7500\n",
      "7600\n",
      "7700\n",
      "7800\n",
      "7900\n",
      "8000\n",
      "8100\n",
      "8200\n",
      "8300\n",
      "8400\n",
      "8500\n",
      "8600\n",
      "8700\n",
      "8800\n",
      "8900\n",
      "9000\n",
      "9100\n",
      "9200\n",
      "9300\n",
      "9400\n",
      "9500\n",
      "9600\n",
      "9700\n",
      "9800\n",
      "9900\n"
     ]
    }
   ],
   "source": [
    "ranks_f, scores_f, to_delete = results_aopc.aix_fidelity_rank(sensitivity_n=99)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 197,
   "metadata": {},
   "outputs": [],
   "source": [
    "ex_dict = dict()\n",
    "ex_dict[\"rate\"] = \"1px\"\n",
    "ex_dict[\"game\"] = \"deletion_game\"\n",
    "ex_dict[\"dataset\"] = \"CIFAR-10-original\"\n",
    "ex_dict[\"perturb\"] = \"mean\"\n",
    "ex_dict[\"method\"] = \"Gradient\"\n",
    "ranks_a, scores_a = results_aopc.aopc_rank(ex_dict)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 198,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "FriedmanchisquareResult(statistic=1917.8697600000014, pvalue=0.0)"
      ]
     },
     "execution_count": 198,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sps.friedmanchisquare(ranks_f[:,0],ranks_f[:,1],ranks_f[:,2],ranks_f[:,3])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 199,
   "metadata": {},
   "outputs": [],
   "source": [
    "ranks_ad = np.delete(ranks_a, to_delete, axis=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 201,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "SpearmanrResult(correlation=0.18225462189713443, pvalue=1.9966569488206764e-75)"
      ]
     },
     "execution_count": 201,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sps.spearmanr(ranks_ad[:,0],ranks_f[:,0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 202,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "SpearmanrResult(correlation=0.12036770532892237, pvalue=1.3602922451388048e-33)"
      ]
     },
     "execution_count": 202,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sps.spearmanr(ranks_ad[:,1],ranks_f[:,1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 205,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.018296384149816092"
      ]
     },
     "execution_count": 205,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sps.spearmanr(ranks_ad[:,2],ranks_f[:,2])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 206,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.axes._subplots.AxesSubplot at 0x7f9e9c9a8550>"
      ]
     },
     "execution_count": 206,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXQAAAD4CAYAAAD8Zh1EAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAAgAElEQVR4nO3deXxV1b338c8v85yQgcwDhDCEGcIkDihgUVtwrtDaarW0T6uPbe/weG9nve3tcO/tcJ2K1mvVCljrQBVFAQEnIEGQIUwhgQxkTshE5rOePxK8KSbkJDnjzu/9evEy5+yVs3/bnHyzz9prryXGGJRSSnk/H3cXoJRSyjE00JVSyiI00JVSyiI00JVSyiI00JVSyiL83LXj2NhYk5GR4a7dK6WUV9q3b1+NMSauv21uC/SMjAzy8vLctXullPJKInJmoG3a5aKUUhahga6UUhahga6UUhahga6UUhahga6UUhahga6UUhahga6UUhahga6UUhahga6UUhbhtjtFlVIK4IU9xUP+njUL0pxQiffTM3SllLIIDXSllLII7XJRystoF4UaiJ6hK6WURQwa6CLytIhUicjhS7RZIiIHROSIiOx0bIlKKaXsYc8Z+jPAioE2ikgU8Biw0hgzFbjNMaUppZQaikED3RizC6i7RJM1wMvGmOLe9lUOqk0ppdQQOKIPfSIwRkR2iMg+EfnKQA1FZK2I5IlIXnV1tQN2rZRS6gJHBLofMBe4Afgc8EMRmdhfQ2PMOmNMjjEmJy6u3yXxlFJKDZMjhi2WArXGmBagRUR2ATOBEw54baWUUnZyxBn6a8DlIuInIiHAAuCoA15XKaXUEAx6hi4i64ElQKyIlAI/BvwBjDFPGGOOishbwEHABjxljBlwiKNSSinnGDTQjTGr7Wjza+DXDqlIKaXUsOidokopZREa6EopZREa6EopZREa6EopZREa6EopZREa6EopZREa6EopZREa6EopZREa6EopZREa6EopZREa6EopZREa6EopZREa6EopZREa6EopZREa6EopZRGDBrqIPC0iVSJyyUUrRGSeiHSJyK2OK08ppZS97DlDfwZYcakGIuIL/BJ42wE1KaWUGoZBA90YswuoG6TZ/cBfgSpHFKWUUmroRtyHLiLJwE3A4yMvRyml1HA54qLob4H/Z4yxDdZQRNaKSJ6I5FVXVztg10oppS4YdJFoO+QAG0QEIBa4XkS6jDGvXtzQGLMOWAeQk5NjHLBvpZRSvUYc6MaYcRe+FpFngNf7C3OllFLONWigi8h6YAkQKyKlwI8BfwBjzBNOrU4ppZTdBg10Y8xqe1/MGHPXiKpRSik1bHqnqFJKWYQGulJKWYQGulJKWYQGulJKWYQGulJKWYQGulJKWYQGulJKWYQGulJKWYQGulJKWYQGulJKWYQGulJKWYQGulJKWYQGulJKWYQGulJKWYQGulJKWYQGulJKWcSggS4iT4tIlYgcHmD7l0TkoIgcEpEPRWSm48tUSik1GHvO0J8BVlxiexFwlTFmOvAwvYtAK6WUci17lqDbJSIZl9j+YZ+Hu4GUkZellFJqqBzdh34P8OZAG0VkrYjkiUhedXW1g3etlFKjm8MCXUSupifQ/99AbYwx64wxOcaYnLi4OEftWimlFHZ0udhDRGYATwHXGWNqHfGaSimlhmbEgS4iacDLwJ3GmBMjL0l5mxf2FA/5e9YsSHNCJUqNboMGuoisB5YAsSJSCvwY8AcwxjwB/AiIAR4TEYAuY0yOswpWSinVP3tGuaweZPu9wL0Oq0gppdSw6J2iSillERroSillERroSillERroSillERroSillERroSillERroSillERroSillERroSillERroSillERroSillERroSillERroSillERroSillERroSillERroSillEYMGuog8LSJVInJ4gO0iIr8XkQIROSgicxxfplJKqcHYc4b+DLDiEtuvA7J6/60FHh95WUoppYZq0EA3xuwC6i7RZBXwrOmxG4gSkURHFaiUUso+juhDTwZK+jwu7X3uM0RkrYjkiUhedXW1A3atlFLqApdeFDXGrDPG5BhjcuLi4ly5a6WUsjxHBHoZkNrncUrvc0oppVzIEYG+CfhK72iXhUCDMabcAa+rlFJqCPwGayAi64ElQKyIlAI/BvwBjDFPAJuB64EC4Dxwt7OKVUopNbBBA90Ys3qQ7Qb4tsMqUkopNSx6p6hSSlmEBrpSSlmEBrpSSlmEBrpSSlmEBrpSSlmEBrpSSlmEBrpSSlmEBrpSSlmEBrpSSlmEBrpSSlmEBrpSSlmEBrpSSlmEBrpSSlmEBrpSSlmEBrpSSlmEBrpSSlmEXYEuIitE5LiIFIjIg/1sTxORd0Vkv4gcFJHrHV+qUupSOrttNLR2uruMYbEZQ97pOv66r5S6lg53l+O17FmCzhd4FFgOlAK5IrLJGJPfp9kPgBeNMY+LSDY9y9JlOKFepdRFapvb+bCwlgPF52jt7OaKCbFcOzUBXx9xd2l2KW9o5bUDZymuO48An5SeY8mksVw1Mc5rjsFTDBrowHygwBhTCCAiG4BVQN9AN0BE79eRwFlHFqmU6l9jayeP7zxFe5eN7MQIAvx8eK+ghtO1LaxZkE5ksL+7S7yk+pYOnnqvCB+BW+emkBkXxuZD5Ww9WklHVzcrpiW6u0SvYk+gJwMlfR6XAgsuavMT4G0RuR8IBZb190IishZYC5CWljbUWpVSfdiM4S/7SujstnH/1RMYGxEEwMT4cP66r5RNB8q4c1GGe4scxG+3nqCts5v7l2aR0Fv/6vlpBOwr5YOCWnLSo4kND3Rzld7DnkC3x2rgGWPMf4rIIuA5EZlmjLH1bWSMWQesA8jJyTEO2rfXeWFP8ZC/Z80C/QOo/t77J2s4Vd3CTbOTPw1zgOnJkdQ2t/N2fiVFNS2Miw11Y5UDK6hq4vk9xcwfF/1pmF9w7dR4Dp9t4PVDZ/nqogxEtOvFHvZcFC0DUvs8Tul9rq97gBcBjDEfAUFArCMKVEp9VlVTG2/nVzAtKYKc9DGf2b54QiyRwf68ebgcYzzz3OlnbxwlJMCXpVPiP7MtPMifpZPHcqKymeMVTW6ozjvZc4aeC2SJyDh6gvwOYM1FbYqBpcAzIjKFnkCvdmShSnk6V37y2nm8Gl8fYeWs5H7PXv19fVg2JZ6/flzKobKGYe3DmfYW1fHu8Wq+f/0UQgP7j6FFmbHknq7nrSMVTEoI17N0Owx6hm6M6QLuA7YAR+kZzXJERB4SkZW9zf4B+LqIfAKsB+4ynnpaoJSXqz/fwSel55ifEU3YAGEIMDstioSIILYerfS4s/SNuSWEB/rx5YXpA7bx9RGuyIqlqqmd4rrzLqzOe9k1Dt0Ys9kYM9EYk2mM+Vnvcz8yxmzq/TrfGLPYGDPTGDPLGPO2M4tWajR772QNgnB5Vtwl2/mIcPmEWGqaO9h3pt5F1Q2upb2LNw+Xc8OMRIIDfC/ZdkZKFIF+PuwtqnNRdd5N7xRVyotUN7WTd7qOWWlRdg1JnJocQYCvDy/tK3VBdfZ583AF5zu6uXVuyqBtA/x8mJkaxaGyBlo7ul1QnXfTQFfKi/zpw9N02wxXDnJ2fkGgny9TkyJ442A5bZ2eEYh/3VdKRkwIc/u5mNufeRnRdNkMB0o851OGp9JAV8pLdHbb2JBbwqSEcOKGMDZ7TvoYmtq72HKkwonV2aek7jwfFdZy85wUuy9yJkcFkxwVTO7peo+7FuBpNNCV8hLbjlZS09zO/IzoIX3fuNhQkqOC+evHF482dr1X9vfUcPOc5CF9X07GGCoa2yg71+qMsixDA10pL7F+bwkJEUFkxYcP6ft8RLh5TjLvn6ymoqHNSdXZ542D5cwfF03KmJAhfd/05Eh8BA574BBMT6KBrpQXKK0/z66T1dyekzKsCatunJ2MzcBbh8udUJ19imvPc7yyiWuzP3sj0WBCAvyYMDaMQ2UN2u1yCRroSnmBF3N7plO6fV7qIC37lxkXRmZcKO8crXRkWUNyYd/XZicM6/unJUVSf76Ts+fc+ynDk2mgK+XhurptvJhXypVZcUPuquhreXYCewrr3DZn+jv5FUyKDyctZnjHkJ0Y0dPtcla7XQaiga6Uh9txvJqKxjZWzx/ZBG3Ls+Ppshl2HK9yUGX2O3e+g9zT9SzLHjvs1wgJ9CMzTrtdLsVRsy0qZSnDmZfFWTbkFhMbFsjSKcMPQ4DZqVHEhgXydn4lq2YNbZTJSG0/VkW3zbB8mN0tF0xLiuSVA2XklzcyNSnSQdVZh56hK+XBKhra2H6sittzUvD3Hdmvq4+PsGzKWHYer6a9y7U3GW09WsnY8EBmJI8shLOTerpdNh9y38VdT6aBrpQHezGvBJuBLw7zYujFlmfH09zexe5C182N0t7Vzc7j1SzLjsdnhEvKhQb6MT42jM2HKrTbpR8a6Ep5KJvNsDG3hMUTYkiPccwiFYsnxBLs78s7+a67a3RPYR0tHd0sG2GX0QVTkyMoqmnhmM6T/hka6Ep5qF0nqyk71zrii6F9Bfn7cuXEWLbmV7nsDHf7sSoC/XxYNN4xa95MTYrUbpcB6EVRpTzUcx+dITYsgOXDuBHnUpZnJ7DlSCWHyhqYkRLl0Ne+mDGGd49XcVlmzKBT5dorLNCPjNjQT++ctXdOmNGwjKOeoSvlgYprz7P9eBVr5qcR6OeYILzgmslj8RF4J9/5NxkV1rRwpvY810x2THfLBdOSIqlpbqeyqd2hr+vt7Ap0EVkhIsdFpEBEHhygze0iki8iR0TkBceWqdTo8tzu0/iK8KVLrOgzXNGhAeRkRLsk0N891jPm/WoHB/rUpAgEndvlYoMGuoj4Ao8C1wHZwGoRyb6oTRbwL8BiY8xU4DtOqFWpUeF8Rxcbc0v43LQE4iOCnLKPa7PjOVbRRImTl3bbfqyKifFhI7rDtT/hQf5kxIZ65Hqp7mRPH/p8oMAYUwggIhuAVUB+nzZfBx41xtQDGGNcfyuasrzh3uzjbX2nr+4/S2NbF3ddluG0fSzPjuff3jjK2/mV3HP5OKfso6mtk71FddxzhXNef1pyJH/75CyVjW1O+8PnbezpckkGSvo8Lu19rq+JwEQR+UBEdovIiv5eSETWikieiORVV1cPr2KlLMxmM/zPB0VkJ0aQY+eKPsORHhPKpPhwpw5ffP9kDV02wzWTHNvdcsE07Xb5DEddFPUDsoAlwGrgSRH5zOVzY8w6Y0yOMSYnLs6+JbSU96lqauPpD4p4ZPtJ1u8t5v2CGrptehOIPd48XMHJqma+uSTT7tEbw7U8O57c0/WcO9/hlNd/52glkcH+di81N1ThQf6kx4TqZF192BPoZUDf29RSep/rqxTYZIzpNMYUASfoCXg1ihhjeL+ghke2F1BW30pYkB9l51rZfKic53efcfnt5t7GZjP89/aTZMaFcsP0RKfvb3l2PN02w/Zjju8h7eq2sf1YFUsnj8VvhFMWXMq05AgqG9upatIpdcG+QM8FskRknIgEAHcAmy5q8yo9Z+eISCw9XTCFDqxTeYH3C2rYfKicCWPD+M6yLO66bBz/eO0kVs1K4kRlE+t2FdLU5p6pW73B2/kVHKto4v5rsoa1iMVQTU+OJD4i0CmjXXrO/Du5dqpjx9BfbFpSpHa79DFooBtjuoD7gC3AUeBFY8wREXlIRFb2NtsC1IpIPvAu8E/GmFpnFa08T2VjG+/kVzIlIZw7F6YTHuT/6bYF42L4yqIMaprb2Zhbgk3n4PgMYwy/21bA+NhQvjAzySX77JmsK56dJ6pp63Tsp6e38ysI8PPhiizndq1GBPuTFhPC4bJGp+7HW9j1WcgYs9kYM9EYk2mM+Vnvcz8yxmzq/doYY75njMk2xkw3xmxwZtHKs3TbDC/tKyXAz4cbZyf32/c7KSGcL8xIorCmhV0n9IL4xTZ9cpaj5Y18++oJLjk7v2B5djznO7r56JTjzr+MMbyTX8kVE2IJDXT+zejTkiKpaGyjRm8y0jtF1ci91zvnyKpZyX93Zn6xueljmJ4cydajlXxcXO/CCj1bc3sXP998lBkpkdw427XzlC/KjCEs0I+3HdjtcrS8idL6Vqd3t1wwLbmn2+WT0nMu2Z8n00BXI9LU1smuk9VMSYxg+iBzXYsIN81OJjLYn+9tPODwj/ne6r+3naSysZ2frpzq0rNzgEA/X66aGMfWo5XYHDQS6e38CkRg6RTXBHpksD/j40LZX3Ju1E+pq4GuRuT53cW0ddrsHmsc5O/LjbOTOV17nj/s1OvmBVXN/PH9Im7PSWF2mvPGnV/K8ux4qpvaOeCgM9wtRyrJSR9DbFigQ17PHnPSxlDX0sHpWufe+erpNNDVsLV1dvPH94uYMDaM5DHBdn9f1thwbpieyGM7Cigexb+A3TbDg389SHCAL/+8YrLb6rh60lj8fYXNB0c+He3xiiaOljdyvQuGXfY1NSmSAD+fUd+Vp4Guhu0v+0qpaW7nqolDH8nwg89PwddH+OnfjjihMu+w43gVeWfq+bcbp7n0bPZikSH+XDN5LK8eKKOz2zai13r541L8fISVLhqpc0GAnw/TkyI5XNZAR9fIjsGbaaCrYenqtrFu1ylmpUYxPnboq+kkRgbznWVZbDtW5ZJZ/zzNmdoWth+r4qbZyS5fsLk/t81Npaa5gx3Hhz8Cqavbxiv7y7h68lhi3PAHanZ6FO1dNvLLR++YdA10NSy7TlZTUtfK2ivHD/sW9bsXj2NifBg/2XSE1o7Rc4G0pb2LjXklRIX489Cqqe4uB4CrJsURGxbAS/tKBm88gA9O1VLV1M4tc9zzByojJpQxIf7knRm93S4a6GpYNuwtITYsgGUjGMng7+vDQ6umUXaulcd2FDiwOs/VZbPx5z3FNLd1cce8tEsO83Qlf18fbpqdzLajVdQ2D28898sflxIZ7O/wuc/t5SNCTkY0hdUtVI/SMem6BJ0asqqmNrYfq+Key8cR4Deyc4KF42O4cVYSf9hZyM1zUhg3jO4bb2GMYdOBs5yubeH2nFRSox07R/hI3To3lSffK+LVA2e55/JxQ5quuK2zm82HyvnivFSHr7A0FDnpY9h+tIrdRbV8YYZr+/E9gZ6hqyF7+eMyumyG23JSB29sh3+9YQqBfj7868uHHDYW2hPtOllD3pl6lkyMY1aqc9fyHI5JCeHMSIlkw97iIf8c8s7U09ltuHWuY94TwxUe5M+05Ag+PlM/KieD00BXQ2KM4cXcEuZljGHC2DCHvObY8CD+5fopfFRYywt7h7eIhafbU1TLliMVzEiJZJmDF312pK8tHsfJqmbeOmL/POmd3TbeO1HN+NhQj/hDtXB8DO1dNg6UjL47RzXQ1ZDknq6nsKaFL85z7CpAq+ensnhCDP+++Sil9dYam/5JyTk2HTjLpPhwbpubio+T5zkfiS/MTCIzLpTfbj1h9yRqeafraGrvcvhC0MOVFh1CYmQQewrrRt2doxroakg25BYTFujH9dMTHPq6IsIvbp6BAf7FQl0vHxfX82JeCRmxoaxZkObyW/uHytdHeGDZRE5UNts1JW1Xt42dJ6rJiAnxmOsfIsLC8TFUNLZRWNPi7nJcSgNd2a2xrZPNh8pZOSuJkADHX09PjQ7hX6+fwnsna3jkXe8f9bKnqJaX9pWSGRfGVxdl4O/EhR4c6YbpiUyMD2Pb0apBz9LzztTT2NbF1ZPHOn2FpaGYlRpFWKAfO0fZzJ7e8Q5THmHTgbO0ddq4Y57zLnx9aUEaN81O5jdbT/Duce9ca9xmDFuOVPDagbNMTgjnzkXpIx4N5Eq+PsJ3lk2kurn9kqsZVTa28daRCjJiQpkQ55jrKY7i7+vD4gmxFFQ1U1bf6u5yXMZ73mXK7TbmljA5IXzQWRVHQkT4+U3TmZwQwQPr91PkZR+Z2zq7eX73GXaeqGZ+RjRrFqR5zZl5X9dNS2BO2hi2H6viYD+Tdl04zgBfH744L9Wjzs4vWDAumiB/H3ae8M4Tg+Gw650mIitE5LiIFIjIg5dod4uIGBHJcVyJyhMcOdvAobIG7nDBL29wgC9/+PJc/Hx9WPPkbs7UekeoH6to5HfbTnKisomVM5O4cXYyfj7eF+bQ84f1xllJpMeE8NK+Uk5UNn16gbGhtZMNucXUn+9g9fw0IoM94+aoiwX5+7JwXAxHzjaOmhuNBn23iYgv8ChwHZANrBaR7H7ahQMPAHscXaRyvxdzSz5dkcgV0mJCeP6eBbR2drN63W5K6jx35EtZfSsv7DnDsx+dIcDPh7VXjGfh+Bh3lzVifr4+fGlBOhHB/jzz4Wl+s/Ukf95zhl9vOcbJyma+MDPJYy6EDuSyCbH4+gi7To6OvnR7rmzNBwqMMYUAIrIBWAXkX9TuYeCXwD85tELldm2d3byyv4wVUxOICglw2X6zkyL4870LWPPkHm55/ENunp1MWoz7A6Sjy0ZFYxuna1o4Wt7ImbrzBPj5sHTyWK6aGOfUVe5dLSzQj/uvmcDhsgbyztRTWN3CovExLMqMJTrUde+F4QoL9CMnYwy5RfWUN7SSGGn/NM/eyJ5ATwb6zthTCizo20BE5gCpxpg3RGTAQBeRtcBagLQ0x45jtprz7V10dNuICPZ3+7jltw5X0NjW5dSLoQOZmhTJxm8sZO2z+3jyvSI+PzOR+RnRTuv26eq2cbyyidyiOurOd9DY2klzexctHV20tHdzvqOLzu7/HfkRFx7IDdMTmZs+hiB/993y7kyBfr7MTY9mbnq0u0sZlismxLG3qI6n3ivih5//TOeCpYx47JmI+AD/Bdw1WFtjzDpgHUBOTo41Bho72NlzrXxQUMPB0ga6jcHPR0iIDGJSQjhz092zos3G3BLSokPc1o0wOSGCTfct5vY/fMRrB3oWU141M5kxDjpDPN/RxdajVWw+WM7OE9W09i6N5ytCeJAfYUF+hAf6kxARREiAH6EBvsSEBZIeE+Ixk2upgY0JDWBmShTr9xZz39UTHPa+8UT2BHoZ0PfULKX3uQvCgWnAjt6zpgRgk4isNMbkOarQ0eDDUzW8frCcAF8f5o+LZmxEIHUtHRwqa+C2Jz7kG1dl8p1lWS6d/Oh0TQsfFdbyj9dOxMeNN8VEhQTwlUUZfHSqlnfyK/ntthNcNTGOyzJjh31mfKKyied3n+Hlj8tobu9ibHggt85NISdjDCV1rUSFuP/TkXKMKyfGsb/kHM98eJrvLp/o7nKcxp5AzwWyRGQcPUF+B7DmwkZjTAMQe+GxiOwA/lHDfGh2nqhmy5EKshMjuGVOCsEB/xtS10way/HKJh7fcYqTlU080TsCxBVezCvBR3D7pEvQMz3q4gmxTE2K4PWD5Ww9WsUHBbVclhnD3PQxdvXvt3d1805+Jc99dIY9RXUE+Ppww4xEvjgvlXkZ0Z/eyTmUmQaV54uPCGJ5djzPfHiatVeOJzTQmhPNDnpUxpguEbkP2AL4Ak8bY46IyENAnjFmk7OLtLr3T1Z/OnHTbXNTP3N7eKC/L7+4ZQZTkyL44WtH+P4rh/nFLdOdPnywq9vGS/tKuXrSWBIig5y6r6GICgngywvTKatvZduxSrYdq2L7sSrGx4WSNTac1OgQYsMCPh3/fai0gROVTew4Uc27x6pobu8iNTqYB6+bzG1zU9yyuo5yvW8tyeSd/ErW7y3m3ivGu7scp7Drz5QxZjOw+aLnfjRA2yUjL2v0OFPbwltHKpiaFMHtOZeeuOnORRlUN3fw+20niY8I5HvXTnJqbVuPVlLV1M7tbrgYao/kMcF8ZVEGdS0dfFxcz8HSc5ecJTA6NIAbpidy/YxErpgQ69YuJOV6s9PGsGh8DE++V8idi9LdOm+7s1jzc4eXaO3oZmNeCZHB/twyJ8Wu/trvLsuisqGN328vYGZqFEtHsGLQYJ58r4jU6GCWesgsegOJDu1ZOWnZlHia27soqTtPQ2snnd02jIEbZycxPi6M8bGhlhpSqIbuW1dncucf9/Lyx2Wsnm+9kXb67nYTYwyvHCijsbWTO+al2X1hT0T46aqpTE2K4B/+8glnzzlnnop9Z+rYd6aeexaP86oQDAv0Y0piBAvHx3BFVhxXToxjxbREJsaHe9VxKOe4fEIs05Mj+cPOU3RbZEbPvvQd7ib55Y0cLmtg+ZT4IS9FFuTvyyNr5tDZZeP/rt9PV7fN4fWt21VIZLC/x3a3KDUcIsK3lmRyuvY8mw+Vu7sch9NAd4P2zm5eP1hOYmQQl2fFDes1xsWG8vObp5N3pp7fbD3h0PqKalp4O7+SOxemO2WaXKXc6XNTE8iMC+WxHacstwCGBrobbDtWRWNrJzfOSh7RggerZiVzx7xUHttxil0OnPd53a5T+Pv48JXL0h32mkp5Ch8f4ZtXZXK0vJEdFpsvXQPdxY6cbeDDUzXMGxftkFXff/yFqUwcG853Nx6gqrFtxK93vKKJjbklrFmQxthwzxmqqJQjrZqVTFJkEI+/e8rdpTiUBroLddsM33/lMMEBfnwu2zFLuAUH+PLImtmc7+jmgQ0HRnShxxjDw6/nEx7kzwNLsxxSn1KeKMDPh69fOZ69p+vIPV3n7nIcRgPdhdbvLeZAyTlumJ7wd3eCjlRWfDgPrZrKR4W1PLJ9+Eu3bT9WxfsFNTywNMvS810oBXDHvDSiQwN4zALLHV6gV7xcpLqpnV++dYzLMmOYmRLl8Ne/dW4KH52q5XfbTjB/XDSLMmOGdPt6R5eNR949yfi4UO5cpH3nyvqCA3y567IM/uudE5yobGJifLi7SxoxPUN3kZ+9kU97p42Hb5zmlFv2RYSHb5xGRkwoD2zYP6QFIYwxvHqgjNrmDn66cqpXLpmm1HDcuTCdYH9f1u0qdHcpDqG/uS7wQUENrx44yzeXZJLpxMV0QwP9ePzLc2nvsnHHut3Ut3TY9X0fnqrlQMk5lk4ZyxXDHEaplDcaExrA7TkpvHagjIqGkQ8qcDcNdCdr6+zmB68eJj0mhG8tyXT6/iYlhPPnexfQ1NbJU+8XUtt86bUUD5c18ObhcrITI1gyybNv8VfKGZH90q8AAAu0SURBVO69YjzdNsP/fFjk7lJGTAPdyf6ws5CimhYeXjXNZSvaTEuO5Pl7e9bj/P32k7xfUIPtohsoOrttbPrkLC/sLSYpKpjb5to3l4xSVpMaHcJ10xN5YXcxTW2d7i5nRPSi6Ahd6sJjVVMbj2wvYEZKJKX1rS6dY3tGShQPLJ3IawfK2HyonN2FtYyLDWVseCAVDW2cqm6msa2LxZkxfG5agteuTq+UI3zjyvG8cbCcDXtL+PqV3ju1rga6k3TbDH/JKyXAz4cbpie6pYbIYH/uXJjOobIG9hefI/9sI/s6uwkN9GNcTAg5GdGWuLKv1EjNSIli4fhonv6giLsWZ3jtwAANdCfZfqySsnOtrJmf5tZ1J0WEGSlRzEiJwhhDc3sXYYF+Tl8cQylv840rM7n7mVz+9slZbp6T4u5yhsWuP0MiskJEjotIgYg82M/274lIvogcFJFtIjKqBzIX17aw43g1c9LGMC050t3lfEpECA/y1zBXqh9LJsUxMT6MdbsKvXbSrkEDXUR8gUeB64BsYLWIZF/UbD+QY4yZAbwE/MrRhXqL5vYu1ueWEBXiz+dnuKerRSk1dCLC2iszOVbRxK6TNe4uZ1jsOUOfDxQYYwqNMR3ABmBV3wbGmHeNMRfuZNkNeOfnlRHqthnW7y2mpb2LLy1Id9moFqWUY6ycmURCRBDrdnnnpF32BHoyUNLncWnvcwO5B3izvw0islZE8kQkr7raWtNWAmw+XE5RTQs3zU4mKSrY3eUopYYowM+Huxdn8EFBLYfLGtxdzpA59FKuiHwZyAF+3d92Y8w6Y0yOMSYnLs5adyR+UFDDR6dqWZwZw+y0Me4uRyk1TKsXpBEW6OeV0wHYE+hlQN91yFJ6n/s7IrIM+D6w0hhz6dsTLeZAST1vHCpnalIE17lpiKJSyjEigvxZsyCNNw6VU1pv/5xInsCeQM8FskRknIgEAHcAm/o2EJHZwB/oCfMqx5fpuY5VNPLSvlLGxYZye06q3m2plAXcvTgDAf74vndNBzBooBtjuoD7gC3AUeBFY8wREXlIRFb2Nvs1EAb8RUQOiMimAV7OUrbmV/Ln3cUkRgZz58J0r70ZQSn19xIjg1k5K4kNe0uoGWQ+JE9i141FxpjNwOaLnvtRn6+XObguj/dOfiXf+vM+EiKD+NricTqiRSmL+fbVE3h1fxlP7DjFDz5/8Uhtz6SnlMPw8selfPP5fWQnRvC1xeMcuvqQUsozZMaFcdPsFJ7bfYZKB6zX6woa6EP0x/eL+N6Ln7BgXDR//vpCDXOlLOyBpVl024zXLFOngW4nYwz/seU4D7+ez4qpCTx91zzCAnUqHKWsLC0mhNtyUnlhb7FXjHjRQLdDt83w/VcP88i7Bayen8qjX5qjfeZKjRL3XzMBHxF++dZxd5cyKA30QbR3dXP/+o95YU8x3746k5/fNB1fHx2aqNRokRQVzDeuyuRvn5xld2Gtu8u5JA30S2hu7+Jrz+Sy+VAFP7hhCv/0uck6U6FSo9D/uSqT5KhgfrLpCF3dNneXMyAN9AFUNbWx5snd7C6s4z9vm8m9V3jvKiZKqZEJDvDlh5+fwrGKJp7ffcbd5QxIA70fxyuauOnRDzlZ2cy6O+dyy9xROXmkUqqPz01N4IqsWH615ThFNS3uLqdfGugX2XWimlsf/5DObhsvfmMRS6fEu7skpZQHEBF+desM/H19eGDDfjq6PK/rRQO9jxf2FHP3M7kkjwnm1W8vZnqK56w2pJRyv8TIYH55y3QOljbwm60n3F3OZ+hAaqCz28Yv3jzGH98vYsmkOB5ZM0fHmCul+rViWiKr56fxxM5TTEuK5AYPWpls1KfW2XOt3L9+P/vO1PPVRen88PPZ+OkkW0qpS/jxF7IpqGriOxv3Exnsz+VZse4uCRjFXS7GGF47UMb1v3+PY+WN/H71bH66apqGuVJqUEH+vjz11XlkxoWx9rk89p2pc3dJwCgN9LJzrdz7pzwe2HCA9JhQ/nb/5aycmeTuspRSXiQy2J9nvzafseGBrF63hxfzSgb/JicbVV0udS0dPPZuAc/uPoOPwA9umMLdi8fpnZ9KqWEZGxHEK99azP3r9/PPLx3kk5Jz/POKyUQG+7ulnlER6PlnG3lu92le3X+W9q5ubpmTwneWTyRZF3JWSo3QmNAAnrl7Hr/acpwn3yvkzcMVfHdZFrfPSyXQz7VzPtkV6CKyAvgd4As8ZYz5xUXbA4FngblALfBFY8xpx5Zqv8a2Tg6WNPDeyWq2Hq3kVHULQf4+rJyZxNevGE9WfLi7SlNKWZCfrw//ev0UVs5M4uHX8/nha0f4j7dPsGpWEtdNS2R2WpRLJvQbNNBFxBd4FFgOlAK5IrLJGJPfp9k9QL0xZoKI3AH8EviiMwouO9dK3uk6jOmZBbGlo4uG853UNLdTXHee07XnP72Ly99XWDg+hq9elsGqmclEhrjnY5BSanSYlhzJhrUL+aCglhfzStiYW8KzH53B31fITowgIzaU1DEhXJ4Vy8LxMQ7fvz1n6POBAmNMIYCIbABWAX0DfRXwk96vXwIeERExxhgH1grA/uJ6Hthw4DPPhwf5kRYdwuSEcG6Zk8zM1ChmpUYRHqQhrpRyHRHh8qxYLs+KpbGtk9yiOnJP13Ow9BwfF9fz+sFyfASnBLoMlrkiciuwwhhzb+/jO4EFxpj7+rQ53NumtPfxqd42NRe91lpgbe/DSYDnTzA8uFigZtBW1qDHak2j5Vitcpzpxpi4/ja49KKoMWYdsM6V+3Q2EckzxuS4uw5X0GO1ptFyrKPhOO0Zh14GpPZ5nNL7XL9tRMQPiKTn4qhSSikXsSfQc4EsERknIgHAHcCmi9psAr7a+/WtwHZn9J8rpZQa2KBdLsaYLhG5D9hCz7DFp40xR0TkISDPGLMJ+CPwnIgUAHX0hP5oYakupEHosVrTaDlWyx/noBdFlVJKeYdROZeLUkpZkQa6UkpZhAb6EInIbSJyRERsIjLgECgRWSEix0WkQEQedGWNjiIi0SLyjoic7P3vmAHadYvIgd5/F18w91iD/YxEJFBENvZu3yMiGa6v0jHsONa7RKS6z8/xXnfU6Qgi8rSIVPXeH9PfdhGR3/f+vzgoInNcXaOzaKAP3WHgZmDXQA36TJdwHZANrBaRbNeU51APAtuMMVnAtt7H/Wk1xszq/bfSdeUNn50/o0+ntAB+Q8+UFl5nCO/HjX1+jk+5tEjHegZYcYnt1wFZvf/WAo+7oCaX0EAfImPMUWPMYHe4fjpdgjGmA7gwXYK3WQX8qffrPwE3urEWR7PnZ9T3+F8CloqIN861bJX3o12MMbvoGW03kFXAs6bHbiBKRDxnHbkR0EB3jmSg72z3pb3PeZt4Y0x579cVQPwA7YJEJE9EdouIt4S+PT+jT9sYY7qABsDxE3A4n73vx1t6uyBeEpHUfrZbhVV+Pz9jVMyHPlQishVI6GfT940xr7m6Hme61LH2fWCMMSIy0BjXdGNMmYiMB7aLyCFjzClH16qc6m/AemNMu4h8g55PJte4uSY1RBro/TDGLBvhS9gzXYJHuNSxikiliCQaY8p7P5JWDfAaZb3/LRSRHcBswNMDfShTWpR6+ZQWgx6rMabvcT0F/MoFdbmL1/x+DpV2uTiHPdMleIO+Uzp8FfjMpxMRGdO7wAkiEgss5u+nVvZUo2lKi0GP9aI+5JXAURfW52qbgK/0jnZZCDT06Vr0bsYY/TeEf8BN9PS5tQOVwJbe55OAzX3aXQ+coOdM9fvurnuYxxpDz+iWk8BWILr3+Rx6Vq4CuAw4BHzS+9973F33EI7vMz8j4CFgZe/XQcBfgAJgLzDe3TU78Vj/HTjS+3N8F5js7ppHcKzrgXKgs/d39R7gm8A3e7cLPaN+TvW+Z3PcXbOj/umt/0opZRHa5aKUUhahga6UUhahga6UUhahga6UUhahga6UUhahga6UUhahga6UUhbx/wGEiP8uhdFwqgAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "sc = []\n",
    "for idx in range(ranks_ad.shape[0]):\n",
    "    sc.append(sps.spearmanr(ranks_ad[idx,:], ranks_f[idx,:])[0])\n",
    "sns.distplot(np.array(sc))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 204,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "SpearmanrResult(correlation=0.15438542018672774, pvalue=2.180514237299371e-54)"
      ]
     },
     "execution_count": 204,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sps.spearmanr(ranks_ad[:,3],ranks_f[:,3])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 188,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[1.        , 0.02618485],\n",
       "       [0.02618485, 1.        ]])"
      ]
     },
     "execution_count": 188,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.corrcoef(ranks_ad[:,2],ranks_f[:,2])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "sps.friedmanchisquare()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 145,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "2245"
      ]
     },
     "execution_count": 145,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.sum(ranks[:,0]==4)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 146,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "2323"
      ]
     },
     "execution_count": 146,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.sum(ranks[:,1]==4)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 147,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "2502"
      ]
     },
     "execution_count": 147,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.sum(ranks[:,2]==4)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 148,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "2709"
      ]
     },
     "execution_count": 148,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.sum(ranks[:,3]==4)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 149,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['Gradient',\n",
       " 'Shap',\n",
       " 'InputTimesGradient',\n",
       " 'DeepTaylor',\n",
       " 'EdgeDetection',\n",
       " 'random']"
      ]
     },
     "execution_count": 149,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "results_aopc.method_arr"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 132,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0\n",
      "100\n",
      "200\n",
      "300\n",
      "400\n",
      "500\n",
      "600\n",
      "700\n",
      "800\n",
      "900\n"
     ]
    }
   ],
   "source": [
    "er = results_aopc.expected_rank(samples=1000)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 150,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2415.975\n",
      "2591.0\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYAAAAD4CAYAAADlwTGnAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAAgAElEQVR4nO3deXwc5Z3n8c9P3VLrlqz7tCXb8iEbXxibM2AMwRzBIUC4MmGSzDKTQMIkM5mFzUxmlh12k93JkGRCDgIkQCDgEAKGGMxNbA7fpyzLli/dp3Xf6n72jy4Toch2S2p19fF7v156UV1dVf3tot2/ruepekqMMSillIo8UXYHUEopZQ8tAEopFaG0ACilVITSAqCUUhFKC4BSSkUop90BxiMjI8MUFRXZHUMppULGjh07WowxmWM9F1IFoKioiO3bt9sdQymlQoaInDjdc9oEpJRSEUoLgFJKRSgtAEopFaG0ACilVITSAqCUUhFKC4BSSkUoLQBKKRWhtAAopVSE0gKglFIRKqSuBFZqvJ7ZUuXTcrevnD7FSZQKPnoEoJRSEUoLgFJKRSgtAEopFaG0D0CpUeo7+th2vI3W7gFmZSYyNyeJ7ORYu2Mp5XdaAJTC21l8pLmbV/bW0dg58BfPl+Ymc/XCHNITXdphrMKGFgAV8foG3byyt45d1e2kJcRw7Tm5FKUnkBznpLl7gKPNPWw+3MIP3zrM5fOyuG1FISJid2ylJk0LgIpofYNuHn//GPUdfVw2N5NVc7OIdvy5aywpNpqZGYmsKErjj/vqeeNAI7c/uoVrz8kl6ixFQI8UVLDTAqAi1qkv/4aOfu5YOYP5ucmnXTY5LppbzyskJS6azZUtDA55uGFZ/lmLgFLBTAuAikhuj+HJD4/T0NnPHSunM+8MX/6niAhXL8whxhnF2webmJYQw+XzsqY+rFJTRE8DVRHp9QMNnDjZy03nFvj05X+KiLB6XhZLClN5q7yRw01dU5hSqamlBUBFnIMNnWw63MKKojQWF6SOe30R4bNL8slKdvHctmraewenIKVSU08LgIooXf1DPL+jhtyUWK5dlDvh7cQ4o7hjxQzcHsMLO2sxxvgxpVKBoQVARZTX9jcwMOThlvMKP3G2z0RkJLm4akEOlc3d7Kpu91NCpQJHC4CKGMdbethV3c4lJRlkJfnnyt4VxWlMT4tnw756ugeG/bJNpQJFC4CKCG6PYf2eOlLiorlsrv/O3IkS4Yal+QwMediwr95v21UqELQAqIiw9VgrDZ39XHtOLjFO/37ss5NjuWROBrur26lq7fHrtpWaSloAVNgbHPbwTkUzxRkJLMjz/ZTP8bh0TiZJsU7+uK9eO4RVyNACoMLeR0db6R4Y5sr52VM2ho/L6eDTpdlUt/Wxr7ZjSl5DKX/TAqDC2sCQmz8dbqYkK5GijIQpfa2l06eRmxLLa2UNDLk9U/paSvmDFgAV1j482krvoJsr5mdP+WtFiXDNObm09w6x5WjrlL+eUpPlUwEQkTUiUiEilSJy3xjPu0TkOev5LSJSNOK5+635FSJy1Yj53xSRMhHZLyK/FRG944byq97BYTYdbmFudhKFafEBec1ZmYnMzkzkvUPN9OhpoSrInbUAiIgDeBi4GigFbhOR0lGLfQVoM8bMBh4Cvm+tWwrcCiwA1gA/FRGHiOQD3wCWG2MWAg5rOaX85nfba+gbcnPZ3MyAvu4Vpdn0DLr59QfHA/q6So2XL0cAK4BKY8xRY8wg8CywdtQya4EnrOnngdXi7W1bCzxrjBkwxhwDKq3tgXck0jgRcQLxQN3k3opSf+b2GB5//xiF0+KYHqBf/6dMT4tnXk4Sj/zpKJ39QwF9baXGw5cCkA9Uj3hcY80bcxljzDDQAaSfbl1jTC3wH0AVUA90GGNeH+vFReQuEdkuItubm5t9iKsUvHGgkROtvVxckmnL3buumJ9NR98Qj206FvDXVspXtnQCi8g0vEcHxUAekCAiXxhrWWPMI8aY5caY5ZmZgT2UV6Hr0U1HKZgWR+k4hnr2p7zUOK5emMNjm4/R1qOjharg5EsBqAUKRzwusOaNuYzVpJMCtJ5h3SuAY8aYZmPMEPACcOFE3oBSo+2ubmf7iTa+dFExjij77tj1zSvn0DM4zCObjtqWQakz8aUAbANKRKRYRGLwdtauH7XMeuBOa/om4G3jvRxyPXCrdZZQMVACbMXb9HO+iMRbfQWrgfLJvx2l4KkPT5AQ4+DzywtszTEnO4nrF+fx6/eP09w1YGsWpcZy1gJgtenfA2zE+yW9zhhTJiIPiMj11mKPAekiUgl8C7jPWrcMWAccAF4D7jbGuI0xW/B2Fu8E9lk5HvHrO1MRqaN3iFf21rF2aT5JsdF2x+He1SUMuj387N0jdkdR6i/4dE9gY8wGYMOoed8dMd0P3HyadR8EHhxj/r8C/zqesEqdzQu7ahgY9nD7iul2RwFgZmYin1uaz2+2nOCuT80kJ0Uvd1HBQ68EVmHDGMPTW6pYXJjKwvwUu+N87BurS/B4DA+/U2l3FKU+QQuAChvbjrdR2dTNHUHy6/+UwrR4Pn9eIc9uq6KmrdfuOEp9TAuAChu/3VpFUqyT6xZP/F6/U+WeVbMRRI8CVFDRAqDCQmf/EK/ur2ftkjziY3zq2gqovNQ4bltRyO+211DVqkcBKjhoAVBhYcPeevqHPNy4zN5TP8/ka6tm44gSfvz2YbujKAVoAVBh4vc7a5iZmcCSwlS7o5xWdnIsXzh/Bi/srOFoc7fdcZTSAqBC34nWHrYdb+PGZQW2jPszHl+9bBYup4Mfv6VHAcp+wddYqtQ4/X5nLSLwuWWjxyi01zNbqsacf15RGi/trmNGegLZyd7rAm5fGVxnLqnIoEcAKqR5PIbf76jh4tkZ5KbE2R3HJ5eUZBDtjOLtg012R1ERTguACmnbT7RR294XdL/+zyTB5eSiWensq+2gvqPP7jgqgmkBUCFt/Z5aYqOj+HRpjt1RxuXi2ZnERkfxVrkeBSj7aB+ACjmn2tbdHsMLO2spyUripd2hdUO5uBgHF83O4K3yJmrb9ChA2UOPAFTIOtLcTe+gm8UFwTPuz3hcNCuDuGgHb5Y32h1FRSgtACpk7aluJzY6ijnZSXZHmZDYaAeXlGRQ0djFrqo2u+OoCKQFQIWkIbeHA/WdLMhLwekI3Y/xBbPSSYhx8J9vHLI7iopAofsvR0W0ioYuBoY9LC4I3it/feFyOrh0TiabDrew5Wir3XFUhNECoELSvtoOElxOijMS7I4yaStnppOV5OIHbxzCeydVpQJDC4AKOYPDHg42dLIgL9nWm777S7Qjinsun83WYyd5v1KPAlTgaAFQIedQYxdDbsM5QXTXr8m65bxC8lJi+Y/XK/QoQAWMFgAVcvbXdRAf46AoPfSbf05xOR18Y3UJu6vbeadCLw5TgaEFQIWU/iE3B+u7WJCXEhbNPyPdeG4B09Pi+cHr2hegAkMLgAop7x1qZtDtYWF+st1R/C7aEcXfX1FCWV0nG8sa7I6jIoAOBaFCyoZ99cTHOJiZkWh3FL86NbyFxxgyE11896UyWroHiRp1fwMdNlr5kx4BqJAxMOzmrfIm5ueGx9k/Y4kSYfX8LJq6Biir67Q7jgpzWgBUyPjgSCvdA8MsyAu/5p+RFuankJHo4t2KJu0LUFNKC4AKGa+XNZAQ42BWZng1/4wWJcJlczKp7+inorHL7jgqjGkBUCHB7TG8caCRy+ZlER3CY//4anFhKtPio3nnoB4FqKkT/v+SVFjYVdVGS/cgny7NtjtKQDiihE/NyaS6rY8jzT12x1FhSguACgkbyxqIdgir5mXZHSVgzp0+jeRYJ+/qhWFqimgBUEHPGMPrBxq5cFYGybHRdscJGKcjiktKMjna0sOJVj0KUP6nBUAFvYrGLk609vLpBZHR/DPSeUVpJMQ4dHgINSW0AKigt3F/IyJwZYS0/48U44ziotkZHGrs1nsHK7/TAqCC3usHGlg2fRpZSbF2R7HF+TPTiY2O0qMA5XdaAFRQqz7ZS1ldZ8Sc/TOW2GgHF8xMp7y+k2Mt2heg/EcLgApqbxxoBOCqBTk2J7HX+TPTiYoSHtt81O4oKoxoAVBBbWNZA3OzkygKg1s/TkZSbDRLClN5fkcNbT2DdsdRYUILgApard0DbDt+MiLP/hnLxbMz6B/y8JuPTtgdRYUJn4aDFpE1wI8AB/CoMeZ7o553AU8C5wKtwC3GmOPWc/cDXwHcwDeMMRut+anAo8BCwABfNsZ86If3pELQqeGQR9px4iQeAx4z9vORJjs5lkvnZPLEhye469KZuJwOuyOpEHfWIwARcQAPA1cDpcBtIlI6arGvAG3GmNnAQ8D3rXVLgVuBBcAa4KfW9sBbUF4zxswDFgPlk387KpyU1XWSGhdNXkpknv0zlr+5pJiW7gFe3lNvdxQVBnxpAloBVBpjjhpjBoFngbWjllkLPGFNPw+sFhGx5j9rjBkwxhwDKoEVIpICfAp4DMAYM2iMaZ/821HhYmDYTWVTN/PzkhEJz7H/J+Li2RmUZCXyxAfHdZA4NWm+FIB8oHrE4xpr3pjLGGOGgQ4g/QzrFgPNwK9EZJeIPCoiY/byichdIrJdRLY3Nzf7EFeFg8ON3Qx7DAtyw3vs//ESEb54YRH7ajvYWaW/mdTk2NUJ7ASWAT8zxiwFeoD7xlrQGPOIMWa5MWZ5ZmZmIDMqGx2o7yQ+xsGM9Mg++2csn1uaT1Kskyc+OG53FBXifCkAtUDhiMcF1rwxlxERJ5CCtzP4dOvWADXGmC3W/OfxFgSlcHsMBxs6mZcTvrd+nIwEl5Obzy1kw756mjr77Y6jQpgvBWAbUCIixSISg7dTd/2oZdYDd1rTNwFvG28D5XrgVhFxiUgxUAJsNcY0ANUiMtdaZzVwYJLvRYWJoy3d9A95wv7Wj5PxxQtmMOwxPK1nR6lJOGsBsNr07wE24j1TZ50xpkxEHhCR663FHgPSRaQS+BZWc44xpgxYh/fL/TXgbmOM21rn68DTIrIXWAL8b/+9LRXKDtR1Eu0QZmeF960fJ6MoI4HL5mby7LYqht0eu+OoEOXTdQDGmA3AhlHzvjtiuh+4+TTrPgg8OMb83cDy8YRV4c9jDOX1nczJToqIWz9Oxu0rpnPXUzt462BTxA+VoSZG/4WpoFLb1kdn/zClevbPWV0+L4uc5FhtBlITpgVABZWyug6iBOblaAE4G6cjilvOK2TT4WaqWnvtjqNCkBYAFTSMMeyv62RWZiJxMTrMgS9uXVGIAL/dpkcBavy0AKig0dDZz8meQRbkpdgdJWTkpsRx+bxsfre9msFh7QxW46MFQAWNsrpOBJifm2R3lJBy+8pCWroHeftgo91RVIjRAqCCxoG6Tmakx5MUG213lJDyqZJMspNdrNteY3cUFWK0AKig0NI9QENnvzb/TIDTEcWNywp4t6KJRr0yWI2DT9cBKDXVDtR1AujVv2dxuvsixEU78Bj47ov7uXRuFgC3r5weyGgqBOkRgAoKZXUd5KfGkRofY3eUkJSe6KIoPYHtJ9p0mGjlMy0Aynb1HX1Ut/Xpr/9JWl40jdaeQY7rNQHKR1oAlO027m8A0Pb/SVqYl4LLGcXOqja7o6gQoQVA2e61sgayklxkJrnsjhLSYpxRLMhLYX9tB0M6QJzygRYAZavW7gG2HjupzT9+snR6KgPDHsrrO+2OokKAFgBlqzfLG/EYbf7xl+KMBFLiotmlt4tUPtACoGz12v4GCtPiyE2JtTtKWIgSYUlhKoebumjuGrA7jgpyWgCUbTp6h9hc2cKaBTmI6K0f/WVJYSoeAy/vqbM7igpyeiGYss3GsgaG3IbrFuVRVqdt1v6SnRxLXmosj24+Smy0b6Oq6kVjkUmPAJRtXt5bx/S0eBYVaPu/vy0pnEZdez8t2gykzkALgLJFa/cAHxxp5bpFudr8MwXOyU9BgL212hmsTk8LgLLFq/sbcHu8zT/K/1LiopmRnsCemg4dGkKdlhYAZYtX9tYxKzNBx/6fQosKUmju8o6yqtRYtACogGvs7GfLsZNctyhPm3+m0ML8FKIE9tZ02B1FBSktACrgNuyrxxj4zOJcu6OEtUSXk1mZieytaddmIDUmLQAq4F7eU8e8nCRmZ2nzz1RbVJBKW+8QNW19dkdRQUgLgAqo2vY+dla185nF2vkbCKW5yThE2F+nzUDqL2kBUAH1x73eq1OvW6TNP4EQF+NgVlYCZXWd2gyk/oIWABVQL++pZ1FBCjPSE+yOEjEW5KVwsmeQ+g49G0h9khYAFTDHW3rYV9uhv/4DrDQ3mShBm4HUX9ACoALmFav551q9+CugElxOijISKKvV8ZbUJ+lgcGpKPbOlCgBjDE9+eILpafG8V9Fsc6rIszAvhfV76mjs7Cc7WYfeVl56BKACor6jn6auAZZOT7U7SkQqzUtG0GYg9UlaAFRA7KpqwyHCOXrnL1skx0YzPS2ech12W42gBUBNObfHsLemg7k5ScS7tNXRLvNzk6nr6Kejb8juKCpIaAFQU+5IczddA8MsKdTmHzvNy/FeeX2wQY8ClJcWADXldle3ExsdxdwcHfrBTplJLtISYiiv1wKgvLQAqCk1MOymrK6Dc/JTiHbox81OIsL8nCSONvcwMOy2O44KAj79ixSRNSJSISKVInLfGM+7ROQ56/ktIlI04rn7rfkVInLVqPUcIrJLRF6Z7BtRwelAXSdDbsOSwml2R1HAvNxkhj2GyqZuu6OoIHDWAiAiDuBh4GqgFLhNREpHLfYVoM0YMxt4CPi+tW4pcCuwAFgD/NTa3in3AuWTfRMqeO2ubmdafDQz0uPtjqKAovQEYqOjOFjfZXcUFQR8OQJYAVQaY44aYwaBZ4G1o5ZZCzxhTT8PrBbvnT7WAs8aYwaMMceASmt7iEgBcC3w6OTfhgpGTZ39VDZ1s7gwlSi98UtQcEQJc7KTONjQiUcHh4t4vhSAfKB6xOMaa96YyxhjhoEOIP0s6/4Q+CfAc6YXF5G7RGS7iGxvbtYrSEPJ+j11GNCzf4LM/Jxkegbd1JzstTuKspktvXIich3QZIzZcbZljTGPGGOWG2OWZ2ZmBiCd8pcXdtaSnxpHVpIOPRBM5mQnESVQ3qDNQJHOlwJQCxSOeFxgzRtzGRFxAilA6xnWvQi4XkSO421SulxEfjOB/CpIVTR0caC+U4d+CEJxMQ6K0hP0egDlUwHYBpSISLGIxODt1F0/apn1wJ3W9E3A28Z794n1wK3WWULFQAmw1RhzvzGmwBhTZG3vbWPMF/zwflSQ+MOuWhxRwqICLQDBaF5uMo2dA5zsGbQ7irLRWQuA1aZ/D7AR7xk764wxZSLygIhcby32GJAuIpXAt4D7rHXLgHXAAeA14G5jjJ6AHOY8HsNLu2v5VEkGiTr0Q1Car1cFK3wcDtoYswHYMGred0dM9wM3n2bdB4EHz7Dtd4F3fcmhQsNHx1qp7+jn/mvm090/bHccNYb0RBeZSS7K6zu5cFaG3XGUTfTSTOV3L+6qJdHl5Mr52XZHUWcwPyeJYy099A/pQXmk0gKg/Kp/yM2r+xpYszCHuBjH2VdQtpmfm4zHwKFGPRsoUmkBUH71ZnkjXQPD3LB09KUiKtgUpsUTH+OgQk8HjVhaAJRf/WFnLTnJsZw/M93uKOososR7VXBFYxcej14VHIm0ACi/ae0e4L1DzaxdkocjSod+CAVzspPoHXSzp6bd7ijKBnqOnhq3Uzd6H+3DIy0Mewwup+O0y6jgMicrEQHeqWhm6XQdsTXS6BGA8ptd1e3kpsSSk6JDP4SKeJeTwrR43jnYZHcUZQMtAMovWroGqGnr04HfQtC8nCT21XbQ1NVvdxQVYFoAlF/sqm5HgMU69EPImZPtvSr43QodbTfSaAFQk2aMYXd1G7MyE0mOi7Y7jhqn3JRYspNdvFuhzUCRRguAmrSqk7209Q6xREf+DEkiwqq5WWw61MKQ+4y351BhRguAmrRd1e1EO4QFucl2R1ETtGpeFl0Dw2w/3mZ3FBVAWgDUpAy7Peyr6aA0NxlXtA79EKoump1BtEN4R5uBIooWADUpFY1d9A259RzyEJfocrKyOF1PB40wWgDUpOyubifR5WRWZqLdUdQkXTY3k8NN3VTrvYIjhhYANWF9g24ONnSxuCBFh34IA5fPywLQs4EiiBYANWH7ajtwewxLCrX5JxwUZyQwIz2et7UZKGJoAVATtqu6jcwkF3mpOvRDODh1OugHR1r1JjERQguAmpCTPYOcaO1laWEqItr8Ey5WzctiYNjDB0da7I6iAkALgJqQ3dXe4YMX69g/YeX8mWkkxDh4s1ybgSKBFgA1bt6hH9opzkhgWnyM3XGUH7mcDi6dm8lb5Y16k5gIoAVAjVttex8t3QM68meYumJ+No2dA+yv67A7ippiWgDUuO2qbscZJSzMS7E7ipoCq+ZmESXw5oFGu6OoKaYFQI3LkNvD3up25uUkERejQz+Eo2kJMSwvSuMN7QcIe1oA1Li8V9FMz6AO/RDurpyfTXl9JzVtelVwONMCoMbldzuqSXA5P76JiApPV5RmA/CWHgWENS0Aymet3QO8Vd7E0sJUHfohzBVnJDArM4GNZQ12R1FTSAuA8tlLu+sY9hiWafNPRLjmnFw+OtpKa/eA3VHUFNECoHz2/I4azslPISdFh36IBFcvzMVjYGOZng0UrrQAKJ+U1XVwoL6Tm5cX2B1FBcj83CSKMxJ4dX+93VHUFNECoHzy/I4aYhxRXL84z+4oKkBEhKsX5vDBkVZO9gzaHUdNAS0A6qwGhz28tLuOK0uzSdWhHyLKNefk4vYY3jigncHhSAuAOqu3DzZxsmeQm87V5p9IsyAvmelp8fxxnxaAcOS0O4AKfs/vqCYrycUlJRl2R1FT5JktVad9rig9gc2Hm3nkT0e561MzA5hKTTU9AlBn1Nw1wDsVzdywLB+nQz8ukWhJYSoeA3tr2u2OovxM/0WrM3pxVy1uj+Fmbf6JWDkpseSnxrHzRJvdUZSfaQFQp2WM4Xc7qllSmMrsLB36IZItnZ5KXUc/5fWddkdRfuRTARCRNSJSISKVInLfGM+7ROQ56/ktIlI04rn7rfkVInKVNa9QRN4RkQMiUiYi9/rrDSn/2X6ijUON3dxyXqHdUZTNFhek4hDh9ztq7I6i/OisBUBEHMDDwNVAKXCbiJSOWuwrQJsxZjbwEPB9a91S4FZgAbAG+Km1vWHgH4wxpcD5wN1jbFPZ7OmPTpDkcrJ2iZ77H+kSXE7m5Sbx4u5ahtweu+MoP/HlCGAFUGmMOWqMGQSeBdaOWmYt8IQ1/TywWrx3Cl8LPGuMGTDGHAMqgRXGmHpjzE4AY0wXUA7kT/7tKH9p7R5gw74GPrcsn/gYPVlMwbLp02jpHuSdgzpCaLjwpQDkA9UjHtfwl1/WHy9jjBkGOoB0X9a1mouWAlvGenERuUtEtovI9ubmZh/iKn94fkcNg24Pd5w/w+4oKkjMyU4iO9nFkx+esDuK8hNbO4FFJBH4PfD3xpgxe5eMMY8YY5YbY5ZnZmYGNmCE8ngMz2ytYkVRmo77rz7miBK+eEERmytbONigncHhwJcCUAuM7AUssOaNuYyIOIEUoPVM64pINN4v/6eNMS9MJLyaGu8dbuZEay93nD/d7igqyNyxcjqx0VE8vvmY3VGUH/hSALYBJSJSLCIxeDt1149aZj1wpzV9E/C2McZY82+1zhIqBkqArVb/wGNAuTHmP/3xRpT/PL75GFlJLq5emGt3FBVkUuNjuHFZAS/urqNF7xMQ8s5aAKw2/XuAjXg7a9cZY8pE5AERud5a7DEgXUQqgW8B91nrlgHrgAPAa8Ddxhg3cBHwV8DlIrLb+rvGz+9NTcDBhk42HW7hzguLiHHqZSLqL3354mIGhz385iPtCwh1Pp3eYYzZAGwYNe+7I6b7gZtPs+6DwIOj5m0G9J6CQeixTceIi3Zwx0pt/lFjm5WZyOXzsnjig+N8+eJikmOj7Y6kJkjP71Mf+8V7R3hhVy3LZ0xjg47+qM7gm1fM4TM/2cwv3jvCt6+aZ3ccNUF6jK8+9tHRk3g8hotm6aif6szOKUjh+sV5PLb5GA0d/XbHUROkBUAB0Nk/xIdHW5ifm0xGksvuOCoE/OOn5+L2GH745iG7o6gJ0gKgAHjyg+P0D3lYNS/L7igqRExPj+eOlTNYt72aA3V6XUAo0gKg6B4Y5tHNx5iXk0R+apzdcVQIuXd1CWkJLr61bjcDw26746hx0gKgeOrDE7T3DnG5/vpX4zQtIYbv33gOBxu6+NGbh+2Oo8ZJC0CE6x4Y5pebjnLZ3EwKpsXbHUeFoNXzs7lleSE/f+8I24+ftDuOGgctABHuF+8d4WTPIN+8Yo7dUVQI++fr5pM/LY6vPr2T6pO9dsdRPtICEMHqO/r45aajXL84j8WFqXbHUSEsKTaax+88j4EhN3f+aivtvYN2R1I+0AIQwX7w+iE8Hvj2VXPtjqLCQEl2Er/84nJqTvbxlSe209k/ZHckdRZ6JXCEKqvr4Pc7a7jrkpkUpmnbv/LNM1uqzrrMTecW8Oy2Kj7/8w/51ZfOIzdFzywLVnoEEIE8HsM/v7ifafExfG3VbLvjqDCzMD+Fv76wmJq2Pm54+AP21rTbHUmdhhaACPT01ip2VbXzz9fOJyVOB/JS/jc7K5F1f3sBUQI3/uwDfv7eETweY3csNYo2AYW50YfsnX1DPPTmIWZlJtA36PbpkF6piSjNS2bDvZfwP/6wj++9epC3Dzbx759dqHeZCyJ6BBBBjDG8vLcOt8fw2SX5eO/Lo9TUSY2P4eHbl/H/blrEocYurvnRJv79lQN0aQdxUNAjgAiyq7qdsrpOrlqQQ3qiDvimptboo8t7LpvNxgONPLb5GM9tr+bqhbksLkjhjvNn2JRQ6RFAhGjtHmD9njqK0hO4pESHe1aBF+9ycsPSfP7u0lkkx0azbns1v9x0jPJ6HUjOLloAIoDbY1i3vZoogc8vLyBKm36UjQrT4vnqZbP47JJ8mrr6uR/oIK4AAAxQSURBVPbHm/jXl/bT0avNQoGmTUARYGNZA9VtfdxyXiGp8TF2x1GKKBFWFKexMD+ZqpO9PPXRCV7eW8+3r5rLLcsLiYrSHymBoEcAYW5XVRubK1u4YGY6iwt0uAcVXOJjnDywdiGvfP0SZmcmcv8L+7j5Fx9S2dRtd7SIoAUgjO2v7eAPu2opzkjgmnNy7Y6j1GmV5iXz3N+ezw9uXkxlUzfX/HgTP323kmG3x+5oYU2MCZ2LM5YvX262b99ud4yQUH2yl8/97AMGhz3cvWo2iS5t7VOhoat/iPV76iir6yQvNZYblxV8PJzE7Sun25wu9IjIDmPM8rGe0yOAMNTaPcAXH9/K4LCHv76wSL/8VUhJio3mjpUzuH3FdDr6hnn4nUreq2jCE0I/VkOFfjOEmY6+Ib70623Ud/Tx9N+spKJB21JVaFqYn8LMjARe3FPHxgONHG7u5or52eSkxNodLWzoEUAYae8d5AuPbqG8vpOHb1/GuTPS7I6k1KTEu5zcdl4hn1uaT/XJXq7+0Z94vazB7lhhQwtAmGjpHuC2X26horGLR/5qOavnZ9sdSSm/EBGWF6Vxz6oS8lLjuOupHfzLi/vpH9Kb0E+WFoAwcKixixt++j7HWrp57M7lrNKbu6swlJnk4oWvXch/u6SYpz46wWcffl9PF50kLQAh7t2KJm786Qf0D3l47q4LuKQk0+5ISk0Zl9PBd64t5VdfOo+mrgE+81+beX5Hjd2xQpYWgBA1OOzh/7xazpd+vY2CtHheuvsiva+vihir5mbx6r2XsLgwhX/83R6+9dxuugeG7Y4VcvQsoBB0oK6T+17Yy96aDm5fOZ1/ubaUuBiH3bGUmnKjRxi9blEeCS4nf9hVy3uHmrltxXTyUvWaAV9pAQghHX1DPPTGIZ788Dip8TH8/AvLWLNQr/BVkStKhNXzsinOSGDdtmp+9t4RrlmYw/kz0+2OFhK0AISA9t5BHn//OL96/xjdA8N8YeUM/uHTc3RgN6UsMzMS+frlJTy/o4aX99ZT3tDFZfOyyE/VG9KfiRaAIGWMYVd1O89ureKVvfX0Drq5akE231hdwoK8FLvjKRV0ElxOvnjBDLYeP8mr+xq46qE/8d/XzOX2lTNw6OiiY9ICEET6h9zsrGrjrfImXj/QQPXJPuJjHHxmUR5/fVER83OT7Y6oVFATEVYWpzMnK4kPj7byLy+V8dut1TywdgHLi/TCyNF0MLgpdrqbrhtj6Ogbora9j5q2Pqrbeqlq7WXYY3BECbMzE1mQl8w5+Sm4orWDV6nxum1FIX/cV8+/v1JOQ2c/q+Zm8s0r57AowoZFP9NgcHoEEAAeY2jpHqCho5+Gjn7qO/qpbe/7+LS1KIGc5FhWFqcxKzOR4owE/dJXapJEhOsW5bFqbhZPfHicR/50lOt/8j4ritO4Y+V01izMweWM7H9nWgD8rL13kPL6Lg42dFJe38n7la00dvYz7PEeaUUJZCS6KMlKpGBaHAXT4slJiSXaoZdkKDUVElxOvnbZbL54QRHPbDnBbz6q4t5nd5PkcnLp3EyuLM3mvKK0j08fjSQ+FQARWQP8CHAAjxpjvjfqeRfwJHAu0ArcYow5bj13P/AVwA18wxiz0ZdtBjNjDC3dgxxv7eFYcw9HW3qoaOjkYEMX9R39Hy+XnhDDtPgYzp+ZTk5yLDkpsWQluXDql71SAZfocnLXp2bxNxfPZHNlC3/cW8+b5Y28srce8B6Fl+YlMzMjgRnp8WQkukhPdJGRGEN6oovkWCcSZvfTPmsfgIg4gEPAlUANsA24zRhzYMQyXwMWGWP+TkRuBW4wxtwiIqXAb4EVQB7wJjDHWu2M2xzLZPsAjDF4DAx7PLg9hmGPwe02uI3B7TEMDnvoGRymu3+YrgHvf9t7B2nqGqCxs9/67wA1J3vpGnHVYbRDmJWZyPzcZOblJDEvN5n5uUlkJrr47dbqCedVSk0tjzHUtfdRdbKX6pO9NHUN0Nw18PER+0iCt4gkuJwkxlr/dTlIiPE+/vg518hpB/ExTmKjHbicUbiio3A5rWlnFDHOKBxRQpSc+vNe2yCC34rNZPsAVgCVxpij1saeBdYCI7+s1wL/Zk0/D/xEvOnXAs8aYwaAYyJSaW0PH7bpN4v/5+v0DAyP+T/VF6eabbKTY8lLiWX5jGnMzEygOMP7l58ap7/qlQpBUSIUTIunYFo8zPLO8xhD98AwPdaPwO4B71/fkJui9ATvfOuvZ2CYlq5B77T143Gi3zOjiVUMogSykmJ5/77L/bLdkXwpAPnAyJ+xNcDK0y1jjBkWkQ4g3Zr/0ah1863ps20TABG5C7jLetgtIhU+ZPa7Y5ABtNjx2hOkeadOKGWF0MobSlkhQHkrAbl/wqvPON0TQd8JbIx5BHjE7hwisv10h1HBSPNOnVDKCqGVN5SyQujlHc2XdotaoHDE4wJr3pjLiIgTSMHbGXy6dX3ZplJKqSnkSwHYBpSISLGIxAC3AutHLbMeuNOavgl423h7l9cDt4qIS0SKgRJgq4/bVEopNYXO2gRktenfA2zEe8rm48aYMhF5ANhujFkPPAY8ZXXynsT7hY613Dq8nbvDwN3GGDfAWNv0/9vzK9ubocZJ806dUMoKoZU3lLJC6OX9hJAaCkIppZT/6LmLSikVobQAKKVUhIrYAiAihSLyjogcEJEyEbnXmv+/RGSviOwWkddFJM+af5mIdFjzd4vId0dsa42IVIhIpYjcF8i8I57/BxExIpJhPRYR+bGVaa+ILBux7J0ictj6u3P0a9mU17b9e4bPwr+JSO2ITNeMWOd+K0+FiFwVqKwTySsiRSLSN2L+z0ds61wR2Wfl/bH46/JTH/Jaz31dRA5a8//viPm27N/xZrV7306aMSYi/4BcYJk1nYR3aIpSIHnEMt8Afm5NXwa8MsZ2HMARYCYQA+wBSgOV13pciLdD/QSQYc27BngV7xXs5wNbrPlpwFHrv9Os6WlBkNe2/XuGz8K/Af84xvKlVg4XUGzlc9j9WThD3iJg/2m2tdX6fIj1ebk6gHlX4R0exmU9l2X3/p1AVlv37WT/IvYIwBhTb4zZaU13AeVAvjGmc8RiCcDZesk/HirDGDMInBrWIiB5racfAv5pVNa1wJPG6yMgVURygauAN4wxJ40xbcAbwJogyHs6U75/z5J1LB8PcWKMOYb3Qs0Vgcg6wbxjsj4PycaYj4z3G+tJ4LN+DcsZ834V+J7xDhWDMabJWsW2/TuBrGMK1L6drIgtACOJSBGwFNhiPX5QRKqBO4Dvjlj0AhHZIyKvisgCa95YQ2WM+x/jRPOKyFqg1hizZ9Rip8sVrHkhCPbv6M8CcI/VhPa4iEw7SyZb9+0Z8gIUi8guEXlPRC6x5uVbGe3KOwe4RES2WLnOG5HL9v3rY1YIkn07ERFfAEQkEfg98Penfv0bY75jjCkEngbusRbdCcwwxiwG/gt40e68eK+t+B98skgFlXHktX3/jvFZ+BneIcKWAPXADwKd6UzGkbcemG6MWQp8C3hGRAJ+f9Ex8jrxNkWeD3wbWBcs7eTjyBoU+3aiIroAiEg03v/JTxtjXhhjkaeBGwGMMZ3GmG5regMQLd4OzIANazFG3ll420j3iMhx67V3ikjOGXIFZV679+9YnwVjTKMxxm2M8QC/5M8j2Qbjvj1tXqsppdWa3oG3HX2Ola3Arrx4fxW/YDVTbgU8eAdXs3X/jidrMOzbSQlkh0Mw/eHtmHkS+OGo+SUjpr8OPG9N5/DnC+dWAFXWNpx4O1KL+XPH1IJA5R21zHH+3Kl6LZ/sBN5qzU8DjuHtAJ5mTacFQV7b9u8ZPgu5I6a/ibddGmABn+ykPIq3g9LWz8IZ8mYCDmt6Jt4vojTr8eiOymsCmPfvgAes6Tl4m3fEzv07gay27ttJv1+7A9j2xuFivJ2Qe4Hd1t81eCv/fmv+y3g7hsHbFFRmfeg+Ai4csa1r8J4tcAT4TiDzjlrmOH/+QhXgYSvTPmD5iOW+jLdjrRL4UpDktW3/nuGz8JS17/biHatq5Bfsd6w8FYw4u8POz8Lp8uI9ii2zltsJfGbEtpZbn/cjwE+winCA8sYAv7Fefydwud37d7xZ7d63k/3ToSCUUipCRXQfgFJKRTItAEopFaG0ACilVITSAqCUUhFKC4BSSkUoLQBKKRWhtAAopVSE+v/ziCsJ+Rg7PgAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "sns.distplot(er[:,0])\n",
    "print(np.percentile(er[:,0],2.5))\n",
    "print(np.percentile(er[:,0],97.5))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 155,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "117240.03067484664"
      ]
     },
     "execution_count": 155,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "12*ranks.shape[1] / (ranks.shape[0]*(ranks.shape[0]+1)) * \\\n",
    "    np.sum(np.power( np.mean(ranks,axis=1) - ((ranks.shape[0]+1)/2),2))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "_,_,temp = results_aopc.cols_to_plot(results_aopc.baseline_dict_raw['richard_1px_deletion_game_CIFAR-10-original_mean_random_2468'],return_colvals=True)\n",
    "plt.plot(np.mean(temp,axis=0))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "_,_,temp = results_aopc.cols_to_plot(results_aopc.exmethod_dict_raw['richard_1px_deletion_game_CIFAR-10-original_mean_Gradient'],return_colvals=True)\n",
    "plt.plot(np.mean(temp,axis=0))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "score_rel_corr = np.zeros((10000,100))\n",
    "omitted = []\n",
    "for idx in range(10000):\n",
    "    if idx % 100 == 0:\n",
    "        print(idx)\n",
    "    for jdx in range(100):\n",
    "        if(np.std(method_scores[\"EdgeDetection\"][:,idx,jdx]) != 0 and \\\n",
    "           np.std(method_relevance[\"EdgeDetection\"][:,idx,jdx]) != 0):\n",
    "            score_rel_corr[idx,jdx] = np.corrcoef(method_scores[\"EdgeDetection\"][:,idx,jdx],\\\n",
    "                                                  method_relevance[\"EdgeDetection\"][:,idx,jdx])[1,0]\n",
    "        else:\n",
    "            omitted.append((idx,jdx))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "to_delete = list(set([o[0] for o in omitted]))\n",
    "score_rel_corr_d = np.delete(score_rel_corr, to_delete, 0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plt.plot(np.mean(score_rel_corr_d,axis=0))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "sns.distplot(score_rel_corr_d[:,0])\n",
    "sns.distplot(score_rel_corr_d[:,50])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "fig, ax = plt.subplots()\n",
    "method_lq = np.percentile(score_rel_corr_d[:,0],25)\n",
    "method_uq = np.percentile(score_rel_corr_d[:,0],75)\n",
    "method_median = np.median(score_rel_corr_d[:,0])\n",
    "method_mean = np.mean(score_rel_corr_d[:,0])\n",
    "plt.sca(ax)\n",
    "sns.distplot(score_rel_corr_d[:,0], kde=True, rug=False)\n",
    "ax.axvline(method_mean, color=\"#116611\", linestyle='-')\n",
    "ax.axvline(method_median, color=\"#116611\", linestyle='--')\n",
    "ax.axvline(method_lq, color=\"#116611\", linestyle=':')\n",
    "ax.axvline(method_uq, color=\"#116611\", linestyle=':')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "np.sum(score_rel_corr_d[:,0]>method_uq)/score_rel_corr_d.shape[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plt.plot(np.mean(score_rel_corr_d,axis=0))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plt.plot(np.mean(method_relevance[\"Shap\"][:,0,:],axis=0))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#plt.plot(np.mean(method_relevance[\"Gradient\"][:,0,:],axis=0))\n",
    "#plt.plot(np.mean(method_relevance[\"InputTimesGradient\"][:,0,:],axis=0))\n",
    "plt.plot(np.mean(method_relevance[\"Shap\"][:,0,:],axis=0))\n",
    "plt.plot(np.mean(method_relevance[\"DeepTaylor\"][:,0,:],axis=0))\n",
    "plt.plot(np.mean(method_relevance[\"EdgeDetection\"][:,0,:],axis=0))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plt.plot(np.mean(method_relevance[\"Gradient\"][:,0,:],axis=0))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plt.plot(np.diff(method_relevance[\"EdgeDetection\"][70,0,:]))\n",
    "plt.plot(np.diff(method_relevance[\"DeepTaylor\"][70,0,:]))\n",
    "plt.plot(np.diff(method_relevance[\"Shap\"][70,0,:]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "np.corrcoef(np.diff(method_relevance[\"Gradient\"][70,0,:]),np.diff(method_relevance[\"Shap\"][70,0,:]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "_,_,v = results_aopc.cols_to_plot(results_aopc.exmethod_dict_raw[\"richard_deletion_game_CIFAR-10-original_mean_Gradient\"],return_colvals=True)\n",
    "to_plot = results_aopc.exmethod_dict_raw[\"richard_deletion_game_CIFAR-10-original_random_Gradient\"][\"original_prediction_i\"]==8\n",
    "v = v[to_plot,:]\n",
    "for idx in range(v.shape[0]):\n",
    "    plt.plot(v[idx,:],alpha=0.02,color='black')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "to_plot"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "results_aopc.exmethod_dict_raw[\"richard_deletion_game_CIFAR-10-original_mean_Gradient\"][[\"original_prediction_i\",\"1.0\"]]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "ex_dict = dict()\n",
    "ex_dict[\"rate\"] = \"1px\"\n",
    "ex_dict[\"game\"] = \"deletion_game\"\n",
    "ex_dict[\"dataset\"] = \"CIFAR-10-original\"\n",
    "ex_dict[\"perturb\"] = \"mean\"\n",
    "ex_dict[\"method\"] = \"Gradient\"\n",
    "results_aopc.plot_all_methods_experiment_against_baseline_by_deletion(ex_dict,ylim=[-0.3,0.3],\\\n",
    "                                                                      size_inches=(14,7),\\\n",
    "                                                                      include_edgedetect=True)\n",
    "plt.gca().legend()\n",
    "#plt.savefig(\"presentation/AOPC_1px_deletion_game_all_methods.pdf\", dpi=300)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "ex_dict = dict()\n",
    "ex_dict[\"rate\"] = \"1px\"\n",
    "ex_dict[\"game\"] = \"preservation_game\"\n",
    "ex_dict[\"dataset\"] = \"CIFAR-10-original\"\n",
    "ex_dict[\"perturb\"] = \"mean\"\n",
    "ex_dict[\"method\"] = \"Gradient\"\n",
    "results_aopc.plot_all_methods_experiment_against_baseline_by_deletion(ex_dict,ylim=[-0.2,0.2],\\\n",
    "                                                                      size_inches=(14,7),\\\n",
    "                                                                      include_edgedetect=True)\n",
    "plt.gca().legend()\n",
    "plt.savefig(\"presentation/AOPC_1px_preservation_game_all_methods.pdf\", dpi=300)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "ex_dict = dict()\n",
    "ex_dict[\"rate\"] = \"\"\n",
    "ex_dict[\"game\"] = \"deletion_game\"\n",
    "ex_dict[\"dataset\"] = \"CIFAR-10-original\"\n",
    "ex_dict[\"perturb\"] = \"mean\"\n",
    "ex_dict[\"method\"] = \"Gradient\"\n",
    "results_aopc.plot_all_methods_experiment_against_baseline_by_deletion(ex_dict,ylim=[-0.5,0.5],\\\n",
    "                                                                      size_inches=(14,7),\\\n",
    "                                                                      include_edgedetect=True)\n",
    "plt.gca().legend()\n",
    "plt.savefig(\"presentation/AOPC_deletion_game_all_methods.pdf\", dpi=300)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "ex_dict = dict()\n",
    "ex_dict[\"rate\"] = \"\"\n",
    "ex_dict[\"game\"] = \"preservation_game\"\n",
    "ex_dict[\"dataset\"] = \"CIFAR-10-original\"\n",
    "ex_dict[\"perturb\"] = \"mean\"\n",
    "ex_dict[\"method\"] = \"Gradient\"\n",
    "results_aopc.plot_all_methods_experiment_against_baseline_by_deletion(ex_dict,ylim=[-0.7,0.1],\\\n",
    "                                                                      size_inches=(14,7),\\\n",
    "                                                                      include_edgedetect=True)\n",
    "plt.gca().legend()\n",
    "plt.savefig(\"presentation/AOPC_preservation_game_all_methods.pdf\", dpi=300)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "ex_dict = dict()\n",
    "ex_dict[\"rate\"] = \"1px\"\n",
    "ex_dict[\"game\"] = \"deletion_game\"\n",
    "ex_dict[\"dataset\"] = \"CIFAR-10-original\"\n",
    "ex_dict[\"perturb\"] = \"random\"\n",
    "ex_dict[\"method\"] = \"Gradient\"\n",
    "results_aopc.plot_experiment_against_baseline_by_method(ex_dict,ylim=[-0.8,0.8],size_inches=(15,12),n_comparison_lines=2500)\n",
    "plt.savefig(\"presentation/AOPC_1px_deletion_game_all_methods_mean_show_individual.pdf\", dpi=300)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "ex_dict = dict()\n",
    "ex_dict[\"rate\"] = \"1px\"\n",
    "ex_dict[\"game\"] = \"deletion_game\"\n",
    "ex_dict[\"dataset\"] = \"CIFAR-10-original\"\n",
    "ex_dict[\"perturb\"] = \"random\"\n",
    "ex_dict[\"method\"] = \"DeepTaylor\"\n",
    "results_aopc.plot_experiment_against_baseline_by_class(ex_dict,ylim=[-0.8,0.8],size_inches=(15,15),n_comparison_lines=500)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "ex_dict = dict()\n",
    "ex_dict[\"rate\"] = \"1px\"\n",
    "ex_dict[\"game\"] = \"deletion_game\"\n",
    "ex_dict[\"dataset\"] = \"CIFAR-10-original\"\n",
    "ex_dict[\"perturb\"] = \"mean\"\n",
    "ex_dict[\"method\"] = \"DeepTaylor\"\n",
    "results_aopc.plot_all_methods_experiment_against_baseline_by_class(ex_dict,ylim=[-0.3,0.3],size_inches=(14,20),\\\n",
    "                                                                   n_comparison_lines=0,legendloc=\"lower left\",\\\n",
    "                                                                  include_edgedetect=True)\n",
    "plt.savefig(\"presentation/AOPC_1px_deletion_game_all_methods_mean_by_class.pdf\", dpi=300)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "ex_dict = dict()\n",
    "ex_dict[\"rate\"] = \"1px\"\n",
    "ex_dict[\"game\"] = \"deletion_game\"\n",
    "ex_dict[\"dataset\"] = \"CIFAR-10-original\"\n",
    "ex_dict[\"perturb\"] = \"random\"\n",
    "ex_dict[\"method\"] = \"DeepTaylor\"\n",
    "results_aopc.plot_all_methods_experiment_against_baseline_by_class(ex_dict,ylim=[-0.3,0.3],size_inches=(14,20),\\\n",
    "                                                                   n_comparison_lines=0,legendloc=\"lower left\",\\\n",
    "                                                                  include_edgedetect=True)\n",
    "plt.savefig(\"presentation/AOPC_1px_deletion_game_all_methods_random_by_class.pdf\", dpi=300)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "ex_dict = dict()\n",
    "ex_dict[\"rate\"] = \"1px\"\n",
    "ex_dict[\"game\"] = \"preservation_game\"\n",
    "ex_dict[\"dataset\"] = \"CIFAR-10-original\"\n",
    "ex_dict[\"perturb\"] = \"mean\"\n",
    "ex_dict[\"method\"] = \"DeepTaylor\"\n",
    "results_aopc.plot_all_methods_experiment_against_baseline_by_class(ex_dict,ylim=[-0.4,0.3],size_inches=(14,20),\\\n",
    "                                                                   n_comparison_lines=0,legendloc=\"upper right\",\\\n",
    "                                                                  include_edgedetect=True)\n",
    "plt.savefig(\"presentation/AOPC_1px_preservation_game_all_methods_mean_by_class.pdf\", dpi=300)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "ex_dict = dict()\n",
    "ex_dict[\"rate\"] = \"1px\"\n",
    "ex_dict[\"game\"] = \"preservation_game\"\n",
    "ex_dict[\"dataset\"] = \"CIFAR-10-original\"\n",
    "ex_dict[\"perturb\"] = \"random\"\n",
    "ex_dict[\"method\"] = \"DeepTaylor\"\n",
    "results_aopc.plot_all_methods_experiment_against_baseline_by_class(ex_dict,ylim=[-0.6,0.3],size_inches=(14,20),\\\n",
    "                                                                   n_comparison_lines=0,legendloc=\"upper right\",\\\n",
    "                                                                  include_edgedetect=True)\n",
    "plt.savefig(\"presentation/AOPC_1px_preservation_game_all_methods_random_by_class.pdf\", dpi=300)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "results_aopc.plot_aopc_distribution(ex_dict=ex_dict,ind=-1,xlim=(-0.9,0.9),ylim=(0,1250),size_inches=(12,10))\n",
    "plt.savefig(\"presentation/AOPC_distributions_100px_deleted.pdf\", dpi=300)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "results_aopc.plot_aopc_distribution(ex_dict=ex_dict,ind=-1,xlim=(-0.9,0.9),ylim=(0,1250),size_inches=(12,10))\n",
    "plt.savefig(\"presentation/AOPC_distributions_100px_deleted.pdf\", dpi=300)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "results_aopc.baseline_dict_raw['richard_1px_deletion_game_CIFAR-10-original_random_random_97486'].head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "m, mr = results_aopc.correlation_metric_ranking(ex_dict)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "mr[\"r\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "m[\"r\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "np.sum(mr[\"r\"]==1.,axis=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "np.sum(mr[\"r\"]==2.,axis=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "np.sum(mr[\"r\"]==3.,axis=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "np.sum(mr[\"r\"]==4.,axis=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "np.sum(mr[\"r\"]==5.,axis=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "m[\"m\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "sr = []\n",
    "for idx in range(10000):\n",
    "    sr.append(sps.spearmanr(mr[\"r\"][idx,:], mr[\"m\"][idx,:])[0])\n",
    "sns.distplot(np.array(sr))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "corr_scores = dict()\n",
    "ex_dict[\"method\"] = \"Gradient\"\n",
    "corr_scores[\"Gradient\"] = results_corr.score_relevance_correlation(ex_dict=ex_dict)\n",
    "ex_dict[\"method\"] = \"InputTimesGradient\"\n",
    "corr_scores[\"InputTimesGradient\"] = results_corr.score_relevance_correlation(ex_dict=ex_dict)\n",
    "ex_dict[\"method\"] = \"Shap\"\n",
    "corr_scores[\"Shap\"] = results_corr.score_relevance_correlation(ex_dict=ex_dict)\n",
    "ex_dict[\"method\"] = \"DeepTaylor\"\n",
    "corr_scores[\"DeepTaylor\"] = results_corr.score_relevance_correlation(ex_dict=ex_dict)\n",
    "ex_dict[\"method\"] = \"EdgeDetection\"\n",
    "corr_scores[\"EdgeDetection\"] = results_corr.score_relevance_correlation(ex_dict=ex_dict)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "for m in corr_scores.keys():\n",
    "    sns.distplot(corr_scores[m][~np.isnan(corr_scores[m])], label=results_corr.get_method_name(m))\n",
    "plt.gca().legend()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Model(): \n",
    "    def predict(self, x):\n",
    "        return [0]\n",
    "    \n",
    "    def predict_proba(self, x):\n",
    "        return [np.array([np.sum(x),0,0])]\n",
    "    \n",
    "def faithfulness_metric(model, x, coefs, base):\n",
    "    \"\"\" This metric evaluates the correlation between the importance assigned by the interpretability algorithm\n",
    "    to attributes and the effect of each of the attributes on the performance of the predictive model.\n",
    "    The higher the importance, the higher should be the effect, and vice versa, The metric evaluates this by\n",
    "    incrementally removing each of the attributes deemed important by the interpretability metric, and\n",
    "    evaluating the effect on the performance, and then calculating the correlation between the weights (importance)\n",
    "    of the attributes and corresponding model performance. [#]_\n",
    "    References:\n",
    "        .. [#] `David Alvarez Melis and Tommi Jaakkola. Towards robust interpretability with self-explaining\n",
    "           neural networks. In S. Bengio, H. Wallach, H. Larochelle, K. Grauman, N. Cesa-Bianchi, and R. Garnett, editors,\n",
    "           Advances in Neural Information Processing Systems 31, pages 7775-7784. 2018.\n",
    "           <https://papers.nips.cc/paper/8003-towards-robust-interpretability-with-self-explaining-neural-networks.pdf>`_\n",
    "    Args:\n",
    "        model: Trained classifier, such as a ScikitClassifier that implements\n",
    "            a predict() and a predict_proba() methods.\n",
    "        x (numpy.ndarray): row of data.\n",
    "        coefs (numpy.ndarray): coefficients (weights) corresponding to attribute importance.\n",
    "        base ((numpy.ndarray): base (default) values of attributes\n",
    "    Returns:\n",
    "        float: correlation between attribute importance weights and corresponding effect on classifier.\n",
    "    \"\"\"\n",
    "\n",
    "    #find predicted class\n",
    "    pred_class = model.predict(x.reshape(1,-1))[0]\n",
    "\n",
    "    #find indexs of coefficients in decreasing order of value\n",
    "    ar = np.argsort(-coefs)  #argsort returns indexes of values sorted in increasing order; so do it for negated array\n",
    "    pred_probs = np.zeros(x.shape[0])\n",
    "    for ind in np.nditer(ar):\n",
    "        x_copy = x.copy()\n",
    "        x_copy[ind] = base[ind]\n",
    "        print(x_copy)\n",
    "        x_copy_pr = model.predict_proba(x_copy.reshape(1,-1))\n",
    "        pred_probs[ind] = x_copy_pr[0][pred_class]\n",
    "    print(coefs)\n",
    "    print(pred_probs)\n",
    "    return -np.corrcoef(coefs, pred_probs)[0,1]\n",
    "\n",
    "m = Model()\n",
    "x = np.array([4,3,6,5,2,8,7])\n",
    "coefs = np.array([3,3,4,5,2,12,7])/3\n",
    "base = np.zeros(7)\n",
    "\n",
    "faithfulness_metric(m,x,coefs,base)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "np.random.rand(3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def GenerateSameRandomPixelWeights(images_shape, rseed):\n",
    "    pixel_weight_size = list(images_shape[1:])\n",
    "    pixel_weight_size[-1] = 1\n",
    "    \n",
    "    dataset_pixel_weight_lists = []\n",
    "    random_generator = np.random.RandomState(rseed)\n",
    "    pix_weights = random_generator.uniform(size=pixel_weight_size)\n",
    "\n",
    "    for image_i in range(images_shape[0]):\n",
    "        if(image_i % 100 == 0):\n",
    "            print(\"Generating Random Pixel List for:\" + str(image_i))\n",
    "        pixel_weight_list = CreateOrderedPixelsList(pix_weights,return_abs=False)\n",
    "        dataset_pixel_weight_lists.append(pixel_weight_list)\n",
    "        \n",
    "    return dataset_pixel_weight_lists\n",
    "\n",
    "def CreateOrderedPixelsList(attribution_map, return_abs=False):\n",
    "    pixel_weight_dict = OrderedDict()\n",
    "    for i in range(attribution_map.shape[0]):\n",
    "        for j in range(attribution_map.shape[1]):\n",
    "            #TODO: decide best way to aggregate colour attribution\n",
    "            if(len(attribution_map[i][j].shape) > 0):\n",
    "                attribution_value = sum(attribution_map[i][j])\n",
    "            else:\n",
    "                attribution_value = attribution_map[i][j]\n",
    "            if return_abs:\n",
    "                attribution_value = np.abs(attribution_value)\n",
    "            pixel_weight_dict[str(i)+\",\"+str(j)] = attribution_value\n",
    "    #TODO: confirm not taking the abs is correct\n",
    "    return OrderedDict(sorted(pixel_weight_dict.items(), key=lambda item: -item[1]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "temp = GenerateSameRandomPixelWeights([5,32,32,3],1234*33)\n",
    "print([item for item in temp[0].items()])"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}