Adversarial_Attacks_robstnessCheck.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "1e7c3f40",
   "metadata": {},
   "source": [
    "# Auxiliary_Rotation Attack (with Translation)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "7d939387",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-12-15T02:57:38.230968Z",
     "start_time": "2022-12-15T02:55:50.687006Z"
    },
    "scrolled": false
   },
   "source": [
    "# import necessary dependencies\n",
    "import argparse\n",
    "import os, sys\n",
    "import time\n",
    "import datetime\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.nn.functional as F\n",
    "import torchvision\n",
    "import torchvision.transforms as transforms\n",
    "import torchvision.transforms as trn\n",
    "import torchvision.transforms.functional as trnF\n",
    "import torch.nn as nn\n",
    "import torch.optim as optim\n",
    "from tqdm import tqdm_notebook as tqdm\n",
    "from torchvision.transforms import InterpolationMode\n",
    "from torch.utils.data import DataLoader\n",
    "from torchvision import datasets\n",
    "from torchsummary import summary\n",
    "from torch.autograd import Variable\n",
    "import PIL\n",
    "from PIL import Image\n",
    "from torch.utils.data import Subset\n",
    "from torch.utils.tensorboard import SummaryWriter\n",
    "import random\n",
    "import pandas as pd\n",
    "import torch.backends.cudnn as cudnn\n",
    "import opencv_functional as cv2f\n",
    "\n",
    "!pip install \"opencv-python-headless<4.3\"\n",
    "#conda install pillow=6.1\n",
    "print('Architecture List:')\n",
    "print(torch.cuda.get_arch_list())\n",
    "random.seed(10)\n",
    "print(random.random())\n",
    "import warnings\n",
    "warnings.filterwarnings(\"ignore\", category=DeprecationWarning)\n",
    "warnings.simplefilter('ignore')\n",
    "\n",
    "from torchvision.utils import save_image\n",
    "from tqdm import tqdm\n",
    "import sklearn.metrics as sk\n",
    "import itertools\n",
    "import pdb\n",
    "import cv2\n",
    "\n",
    "\n",
    "# Residual block\n",
    "class ResBlock(nn.Module):\n",
    "    def __init__(self, in_channels, out_channels, stride=1, downsample=None):\n",
    "        super(ResBlock, self).__init__()\n",
    "        self.conv1 = nn.Conv2d(in_channels, out_channels, kernel_size=3, stride=stride, padding=1, bias=False)\n",
    "        self.bn1 = nn.BatchNorm2d(out_channels)\n",
    "        self.conv2 = nn.Conv2d(out_channels, out_channels, kernel_size=3, stride=1, padding=1, bias=False)\n",
    "        self.bn2 = nn.BatchNorm2d(out_channels)\n",
    "        self.downsample = downsample\n",
    "\n",
    "    def forward(self,x):\n",
    "        identity = x\n",
    "        out = F.relu(self.bn1(self.conv1(x.to(device))))\n",
    "        out = self.bn2(self.conv2(out))\n",
    "        if self.downsample:\n",
    "            identity = self.downsample(x)\n",
    "        out += identity\n",
    "        out = F.relu(out)\n",
    "        return out\n",
    "    \n",
    "    \n",
    "# ResNet\n",
    "class ResNet(nn.Module):\n",
    "    def __init__(self, block, numBlocks, num_classes=10):\n",
    "        super(ResNet, self).__init__()\n",
    "        self.in_channels = 16\n",
    "        self.conv1   = nn.Conv2d(3, 16, kernel_size=3, stride=1, padding=1, bias=False)\n",
    "        self.bn1     = nn.BatchNorm2d(16)\n",
    "        self.layer1  = self.make_layer(block, 16, numBlocks[0], stride=1)\n",
    "        self.layer2  = self.make_layer(block, 32, numBlocks[1], stride=2)\n",
    "        self.layer3  = self.make_layer(block, 64, numBlocks[2], stride=2)\n",
    "        self.avgpool = nn.AvgPool2d(8)\n",
    "        self.fc1     = nn.Linear(64, num_classes)\n",
    "\n",
    "    def make_layer(self, block, out_channels, numBlocks, stride):\n",
    "        downsample = None\n",
    "        if (stride != 1) or (self.in_channels != out_channels):\n",
    "            downsample = nn.Sequential(\n",
    "                nn.Conv2d(self.in_channels, out_channels, kernel_size=1, stride=stride, bias=False),\n",
    "                nn.BatchNorm2d(out_channels))\n",
    "        layers = []\n",
    "        layers.append(block(self.in_channels, out_channels, stride, downsample))\n",
    "        self.in_channels = out_channels\n",
    "        for i in range(1, numBlocks):\n",
    "            layers.append(block(out_channels, out_channels))\n",
    "        return nn.Sequential(*layers)\n",
    "\n",
    "    def forward(self,x):\n",
    "        out = F.relu(self.bn1(self.conv1(x)))\n",
    "        out = self.layer1(out)\n",
    "        out = self.layer2(out)\n",
    "        out = self.layer3(out)\n",
    "        out = self.avgpool(out)\n",
    "        out = out.view(out.size(0),-1)\n",
    "        #out = self.fc1(out)\n",
    "        return self.fc1(out),out\n",
    "\n",
    "\n",
    "#------------------ Data loaders: Train Validation, test and Corrpted data loader#######################################\n",
    "normalize     = transforms.Normalize((0.4914, 0.4822, 0.4465), (0.2023, 0.1994, 0.2010))\n",
    "randomly_crop = transforms.RandomCrop(32,padding=4)\n",
    "\n",
    "\n",
    "class PerturbDataset(torch.utils.data.Dataset):\n",
    "\n",
    "    def __init__(self, dataset, train_mode=True):\n",
    "        self.dataset = dataset\n",
    "        self.num_points = len(self.dataset.data)\n",
    "        self.train_mode = train_mode\n",
    "\n",
    "    def __getitem__(self, index):\n",
    "        x_orig, classifier_target = self.dataset[index]\n",
    "\n",
    "        if self.train_mode == True and np.random.uniform() < 0.5:\n",
    "            x_orig = np.copy(x_orig)[:, ::-1]\n",
    "        else:\n",
    "            x_orig =  np.copy(x_orig)\n",
    "\n",
    "        if self.train_mode == True:\n",
    "            x_orig = Image.fromarray(x_orig)\n",
    "            x_orig = randomly_crop(x_orig)\n",
    "            x_orig = np.asarray(x_orig)\n",
    "\n",
    "        x_tf_0 = np.copy(x_orig)\n",
    "        x_tf_90 = np.rot90(x_orig.copy(), k=1).copy()\n",
    "        x_tf_180 = np.rot90(x_orig.copy(), k=2).copy()\n",
    "        x_tf_270 = np.rot90(x_orig.copy(), k=3).copy()\n",
    "\n",
    "        #---------------Translation\n",
    "        possible_translations = list(itertools.product([0, 8, -8], [0, 8, -8]))\n",
    "        num_possible_translations = len(possible_translations)\n",
    "        tx, ty = possible_translations[random.randint(0, num_possible_translations - 1)]\n",
    "        tx_target = {0: 0, 8: 1, -8: 2}[tx]\n",
    "        ty_target = {0: 0, 8: 1, -8: 2}[ty]\n",
    "        x_tf_trans = cv2f.affine(np.asarray(x_orig).copy(), 0, (tx, ty), 1, 0, interpolation=cv2.INTER_CUBIC, mode=cv2.BORDER_REFLECT_101)\n",
    "\n",
    "        return \\\n",
    "            normalize(trnF.to_tensor(x_tf_0)), \\\n",
    "            normalize(trnF.to_tensor(x_tf_90)), \\\n",
    "            normalize(trnF.to_tensor(x_tf_180)), \\\n",
    "            normalize(trnF.to_tensor(x_tf_270)), \\\n",
    "            normalize(trnF.to_tensor(x_tf_trans)), \\\n",
    "            torch.tensor(tx_target), \\\n",
    "            torch.tensor(ty_target), \\\n",
    "            torch.tensor(classifier_target)\n",
    "\n",
    "    def __len__(self):\n",
    "        return self.num_points\n",
    "    \n",
    "    \n",
    "def test_CIFAR10(net,criterion,root_data_dir,batch_size):\n",
    "    testset = torchvision.datasets.CIFAR10(root=root_data_dir, train=False, download=True)\n",
    "    testset   = PerturbDataset(testset, train_mode=False)\n",
    "    testloader = torch.utils.data.DataLoader(testset, batch_size=batch_size, shuffle=False, num_workers=4)\n",
    "    classes = ('plane', 'car', 'bird', 'cat', 'deer', 'dog', 'frog', 'horse', 'ship', 'truck')\n",
    "\n",
    "    criterion = criterion\n",
    "\n",
    "    net.eval()\n",
    "    test_loss = 0\n",
    "    correct = 0\n",
    "    total = 0\n",
    "    with torch.no_grad():\n",
    "        for batch_idx, (x_tf_0, x_tf_90, x_tf_180, x_tf_270, x_tf_trans, target_trans_x, target_trans_y, target_class) in enumerate(testloader):\n",
    "            batch_size  = x_tf_0.shape[0]\n",
    "            batch       = np.concatenate((x_tf_0,x_tf_90,x_tf_180,x_tf_270,x_tf_trans), 0)\n",
    "            batch       = torch.FloatTensor(batch).cuda()\n",
    "            target_rots = torch.cat((torch.zeros(batch_size),torch.ones(batch_size),2 * torch.ones(batch_size),3 * torch.ones(batch_size)), 0).long()\n",
    "\n",
    "            # copy inputs to device\n",
    "            batch          = batch.to(batch)\n",
    "            target_class   = target_class.to(device)\n",
    "            target_rots    = target_rots.to(device)\n",
    "            target_trans_x = target_trans_x.to(device)\n",
    "            target_trans_y = target_trans_y.to(device)\n",
    "\n",
    "            # compute the output and loss\n",
    "            logits, pen           = net(batch)\n",
    "            classification_logits = logits[:batch_size]\n",
    "            rot_logits            = net.rot_head(pen[:4*batch_size])\n",
    "            x_trans_logits        = net.x_trans_head(pen[4*batch_size:]) \n",
    "            y_trans_logits        = net.y_trans_head(pen[4*batch_size:])\n",
    "\n",
    "            classification_loss   = criterion(classification_logits, target_class)\n",
    "            rot_loss              = criterion(rot_logits, target_rots)*ROTATION_LOSS_WEIGHT\n",
    "            x_trans_loss          = criterion(x_trans_logits, target_trans_x.cuda()) * TRANS_LOSS_WEIGHT\n",
    "            y_trans_loss          = criterion(y_trans_logits, target_trans_y.cuda()) * TRANS_LOSS_WEIGHT\n",
    "            loss = classification_loss + ((rot_loss + x_trans_loss + y_trans_loss) / 3.0)            \n",
    "\n",
    "            \n",
    "            test_loss += loss.item()\n",
    "            _, predicted = classification_logits.max(1)\n",
    "            total += target_class.size(0)\n",
    "            correct += predicted.eq(target_class).sum().item()\n",
    "    num_val_steps = len(testloader)\n",
    "    val_acc = correct / total\n",
    "    print(\"Test Loss=%.4f, Test accuracy=%.4f\" % (test_loss / (num_val_steps), val_acc))\n",
    "    return test_loss / (num_val_steps), val_acc\n",
    "\n",
    "\n",
    "####################Hyoerperametter###################################\n",
    "# a few arguments, do NOT change these\n",
    "DATA_ROOT           = \"./data\"\n",
    "CORRUPTED_DATA_ROOT = \"/data/usr/ft42/nobackup/CIFAR_10_C/CIFAR-10-C/\"\n",
    "# the folder where the trained model is saved\n",
    "CHECKPOINT_FOLDER   = \"savedFiles/saved_models/\"\n",
    "# the folder where the figures are saved\n",
    "FIGURES_FOLDER      = \"savedFiles/saved_figures/\"\n",
    "# the folder where the csvs are saved\n",
    "CSVS_FOLDER         = \"savedFiles/saved_csvs/\"\n",
    "# name of the saved model\n",
    "MODELNAME_REG       = \"resnet20_RotationLoss\"\n",
    "\n",
    "\n",
    "# GPU check\n",
    "device = 'cuda' if torch.cuda.is_available() else 'cpu'\n",
    "if device =='cuda':\n",
    "    print(\"Run on GPU...\")\n",
    "else:\n",
    "    print(\"Run on CPU...\")\n",
    "\n",
    "\n",
    "# hyperparameters, do NOT change right now\n",
    "TRAIN_BATCH_SIZE = 256    # training batch size\n",
    "VAL_BATCH_SIZE   = 100    # validation batch size\n",
    "INITIAL_LR       = 0.1    # initial learning rate\n",
    "MOMENTUM         = 0.9    # momentum for optimizer\n",
    "REG              = 1e-3   # L2 regularization strength\n",
    "EPOCHS           = 200    # total number of training epochs\n",
    "DECAY_EPOCHS     = 20     # parameter for LR schedule (decay after XX epochs)\n",
    "DECAY            = 0.5    # parameter for LR schedule (decay multiplier)\n",
    "NUM_OF_ROTATION = [4]\n",
    "ROTATION_LOSS_WEIGHT = 3.0\n",
    "TRANS_LOSS_WEIGHT    = 0.0\n",
    "MODELNAME = MODELNAME_REG+'_{}_rotweight{}_tranlweiht{}'.format(NUM_OF_ROTATION[0],ROTATION_LOSS_WEIGHT,TRANS_LOSS_WEIGHT)\n",
    "\n",
    "\n",
    "net          = ResNet(ResBlock, [3, 3, 3])\n",
    "net.x_trans_head = nn.Linear(64, 3)\n",
    "net.y_trans_head = nn.Linear(64, 3)\n",
    "net.rot_head     = nn.Linear(64, 4)\n",
    "net = net.to(device)\n",
    "net.load_state_dict(torch.load(CHECKPOINT_FOLDER+MODELNAME+'.pt',map_location='cuda:0'))\n",
    "criterion = nn.CrossEntropyLoss().to(device)\n",
    "test_loss,test_acc=test_CIFAR10(net,criterion,root_data_dir=DATA_ROOT,batch_size=VAL_BATCH_SIZE)\n",
    "print(test_loss,test_acc)\n",
    "\n",
    "\n",
    "# Compute the gradient of the loss w.r.t. the input data\n",
    "def gradient_wrt_data(net,device,data,target_class,target_rots,target_trans_x,target_trans_y,criterion):\n",
    "    \n",
    "    batch_size  = 100\n",
    "    dat = data.clone().detach()\n",
    "    dat.requires_grad = True\n",
    "    logits, pen           = net(dat)\n",
    "    classification_logits = logits[:batch_size]\n",
    "    rot_logits            = net.rot_head(pen[:4*batch_size])\n",
    "    x_trans_logits        = net.x_trans_head(pen[4*batch_size:]) \n",
    "    y_trans_logits        = net.y_trans_head(pen[4*batch_size:])\n",
    "    \n",
    "    classification_loss   = criterion(classification_logits, target_class)\n",
    "    rot_loss              = criterion(rot_logits, target_rots)*ROTATION_LOSS_WEIGHT\n",
    "    x_trans_loss          = criterion(x_trans_logits, target_trans_x.cuda()) * TRANS_LOSS_WEIGHT\n",
    "    y_trans_loss          = criterion(y_trans_logits, target_trans_y.cuda()) * TRANS_LOSS_WEIGHT\n",
    "    loss = classification_loss + ((rot_loss + x_trans_loss + y_trans_loss) / 3.0)\n",
    "    \n",
    "    \n",
    "    net.zero_grad()\n",
    "    loss.backward()\n",
    "    #print()\n",
    "    data_grad = dat.grad.data\n",
    "    return data_grad.data.detach()\n",
    "\n",
    "\n",
    "def FGSM_attack(model, device, dat, target_class,target_rots,target_trans_x,target_trans_y,eps):\n",
    "    # TODO: Implement the FGSM attack\n",
    "    # - Dat and lbl are tensors\n",
    "    # - eps is a float\n",
    "    # HINT: FGSM is a special case of PGD\n",
    "    criterion = nn.CrossEntropyLoss().to(device)\n",
    "    x_nat = dat.clone().detach()\n",
    "    # Compute gradient w.r.t. data (we give you this function, but understand it)\n",
    "    x_grad  = gradient_wrt_data(model,device,data=x_nat,target_class=target_class,target_rots=target_rots,target_trans_x=target_trans_x,target_trans_y=target_trans_y,criterion=criterion)\n",
    "    # Perturb the image using the gradient\n",
    "    x_pertured = x_nat.detach() + eps*x_grad.sign()\n",
    "    #print(x_pertured.shape)\n",
    "    # Clip the perturbed datapoints to ensure we are in bounds [0,1]\n",
    "    #print(x_pertured.shape)\n",
    "    #x_pertured = torch.clamp(x_pertured, min=0, max=1).detach()\n",
    "    # Return the final perturbed samples\n",
    "    return x_pertured\n",
    "\n",
    "testset = torchvision.datasets.CIFAR10(root=DATA_ROOT, train=False, download=True)\n",
    "testset   = PerturbDataset(testset, train_mode=False)\n",
    "testloader = torch.utils.data.DataLoader(testset, batch_size=100, shuffle=False, num_workers=4)\n",
    "\n",
    "\n",
    "\n",
    "## Load pretrained models\n",
    "whitebox = net.to(device)\n",
    "whitebox.eval()\n",
    "EPS_Test = [0.0,0.01,0.02,0.03,0.04,0.05,0.06,0.07,0.08,0.09,0.10]\n",
    "print(EPS_Test)\n",
    "\n",
    "\n",
    "Auxiliary_Rotation_FGSM_acc_list = []\n",
    "\n",
    "\n",
    "for roi_EPS in EPS_Test:\n",
    "\n",
    "\n",
    "    # TODO: Set attack parameters here\n",
    "    ATK_EPS          = roi_EPS\n",
    "    whitebox_correct = 0.\n",
    "    running_total    = 0.\n",
    "    for batch_idx, (x_tf_0, x_tf_90, x_tf_180, x_tf_270, x_tf_trans, target_trans_x, target_trans_y, target_class) in enumerate(testloader):\n",
    "        batch_size  = 100 #x_tf_0.shape[0]\n",
    "        batch       = np.concatenate((x_tf_0,x_tf_90,x_tf_180,x_tf_270,x_tf_trans), 0)\n",
    "        batch       = torch.FloatTensor(batch).cuda()\n",
    "        target_rots = torch.cat((torch.zeros(batch_size),torch.ones(batch_size),2 * torch.ones(batch_size),3 * torch.ones(batch_size)), 0).long()\n",
    "\n",
    "        # copy inputs to device\n",
    "        batch          = batch.to(batch)\n",
    "        target_class   = target_class.to(device)\n",
    "        target_rots    = target_rots.to(device)\n",
    "        target_trans_x = target_trans_x.to(device)\n",
    "        target_trans_y = target_trans_y.to(device)\n",
    "        # TODO: Perform adversarial attack here\n",
    "        #-Within this function model has no influence\n",
    "        whitebox_adv_data = FGSM_attack(model=whitebox, device=device, dat=batch, \n",
    "        target_class=target_class,\n",
    "        target_rots=target_rots,\n",
    "        target_trans_x=target_trans_x,\n",
    "        target_trans_y=target_trans_y,\n",
    "        eps=ATK_EPS)\n",
    "        \n",
    "        '''\n",
    "        print((torch.max(torch.abs(whitebox_adv_data[1,:,:,:]-batch[1,:,:,:]))))\n",
    "\n",
    "        # Sanity checking if adversarial example is \"legal\"\n",
    "        assert(torch.max(torch.abs(whitebox_adv_data[100,:,:,:]-batch[100,:,:,:])) <= (ATK_EPS + 1e-5) )\n",
    "        assert(torch.max(torch.abs(whitebox_adv_data-batch)) <= (ATK_EPS + 1e-5) )\n",
    "        assert(whitebox_adv_data.max() == 1.)\n",
    "        assert(whitebox_adv_data.min() == 0.)\n",
    "        '''\n",
    "        \n",
    "        # Compute accuracy on perturbed data\n",
    "        with torch.no_grad():\n",
    "            # Stat keeping - whitebox\n",
    "            logits, pen                 = whitebox(whitebox_adv_data)\n",
    "            classification_logits       = logits[:batch_size]\n",
    "            _, predicted = classification_logits.max(1)\n",
    "            whitebox_correct += predicted.eq(target_class).sum().item()\n",
    "            running_total += target_class.size(0)\n",
    "\n",
    "    # Print final \n",
    "    whitebox_acc = whitebox_correct/running_total\n",
    "    Auxiliary_Rotation_FGSM_acc_list.append(whitebox_acc)\n",
    "    print(\"Attack Epsilon: {}; Whitebox Accuracy: {}\".format(ATK_EPS, whitebox_acc))\n",
    "\n",
    "print(\"Done!\")\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "8b5732b9",
   "metadata": {},
   "source": [
    "# Auxiliary_Rotation Attack (without Translation)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "81678ac0",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-12-15T09:10:06.705654Z",
     "start_time": "2022-12-15T09:09:07.284006Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Looking in indexes: https://pypi.org/simple, https://pypi.ngc.nvidia.com\n",
      "Requirement already satisfied: opencv-python-headless<4.3 in /opt/conda/lib/python3.8/site-packages (4.2.0.34)\n",
      "Requirement already satisfied: numpy>=1.17.3 in /opt/conda/lib/python3.8/site-packages (from opencv-python-headless<4.3) (1.21.4)\n",
      "\u001b[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001b[0m\n",
      "Architecture List:\n",
      "['sm_52', 'sm_60', 'sm_61', 'sm_70', 'sm_75', 'sm_80', 'sm_86', 'compute_86']\n",
      "0.5714025946899135\n",
      "Run on GPU...\n",
      "Files already downloaded and verified\n",
      "Test Loss=0.3618, Test accuracy=0.9231\n",
      "0.3617953511327505 0.9231\n",
      "Files already downloaded and verified\n",
      "[0.0, 0.01, 0.02, 0.03, 0.04, 0.05, 0.06, 0.07, 0.08, 0.09, 0.1]\n",
      "Attack Epsilon: 0.0; Whitebox Accuracy: 0.9231\n",
      "Attack Epsilon: 0.01; Whitebox Accuracy: 0.7105\n",
      "Attack Epsilon: 0.02; Whitebox Accuracy: 0.6019\n",
      "Attack Epsilon: 0.03; Whitebox Accuracy: 0.5198\n",
      "Attack Epsilon: 0.04; Whitebox Accuracy: 0.459\n",
      "Attack Epsilon: 0.05; Whitebox Accuracy: 0.41\n",
      "Attack Epsilon: 0.06; Whitebox Accuracy: 0.3739\n",
      "Attack Epsilon: 0.07; Whitebox Accuracy: 0.3441\n",
      "Attack Epsilon: 0.08; Whitebox Accuracy: 0.3187\n",
      "Attack Epsilon: 0.09; Whitebox Accuracy: 0.2985\n",
      "Attack Epsilon: 0.1; Whitebox Accuracy: 0.2792\n",
      "Done!\n"
     ]
    }
   ],
   "source": [
    "# import necessary dependencies\n",
    "import argparse\n",
    "import os, sys\n",
    "import time\n",
    "import datetime\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.nn.functional as F\n",
    "import torchvision\n",
    "import torchvision.transforms as transforms\n",
    "import torchvision.transforms as trn\n",
    "import torchvision.transforms.functional as trnF\n",
    "import torch.nn as nn\n",
    "import torch.optim as optim\n",
    "from tqdm import tqdm_notebook as tqdm\n",
    "from torchvision.transforms import InterpolationMode\n",
    "from torch.utils.data import DataLoader\n",
    "from torchvision import datasets\n",
    "from torchsummary import summary\n",
    "from torch.autograd import Variable\n",
    "import PIL\n",
    "from PIL import Image\n",
    "from torch.utils.data import Subset\n",
    "from torch.utils.tensorboard import SummaryWriter\n",
    "import random\n",
    "import pandas as pd\n",
    "import torch.backends.cudnn as cudnn\n",
    "import opencv_functional as cv2f\n",
    "\n",
    "!pip install \"opencv-python-headless<4.3\"\n",
    "#conda install pillow=6.1\n",
    "print('Architecture List:')\n",
    "print(torch.cuda.get_arch_list())\n",
    "random.seed(10)\n",
    "print(random.random())\n",
    "import warnings\n",
    "warnings.filterwarnings(\"ignore\", category=DeprecationWarning)\n",
    "warnings.simplefilter('ignore')\n",
    "\n",
    "from torchvision.utils import save_image\n",
    "from tqdm import tqdm\n",
    "import sklearn.metrics as sk\n",
    "import itertools\n",
    "import pdb\n",
    "import cv2\n",
    "\n",
    "\n",
    "# Residual block\n",
    "class ResBlock(nn.Module):\n",
    "    def __init__(self, in_channels, out_channels, stride=1, downsample=None):\n",
    "        super(ResBlock, self).__init__()\n",
    "        self.conv1 = nn.Conv2d(in_channels, out_channels, kernel_size=3, stride=stride, padding=1, bias=False)\n",
    "        self.bn1 = nn.BatchNorm2d(out_channels)\n",
    "        self.conv2 = nn.Conv2d(out_channels, out_channels, kernel_size=3, stride=1, padding=1, bias=False)\n",
    "        self.bn2 = nn.BatchNorm2d(out_channels)\n",
    "        self.downsample = downsample\n",
    "\n",
    "    def forward(self,x):\n",
    "        identity = x\n",
    "        out = F.relu(self.bn1(self.conv1(x.to(device))))\n",
    "        out = self.bn2(self.conv2(out))\n",
    "        if self.downsample:\n",
    "            identity = self.downsample(x)\n",
    "        out += identity\n",
    "        out = F.relu(out)\n",
    "        return out\n",
    "    \n",
    "    \n",
    "# ResNet\n",
    "class ResNet(nn.Module):\n",
    "    def __init__(self, block, numBlocks, num_classes=10):\n",
    "        super(ResNet, self).__init__()\n",
    "        self.in_channels = 16\n",
    "        self.conv1   = nn.Conv2d(3, 16, kernel_size=3, stride=1, padding=1, bias=False)\n",
    "        self.bn1     = nn.BatchNorm2d(16)\n",
    "        self.layer1  = self.make_layer(block, 16, numBlocks[0], stride=1)\n",
    "        self.layer2  = self.make_layer(block, 32, numBlocks[1], stride=2)\n",
    "        self.layer3  = self.make_layer(block, 64, numBlocks[2], stride=2)\n",
    "        self.avgpool = nn.AvgPool2d(8)\n",
    "        self.fc1     = nn.Linear(64, num_classes)\n",
    "\n",
    "    def make_layer(self, block, out_channels, numBlocks, stride):\n",
    "        downsample = None\n",
    "        if (stride != 1) or (self.in_channels != out_channels):\n",
    "            downsample = nn.Sequential(\n",
    "                nn.Conv2d(self.in_channels, out_channels, kernel_size=1, stride=stride, bias=False),\n",
    "                nn.BatchNorm2d(out_channels))\n",
    "        layers = []\n",
    "        layers.append(block(self.in_channels, out_channels, stride, downsample))\n",
    "        self.in_channels = out_channels\n",
    "        for i in range(1, numBlocks):\n",
    "            layers.append(block(out_channels, out_channels))\n",
    "        return nn.Sequential(*layers)\n",
    "\n",
    "    def forward(self,x):\n",
    "        out = F.relu(self.bn1(self.conv1(x)))\n",
    "        out = self.layer1(out)\n",
    "        out = self.layer2(out)\n",
    "        out = self.layer3(out)\n",
    "        out = self.avgpool(out)\n",
    "        out = out.view(out.size(0),-1)\n",
    "        #out = self.fc1(out)\n",
    "        return self.fc1(out),out\n",
    "\n",
    "\n",
    "#------------------ Data loaders: Train Validation, test and Corrpted data loader#######################################\n",
    "normalize     = transforms.Normalize((0.4914, 0.4822, 0.4465), (0.2023, 0.1994, 0.2010))\n",
    "randomly_crop = transforms.RandomCrop(32,padding=4)\n",
    "\n",
    "\n",
    "class PerturbDataset(torch.utils.data.Dataset):\n",
    "\n",
    "    def __init__(self, dataset, train_mode=True):\n",
    "        self.dataset = dataset\n",
    "        self.num_points = len(self.dataset.data)\n",
    "        self.train_mode = train_mode\n",
    "\n",
    "    def __getitem__(self, index):\n",
    "        x_orig, classifier_target = self.dataset[index]\n",
    "\n",
    "        if self.train_mode == True and np.random.uniform() < 0.5:\n",
    "            x_orig = np.copy(x_orig)[:, ::-1]\n",
    "        else:\n",
    "            x_orig =  np.copy(x_orig)\n",
    "\n",
    "        if self.train_mode == True:\n",
    "            x_orig = Image.fromarray(x_orig)\n",
    "            x_orig = randomly_crop(x_orig)\n",
    "            x_orig = np.asarray(x_orig)\n",
    "\n",
    "        x_tf_0 = np.copy(x_orig)\n",
    "        x_tf_90 = np.rot90(x_orig.copy(), k=1).copy()\n",
    "        x_tf_180 = np.rot90(x_orig.copy(), k=2).copy()\n",
    "        x_tf_270 = np.rot90(x_orig.copy(), k=3).copy()\n",
    "\n",
    "        #---------------Translation\n",
    "        possible_translations = list(itertools.product([0, 8, -8], [0, 8, -8]))\n",
    "        num_possible_translations = len(possible_translations)\n",
    "        tx, ty = possible_translations[random.randint(0, num_possible_translations - 1)]\n",
    "        tx_target = {0: 0, 8: 1, -8: 2}[tx]\n",
    "        ty_target = {0: 0, 8: 1, -8: 2}[ty]\n",
    "        x_tf_trans = cv2f.affine(np.asarray(x_orig).copy(), 0, (tx, ty), 1, 0, interpolation=cv2.INTER_CUBIC, mode=cv2.BORDER_REFLECT_101)\n",
    "\n",
    "        return \\\n",
    "            normalize(trnF.to_tensor(x_tf_0)), \\\n",
    "            normalize(trnF.to_tensor(x_tf_90)), \\\n",
    "            normalize(trnF.to_tensor(x_tf_180)), \\\n",
    "            normalize(trnF.to_tensor(x_tf_270)), \\\n",
    "            torch.tensor(classifier_target)\n",
    "\n",
    "    def __len__(self):\n",
    "        return self.num_points\n",
    "    \n",
    "    \n",
    "def test_CIFAR10(net,criterion,root_data_dir,batch_size):\n",
    "    testset = torchvision.datasets.CIFAR10(root=root_data_dir, train=False, download=True)\n",
    "    testset   = PerturbDataset(testset, train_mode=False)\n",
    "    testloader = torch.utils.data.DataLoader(testset, batch_size=batch_size, shuffle=False, num_workers=4)\n",
    "    classes = ('plane', 'car', 'bird', 'cat', 'deer', 'dog', 'frog', 'horse', 'ship', 'truck')\n",
    "\n",
    "    criterion = criterion\n",
    "\n",
    "    net.eval()\n",
    "    test_loss = 0\n",
    "    correct = 0\n",
    "    total = 0\n",
    "    with torch.no_grad():\n",
    "        for batch_idx, (x_tf_0, x_tf_90, x_tf_180, x_tf_270, target_class) in enumerate(testloader):\n",
    "            batch_size  = x_tf_0.shape[0]\n",
    "            batch       = np.concatenate((x_tf_0,x_tf_90,x_tf_180,x_tf_270), 0)\n",
    "            batch       = torch.FloatTensor(batch).cuda()\n",
    "            target_rots = torch.cat((torch.zeros(batch_size),torch.ones(batch_size),2 * torch.ones(batch_size),3 * torch.ones(batch_size)), 0).long()\n",
    "\n",
    "            # copy inputs to device\n",
    "            batch          = batch.to(batch)\n",
    "            target_class   = target_class.to(device)\n",
    "            target_rots    = target_rots.to(device)\n",
    "\n",
    "            # compute the output and loss\n",
    "            logits, pen           = net(batch)\n",
    "            classification_logits = logits[:batch_size]\n",
    "            rot_logits            = net.rot_head(pen[:4*batch_size])\n",
    "\n",
    "            classification_loss   = criterion(classification_logits, target_class)\n",
    "            rot_loss              = criterion(rot_logits, target_rots)*ROTATION_LOSS_WEIGHT\n",
    "            loss = classification_loss + (0.25*rot_loss)            \n",
    "\n",
    "            \n",
    "            test_loss += loss.item()\n",
    "            _, predicted = classification_logits.max(1)\n",
    "            total += target_class.size(0)\n",
    "            correct += predicted.eq(target_class).sum().item()\n",
    "    num_val_steps = len(testloader)\n",
    "    val_acc = correct / total\n",
    "    print(\"Test Loss=%.4f, Test accuracy=%.4f\" % (test_loss / (num_val_steps), val_acc))\n",
    "    return test_loss / (num_val_steps), val_acc\n",
    "\n",
    "\n",
    "####################Hyoerperametter###################################\n",
    "# a few arguments, do NOT change these\n",
    "DATA_ROOT           = \"./data\"\n",
    "CORRUPTED_DATA_ROOT = \"/data/usr/ft42/nobackup/CIFAR_10_C/CIFAR-10-C/\"\n",
    "# the folder where the trained model is saved\n",
    "CHECKPOINT_FOLDER   = \"savedFiles/saved_models/\"\n",
    "# the folder where the figures are saved\n",
    "FIGURES_FOLDER      = \"savedFiles/saved_figures/\"\n",
    "# the folder where the csvs are saved\n",
    "CSVS_FOLDER         = \"savedFiles/saved_csvs/\"\n",
    "# name of the saved model\n",
    "MODELNAME_REG       = \"resnet20_ssrp_w075_d4\"\n",
    "\n",
    "\n",
    "# GPU check\n",
    "device = 'cuda' if torch.cuda.is_available() else 'cpu'\n",
    "if device =='cuda':\n",
    "    print(\"Run on GPU...\")\n",
    "else:\n",
    "    print(\"Run on CPU...\")\n",
    "\n",
    "\n",
    "# hyperparameters, do NOT change right now\n",
    "TRAIN_BATCH_SIZE = 256    # training batch size\n",
    "VAL_BATCH_SIZE   = 100    # validation batch size\n",
    "INITIAL_LR       = 0.1    # initial learning rate\n",
    "MOMENTUM         = 0.9    # momentum for optimizer\n",
    "REG              = 1e-3   # L2 regularization strength\n",
    "EPOCHS           = 200    # total number of training epochs\n",
    "DECAY_EPOCHS     = 20     # parameter for LR schedule (decay after XX epochs)\n",
    "DECAY            = 0.5    # parameter for LR schedule (decay multiplier)\n",
    "NUM_OF_ROTATION = [4]\n",
    "ROTATION_LOSS_WEIGHT = 0.75\n",
    "TRANS_LOSS_WEIGHT    = 0.0\n",
    "MODELNAME = MODELNAME_REG+'_{}_rotweight{}'.format(NUM_OF_ROTATION[0],ROTATION_LOSS_WEIGHT)\n",
    "\n",
    "\n",
    "net          = ResNet(ResBlock, [3, 3, 3])\n",
    "net.rot_head     = nn.Linear(64, 4)\n",
    "net = net.to(device)\n",
    "net.load_state_dict(torch.load('/data/usr/ft42/nobackup/Project/savedFiles/saved_models/resnet20_ssrp_w075_d4_4_rotweight0.75.pt',map_location='cuda:0'))\n",
    "criterion = nn.CrossEntropyLoss().to(device)\n",
    "test_loss,test_acc=test_CIFAR10(net,criterion,root_data_dir=DATA_ROOT,batch_size=VAL_BATCH_SIZE)\n",
    "print(test_loss,test_acc)\n",
    "\n",
    "\n",
    "# Compute the gradient of the loss w.r.t. the input data\n",
    "def gradient_wrt_data(net,device,data,target_class,target_rots,criterion):\n",
    "    \n",
    "    batch_size  = 100\n",
    "    dat = data.clone().detach()\n",
    "    dat.requires_grad = True\n",
    "    logits, pen           = net(dat)\n",
    "    classification_logits = logits[:batch_size]\n",
    "    rot_logits            = net.rot_head(pen[:4*batch_size])\n",
    "\n",
    "    \n",
    "    classification_loss   = criterion(classification_logits, target_class)\n",
    "    rot_loss              = criterion(rot_logits, target_rots)*ROTATION_LOSS_WEIGHT\n",
    "    loss = classification_loss + (0.25*rot_loss)\n",
    "    \n",
    "    \n",
    "    net.zero_grad()\n",
    "    loss.backward()\n",
    "    #print()\n",
    "    data_grad = dat.grad.data\n",
    "    return data_grad.data.detach()\n",
    "\n",
    "\n",
    "def FGSM_attack(model, device, dat, target_class,target_rots,eps):\n",
    "    # TODO: Implement the FGSM attack\n",
    "    # - Dat and lbl are tensors\n",
    "    # - eps is a float\n",
    "    # HINT: FGSM is a special case of PGD\n",
    "    criterion = nn.CrossEntropyLoss().to(device)\n",
    "    x_nat = dat.clone().detach()\n",
    "    # Compute gradient w.r.t. data (we give you this function, but understand it)\n",
    "    x_grad  = gradient_wrt_data(model,device,data=x_nat,target_class=target_class,target_rots=target_rots,criterion=criterion)\n",
    "    # Perturb the image using the gradient\n",
    "    x_pertured = x_nat.detach() + eps*x_grad.sign()\n",
    "    #print(x_pertured.shape)\n",
    "    # Clip the perturbed datapoints to ensure we are in bounds [0,1]\n",
    "    #print(x_pertured.shape)\n",
    "    #x_pertured = torch.clamp(x_pertured, min=0, max=1).detach()\n",
    "    # Return the final perturbed samples\n",
    "    return x_pertured\n",
    "\n",
    "testset = torchvision.datasets.CIFAR10(root=DATA_ROOT, train=False, download=True)\n",
    "testset   = PerturbDataset(testset, train_mode=False)\n",
    "testloader = torch.utils.data.DataLoader(testset, batch_size=100, shuffle=False, num_workers=4)\n",
    "\n",
    "\n",
    "\n",
    "## Load pretrained models\n",
    "whitebox = net.to(device)\n",
    "whitebox.eval()\n",
    "EPS_Test = [0.0,0.01,0.02,0.03,0.04,0.05,0.06,0.07,0.08,0.09,0.10]\n",
    "print(EPS_Test)\n",
    "\n",
    "\n",
    "Auxiliary_Rotation_FGSM_acc_list = []\n",
    "\n",
    "\n",
    "for roi_EPS in EPS_Test:\n",
    "\n",
    "\n",
    "    # TODO: Set attack parameters here\n",
    "    ATK_EPS          = roi_EPS\n",
    "    whitebox_correct = 0.\n",
    "    running_total    = 0.\n",
    "    for batch_idx, (x_tf_0, x_tf_90, x_tf_180, x_tf_270, target_class) in enumerate(testloader):\n",
    "        batch_size  = 100 #x_tf_0.shape[0]\n",
    "        batch       = np.concatenate((x_tf_0,x_tf_90,x_tf_180,x_tf_270), 0)\n",
    "        batch       = torch.FloatTensor(batch).cuda()\n",
    "        target_rots = torch.cat((torch.zeros(batch_size),torch.ones(batch_size),2 * torch.ones(batch_size),3 * torch.ones(batch_size)), 0).long()\n",
    "\n",
    "        # copy inputs to device\n",
    "        batch          = batch.to(batch)\n",
    "        target_class   = target_class.to(device)\n",
    "        target_rots    = target_rots.to(device)\n",
    "        # TODO: Perform adversarial attack here\n",
    "        #-Within this function model has no influence\n",
    "        whitebox_adv_data = FGSM_attack(model=whitebox, device=device, dat=batch, \n",
    "        target_class=target_class,\n",
    "        target_rots=target_rots,\n",
    "        eps=ATK_EPS)\n",
    "        \n",
    "        '''\n",
    "        print((torch.max(torch.abs(whitebox_adv_data[1,:,:,:]-batch[1,:,:,:]))))\n",
    "\n",
    "        # Sanity checking if adversarial example is \"legal\"\n",
    "        assert(torch.max(torch.abs(whitebox_adv_data[100,:,:,:]-batch[100,:,:,:])) <= (ATK_EPS + 1e-5) )\n",
    "        assert(torch.max(torch.abs(whitebox_adv_data-batch)) <= (ATK_EPS + 1e-5) )\n",
    "        assert(whitebox_adv_data.max() == 1.)\n",
    "        assert(whitebox_adv_data.min() == 0.)\n",
    "        '''\n",
    "        \n",
    "        # Compute accuracy on perturbed data\n",
    "        with torch.no_grad():\n",
    "            # Stat keeping - whitebox\n",
    "            logits, pen                 = whitebox(whitebox_adv_data)\n",
    "            classification_logits       = logits[:batch_size]\n",
    "            _, predicted = classification_logits.max(1)\n",
    "            whitebox_correct += predicted.eq(target_class).sum().item()\n",
    "            running_total += target_class.size(0)\n",
    "\n",
    "    # Print final \n",
    "    whitebox_acc = whitebox_correct/running_total\n",
    "    Auxiliary_Rotation_FGSM_acc_list.append(whitebox_acc)\n",
    "    print(\"Attack Epsilon: {}; Whitebox Accuracy: {}\".format(ATK_EPS, whitebox_acc))\n",
    "\n",
    "print(\"Done!\")\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "fb4c7e9d",
   "metadata": {},
   "source": [
    "# Mixup "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "aee62b31",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-12-15T09:10:33.344196Z",
     "start_time": "2022-12-15T09:10:06.708161Z"
    },
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Architecture List:\n",
      "['sm_52', 'sm_60', 'sm_61', 'sm_70', 'sm_75', 'sm_80', 'sm_86', 'compute_86']\n",
      "0.5714025946899135\n",
      "Run on GPU...\n",
      "Files already downloaded and verified\n",
      "Test Loss=0.3053, Test accuracy=0.9169\n",
      "Files already downloaded and verified\n",
      "Test Loss=0.3053, Test accuracy=0.9169\n",
      "Initial Accuracy of Whitebox Model:  0.9169\n",
      "[0.0, 0.01, 0.02, 0.03, 0.04, 0.05, 0.06, 0.07, 0.08, 0.09, 0.1]\n",
      "Files already downloaded and verified\n",
      "Attack Epsilon: 0.0; Whitebox Accuracy: 0.9169\n",
      "Done!\n",
      "Attack Epsilon: 0.01; Whitebox Accuracy: 0.6553\n",
      "Done!\n",
      "Attack Epsilon: 0.02; Whitebox Accuracy: 0.5271\n",
      "Done!\n",
      "Attack Epsilon: 0.03; Whitebox Accuracy: 0.4696\n",
      "Done!\n",
      "Attack Epsilon: 0.04; Whitebox Accuracy: 0.4307\n",
      "Done!\n",
      "Attack Epsilon: 0.05; Whitebox Accuracy: 0.407\n",
      "Done!\n",
      "Attack Epsilon: 0.06; Whitebox Accuracy: 0.3892\n",
      "Done!\n",
      "Attack Epsilon: 0.07; Whitebox Accuracy: 0.377\n",
      "Done!\n",
      "Attack Epsilon: 0.08; Whitebox Accuracy: 0.3671\n",
      "Done!\n",
      "Attack Epsilon: 0.09; Whitebox Accuracy: 0.3581\n",
      "Done!\n",
      "Attack Epsilon: 0.1; Whitebox Accuracy: 0.3511\n",
      "Done!\n"
     ]
    }
   ],
   "source": [
    "# import necessary dependencies\n",
    "import argparse\n",
    "import os, sys\n",
    "import time\n",
    "import datetime\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.nn.functional as F\n",
    "import torchvision\n",
    "import torchvision.transforms as transforms\n",
    "import torch.nn as nn\n",
    "import torch.optim as optim\n",
    "from tqdm import tqdm_notebook as tqdm\n",
    "from torchvision.transforms import InterpolationMode\n",
    "from torch.utils.data import DataLoader\n",
    "from torchvision import datasets\n",
    "from torchsummary import summary\n",
    "from torch.autograd import Variable\n",
    "import PIL\n",
    "from PIL import Image\n",
    "from torch.utils.data import Subset\n",
    "from torch.utils.tensorboard import SummaryWriter\n",
    "import random\n",
    "import pandas as pd\n",
    "\n",
    "print('Architecture List:')\n",
    "print(torch.cuda.get_arch_list())\n",
    "random.seed(10)\n",
    "print(random.random())\n",
    "import warnings\n",
    "warnings.filterwarnings(\"ignore\", category=DeprecationWarning)\n",
    "warnings.simplefilter('ignore')\n",
    "\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.nn.functional as F\n",
    "import attacks\n",
    "\n",
    "\n",
    "\n",
    "# GPU check\n",
    "device = 'cuda' if torch.cuda.is_available() else 'cpu'\n",
    "if device =='cuda':\n",
    "    print(\"Run on GPU...\")\n",
    "else:\n",
    "    print(\"Run on CPU...\")\n",
    "\n",
    "\n",
    "# Residual block\n",
    "class ResBlock(nn.Module):\n",
    "    def __init__(self, in_channels, out_channels, stride=1, downsample=None):\n",
    "        super(ResBlock, self).__init__()\n",
    "        self.conv1 = nn.Conv2d(in_channels, out_channels, kernel_size=3, stride=stride, padding=1, bias=False)\n",
    "        self.bn1 = nn.BatchNorm2d(out_channels)\n",
    "        self.conv2 = nn.Conv2d(out_channels, out_channels, kernel_size=3, stride=1, padding=1, bias=False)\n",
    "        self.bn2 = nn.BatchNorm2d(out_channels)\n",
    "        self.downsample = downsample\n",
    "\n",
    "    def forward(self,x):\n",
    "        identity = x\n",
    "        out = F.relu(self.bn1(self.conv1(x.to(device))))\n",
    "        out = self.bn2(self.conv2(out))\n",
    "        if self.downsample:\n",
    "            identity = self.downsample(x)\n",
    "        out += identity\n",
    "        out = F.relu(out)\n",
    "        return out\n",
    "\n",
    "\n",
    "# ResNet\n",
    "class ResNet(nn.Module):\n",
    "    def __init__(self, block, numBlocks, num_classes=10):\n",
    "        super(ResNet, self).__init__()\n",
    "        self.in_channels = 16\n",
    "        self.conv1   = nn.Conv2d(3, 16, kernel_size=3, stride=1, padding=1, bias=False)\n",
    "        self.bn1     = nn.BatchNorm2d(16)\n",
    "        self.layer1  = self.make_layer(block, 16, numBlocks[0], stride=1)\n",
    "        self.layer2  = self.make_layer(block, 32, numBlocks[1], stride=2)\n",
    "        self.layer3  = self.make_layer(block, 64, numBlocks[2], stride=2)\n",
    "        self.avgpool = nn.AvgPool2d(8)\n",
    "        self.fc1     = nn.Linear(64, num_classes)\n",
    "\n",
    "    def make_layer(self, block, out_channels, numBlocks, stride):\n",
    "        downsample = None\n",
    "        if (stride != 1) or (self.in_channels != out_channels):\n",
    "            downsample = nn.Sequential(\n",
    "                nn.Conv2d(self.in_channels, out_channels, kernel_size=1, stride=stride, bias=False),\n",
    "                nn.BatchNorm2d(out_channels))\n",
    "        layers = []\n",
    "        layers.append(block(self.in_channels, out_channels, stride, downsample))\n",
    "        self.in_channels = out_channels\n",
    "        for i in range(1, numBlocks):\n",
    "            layers.append(block(out_channels, out_channels))\n",
    "        return nn.Sequential(*layers)\n",
    "\n",
    "    def forward(self,x):\n",
    "        out = F.relu(self.bn1(self.conv1(x)))\n",
    "        out = self.layer1(out)\n",
    "        out = self.layer2(out)\n",
    "        out = self.layer3(out)\n",
    "        out = self.avgpool(out)\n",
    "        out = out.view(out.size(0),-1)\n",
    "        out = self.fc1(out)\n",
    "        return out\n",
    "\n",
    "def test_CIFAR10(net,criterion,root_data_dir,batch_size):\n",
    "    transform_test = transforms.Compose([\n",
    "    transforms.PILToTensor(),\n",
    "    transforms.ConvertImageDtype(torch.float),\n",
    "    transforms.Normalize((0.4914, 0.4822, 0.4465), (0.2023, 0.1994, 0.2010)),])\n",
    "\n",
    "    testset = torchvision.datasets.CIFAR10(root=root_data_dir, train=False, download=True, transform=transform_test)\n",
    "    testloader = torch.utils.data.DataLoader(testset, batch_size=batch_size, shuffle=False, num_workers=4)\n",
    "    classes = ('plane', 'car', 'bird', 'cat', 'deer', 'dog', 'frog', 'horse', 'ship', 'truck')\n",
    "\n",
    "    criterion = criterion\n",
    "\n",
    "    net.eval()\n",
    "    test_loss = 0\n",
    "    correct = 0\n",
    "    total = 0\n",
    "    with torch.no_grad():\n",
    "        for batch_idx, (inputs, targets) in enumerate(testloader):\n",
    "            inputs, targets = inputs.to(device), targets.to(device)\n",
    "            outputs = net(inputs)\n",
    "            loss = criterion(outputs, targets)\n",
    "\n",
    "            test_loss += loss.item()\n",
    "            _, predicted = outputs.max(1)\n",
    "            total += targets.size(0)\n",
    "            correct += predicted.eq(targets).sum().item()\n",
    "    num_val_steps = len(testloader)\n",
    "    val_acc = correct / total\n",
    "    print(\"Test Loss=%.4f, Test accuracy=%.4f\" % (test_loss / (num_val_steps), val_acc))\n",
    "    return test_loss / (num_val_steps), val_acc\n",
    "# a few arguments, do NOT change these\n",
    "DATA_ROOT           = \"./data\"\n",
    "CORRUPTED_DATA_ROOT = \"/data/usr/ft42/nobackup/CIFAR_10_C/CIFAR-10-C/\"\n",
    "# the folder where the trained model is saved\n",
    "CHECKPOINT_FOLDER   = \"savedFiles/saved_models/\"\n",
    "# the folder where the figures are saved\n",
    "FIGURES_FOLDER      = \"savedFiles/saved_figures/\"\n",
    "# the folder where the csvs are saved\n",
    "CSVS_FOLDER         = \"savedFiles/saved_csvs/\"\n",
    "# name of the saved model\n",
    "MODELNAME_REG       = \"resnet20_mixup\"\n",
    "Alpha_i             =0.2\n",
    "\n",
    "\n",
    "# a few arguments, do NOT change these\n",
    "DATA_ROOT           = \"./data\"\n",
    "CORRUPTED_DATA_ROOT = \"/data/usr/ft42/nobackup/CIFAR_10_C/CIFAR-10-C/\"\n",
    "# the folder where the trained model is saved\n",
    "CHECKPOINT_FOLDER   = \"savedFiles/saved_models/\"\n",
    "# the folder where the figures are saved\n",
    "FIGURES_FOLDER      = \"savedFiles/saved_figures/\"\n",
    "# the folder where the csvs are saved\n",
    "CSVS_FOLDER         = \"savedFiles/saved_csvs/\"\n",
    "# name of the saved model\n",
    "MODELNAME_REG       = \"resnet20_mixup\"\n",
    "\n",
    "# hyperparameters, do NOT change right now\n",
    "TRAIN_BATCH_SIZE = 256    # training batch size\n",
    "VAL_BATCH_SIZE   = 100    # validation batch size\n",
    "INITIAL_LR       = 0.1    # initial learning rate\n",
    "MOMENTUM         = 0.9    # momentum for optimizer\n",
    "REG              = 1e-3   # L2 regularization strength\n",
    "EPOCHS           = 200    # total number of training epochs\n",
    "DECAY_EPOCHS     = 20     # parameter for LR schedule (decay after XX epochs)\n",
    "DECAY            = 0.5    # parameter for LR schedule (decay multiplier)\n",
    "MIXUP_ALPHA_RANGE = [0.2]\n",
    "MODELNAME = MODELNAME_REG+'_{}'.format(Alpha_i)\n",
    "criterion = nn.CrossEntropyLoss().to(device)\n",
    "\n",
    "# start the training/validation process\n",
    "best_val_acc          = 0\n",
    "current_learning_rate = INITIAL_LR\n",
    "epochs                = np.linspace(1,EPOCHS,EPOCHS)\n",
    "\n",
    "    #-------------- Testingh---------------#\n",
    "test_data_list=[]\n",
    "test_acc_list =[]\n",
    "test_loss_list=[]\n",
    "\n",
    "\n",
    "net = ResNet(ResBlock, [3, 3, 3]).to(device)\n",
    "net = net.to(device)\n",
    "net.load_state_dict(torch.load(CHECKPOINT_FOLDER+MODELNAME+'.pt',map_location='cuda:0'))\n",
    "test_loss,test_acc=test_CIFAR10(net,criterion,root_data_dir=DATA_ROOT,batch_size=VAL_BATCH_SIZE)\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "## Load pretrained models\n",
    "whitebox = net\n",
    "whitebox.eval()\n",
    "test_loss,test_acc=test_CIFAR10(whitebox,criterion,root_data_dir=DATA_ROOT,batch_size=VAL_BATCH_SIZE)\n",
    "print(\"Initial Accuracy of Whitebox Model: \",test_acc)\n",
    "\n",
    "\n",
    "## Test the models against an adversarial attack\n",
    "\n",
    "\n",
    "EPS_Test = [0.0,0.01,0.02,0.03,0.04,0.05,0.06,0.07,0.08,0.09,0.10]\n",
    "print(EPS_Test)\n",
    "\n",
    "\n",
    "Mixup_FGSM_acc_list = []\n",
    "\n",
    "\n",
    "\n",
    "transform_test = transforms.Compose([transforms.PILToTensor(),\n",
    "    transforms.ConvertImageDtype(torch.float),\n",
    "    transforms.Normalize((0.4914, 0.4822, 0.4465), (0.2023, 0.1994, 0.2010)),])\n",
    "\n",
    "testset     = torchvision.datasets.CIFAR10(root=DATA_ROOT, train=False, download=True, transform=transform_test)\n",
    "test_loader = torch.utils.data.DataLoader(testset, batch_size=VAL_BATCH_SIZE, shuffle=False, num_workers=4)\n",
    "\n",
    "for roi_EPS in EPS_Test:\n",
    "\n",
    "\n",
    "    # TODO: Set attack parameters here\n",
    "    ATK_EPS = roi_EPS\n",
    "\n",
    "\n",
    "    whitebox_correct = 0.\n",
    "    running_total = 0.\n",
    "    for batch_idx,(data,labels) in enumerate(test_loader):\n",
    "        data = data.to(device) \n",
    "        labels = labels.to(device)\n",
    "\n",
    "        # TODO: Perform adversarial attack here\n",
    "        #-Within this function model has no influence\n",
    "        whitebox_adv_data = attacks.FGSM_attack(model=whitebox, device=device, dat=data, lbl=labels, eps=ATK_EPS)\n",
    "        #blackbox_adv_data = attacks.FGSM_attack(model=blackbox, device=device, dat=data, lbl=labels, eps=ATK_EPS)\n",
    "\n",
    "        # Sanity checking if adversarial example is \"legal\"\n",
    "        #assert(torch.max(torch.abs(whitebox_adv_data-data)) <= (ATK_EPS + 1e-5) )\n",
    "        #assert(whitebox_adv_data.max() == 1.)\n",
    "        #assert(whitebox_adv_data.min() == 0.)\n",
    "        \n",
    "        #assert(torch.max(torch.abs(blackbox_adv_data-data)) <= (ATK_EPS + 1e-5) )\n",
    "        #assert(blackbox_adv_data.max() == 1.)\n",
    "        #assert(blackbox_adv_data.min() == 0.)\n",
    "        \n",
    "        # Compute accuracy on perturbed data\n",
    "        with torch.no_grad():\n",
    "            # Stat keeping - whitebox\n",
    "            whitebox_outputs = whitebox(whitebox_adv_data)\n",
    "            #print(whitebox_outputs.shape)\n",
    "            _,whitebox_preds = whitebox_outputs.max(1)\n",
    "            whitebox_correct += whitebox_preds.eq(labels).sum().item()\n",
    "            running_total += labels.size(0)\n",
    "\n",
    "        '''\n",
    "        # Plot some samples\n",
    "        if batch_idx == 1:\n",
    "            plt.figure(figsize=(15,5))\n",
    "            for jj in range(6):\n",
    "                plt.subplot(2,6,jj+1);plt.imshow(whitebox_adv_data[jj,0].cpu().numpy(),cmap='gray');plt.axis(\"off\");plt.title('whitebox Adv,ϵ={:.3f}'.format(roi_EPS))\n",
    "            for jj in range(6):\n",
    "                plt.subplot(2,6,6+jj+1);plt.imshow(blackbox_adv_data[jj,0].cpu().numpy(),cmap='gray');plt.axis(\"off\");plt.title('blackbox Adv,ϵ={:.3f}'.format(roi_EPS))\n",
    "            plt.tight_layout()\n",
    "            plt.show()\n",
    "        '''\n",
    "    # Print final\n",
    "    whitebox_acc = whitebox_correct/running_total\n",
    "    Mixup_FGSM_acc_list.append(whitebox_acc)\n",
    "\n",
    "\n",
    "    print(\"Attack Epsilon: {}; Whitebox Accuracy: {}\".format(ATK_EPS, whitebox_acc))\n",
    "\n",
    "    print(\"Done!\")\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "fc9f6268",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-12-15T09:11:00.139143Z",
     "start_time": "2022-12-15T09:10:33.346616Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Run on GPU...\n",
      "Test Loss=0.3053, Test accuracy=0.9169\n",
      "Attack Epsilon: 0.0; Whitebox Accuracy: 0.9169\n",
      "Attack Epsilon: 0.01; Whitebox Accuracy: 0.6553\n",
      "Attack Epsilon: 0.02; Whitebox Accuracy: 0.5271\n",
      "Attack Epsilon: 0.03; Whitebox Accuracy: 0.4696\n",
      "Attack Epsilon: 0.04; Whitebox Accuracy: 0.4307\n",
      "Attack Epsilon: 0.05; Whitebox Accuracy: 0.407\n",
      "Attack Epsilon: 0.06; Whitebox Accuracy: 0.3892\n",
      "Attack Epsilon: 0.07; Whitebox Accuracy: 0.377\n",
      "Attack Epsilon: 0.08; Whitebox Accuracy: 0.3671\n",
      "Attack Epsilon: 0.09; Whitebox Accuracy: 0.3581\n",
      "Attack Epsilon: 0.1; Whitebox Accuracy: 0.3511\n"
     ]
    }
   ],
   "source": [
    "# import necessary dependencies\n",
    "import argparse\n",
    "import os, sys\n",
    "import time\n",
    "import datetime\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.nn.functional as F\n",
    "import torchvision\n",
    "import torchvision.transforms as transforms\n",
    "import torch.nn as nn\n",
    "import torch.optim as optim\n",
    "from tqdm import tqdm_notebook as tqdm\n",
    "from torchvision.transforms import InterpolationMode\n",
    "from torch.utils.data import DataLoader\n",
    "from torchvision import datasets\n",
    "from torchsummary import summary\n",
    "from torch.autograd import Variable\n",
    "import PIL\n",
    "from PIL import Image\n",
    "from torch.utils.data import Subset\n",
    "from torch.utils.tensorboard import SummaryWriter\n",
    "import random\n",
    "import pandas as pd\n",
    "random.seed(10)\n",
    "import warnings\n",
    "warnings.filterwarnings(\"ignore\", category=DeprecationWarning)\n",
    "warnings.simplefilter('ignore')\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.nn.functional as F\n",
    "import attacks\n",
    "\n",
    "\n",
    "\n",
    "# GPU check\n",
    "device = 'cuda' if torch.cuda.is_available() else 'cpu'\n",
    "if device =='cuda':\n",
    "    print(\"Run on GPU...\")\n",
    "else:\n",
    "    print(\"Run on CPU...\")\n",
    "\n",
    "\n",
    "# Residual block\n",
    "class ResBlock(nn.Module):\n",
    "    def __init__(self, in_channels, out_channels, stride=1, downsample=None):\n",
    "        super(ResBlock, self).__init__()\n",
    "        self.conv1 = nn.Conv2d(in_channels, out_channels, kernel_size=3, stride=stride, padding=1, bias=False)\n",
    "        self.bn1 = nn.BatchNorm2d(out_channels)\n",
    "        self.conv2 = nn.Conv2d(out_channels, out_channels, kernel_size=3, stride=1, padding=1, bias=False)\n",
    "        self.bn2 = nn.BatchNorm2d(out_channels)\n",
    "        self.downsample = downsample\n",
    "\n",
    "    def forward(self,x):\n",
    "        identity = x\n",
    "        out = F.relu(self.bn1(self.conv1(x.to(device))))\n",
    "        out = self.bn2(self.conv2(out))\n",
    "        if self.downsample:\n",
    "            identity = self.downsample(x)\n",
    "        out += identity\n",
    "        out = F.relu(out)\n",
    "        return out\n",
    "\n",
    "\n",
    "# ResNet\n",
    "class ResNet(nn.Module):\n",
    "    def __init__(self, block, numBlocks, num_classes=10):\n",
    "        super(ResNet, self).__init__()\n",
    "        self.in_channels = 16\n",
    "        self.conv1   = nn.Conv2d(3, 16, kernel_size=3, stride=1, padding=1, bias=False)\n",
    "        self.bn1     = nn.BatchNorm2d(16)\n",
    "        self.layer1  = self.make_layer(block, 16, numBlocks[0], stride=1)\n",
    "        self.layer2  = self.make_layer(block, 32, numBlocks[1], stride=2)\n",
    "        self.layer3  = self.make_layer(block, 64, numBlocks[2], stride=2)\n",
    "        self.avgpool = nn.AvgPool2d(8)\n",
    "        self.fc1     = nn.Linear(64, num_classes)\n",
    "\n",
    "    def make_layer(self, block, out_channels, numBlocks, stride):\n",
    "        downsample = None\n",
    "        if (stride != 1) or (self.in_channels != out_channels):\n",
    "            downsample = nn.Sequential(\n",
    "                nn.Conv2d(self.in_channels, out_channels, kernel_size=1, stride=stride, bias=False),\n",
    "                nn.BatchNorm2d(out_channels))\n",
    "        layers = []\n",
    "        layers.append(block(self.in_channels, out_channels, stride, downsample))\n",
    "        self.in_channels = out_channels\n",
    "        for i in range(1, numBlocks):\n",
    "            layers.append(block(out_channels, out_channels))\n",
    "        return nn.Sequential(*layers)\n",
    "\n",
    "    def forward(self,x):\n",
    "        out = F.relu(self.bn1(self.conv1(x)))\n",
    "        out = self.layer1(out)\n",
    "        out = self.layer2(out)\n",
    "        out = self.layer3(out)\n",
    "        out = self.avgpool(out)\n",
    "        out = out.view(out.size(0),-1)\n",
    "        out = self.fc1(out)\n",
    "        return out\n",
    "\n",
    "def test_CIFAR10_adversarial(net,criterion,root_data_dir,batch_size,eps):\n",
    "    transform_test = transforms.Compose([\n",
    "    transforms.PILToTensor(),\n",
    "    transforms.ConvertImageDtype(torch.float),\n",
    "    transforms.Normalize((0.4914, 0.4822, 0.4465), (0.2023, 0.1994, 0.2010)),])\n",
    "\n",
    "    testset = torchvision.datasets.CIFAR10(root=root_data_dir, train=False, download=False, transform=transform_test)\n",
    "    testloader = torch.utils.data.DataLoader(testset, batch_size=batch_size, shuffle=False, num_workers=4)\n",
    "    classes = ('plane', 'car', 'bird', 'cat', 'deer', 'dog', 'frog', 'horse', 'ship', 'truck')\n",
    "\n",
    "    criterion = criterion\n",
    "\n",
    "    net.eval()\n",
    "    test_loss = 0\n",
    "    correct = 0\n",
    "    total = 0\n",
    "    for batch_idx, (inputs, targets) in enumerate(testloader):\n",
    "        inputs, targets = inputs.to(device), targets.to(device)\n",
    "        inputs  = attacks.FGSM_attack(model=net, device=device, dat=inputs, lbl=targets, eps=eps)\n",
    "        with torch.no_grad():\n",
    "            outputs = net(inputs)\n",
    "            loss = criterion(outputs, targets)\n",
    "            test_loss += loss.item()\n",
    "            _, predicted = outputs.max(1)\n",
    "            total += targets.size(0)\n",
    "            correct += predicted.eq(targets).sum().item()\n",
    "    num_val_steps = len(testloader)\n",
    "    val_acc = correct / total\n",
    "    #print(\"Test Loss=%.4f, Test accuracy=%.4f\" % (test_loss / (num_val_steps), val_acc))\n",
    "    return test_loss / (num_val_steps), val_acc\n",
    "\n",
    "def test_CIFAR10(net,criterion,root_data_dir,batch_size):\n",
    "    transform_test = transforms.Compose([\n",
    "    transforms.PILToTensor(),\n",
    "    transforms.ConvertImageDtype(torch.float),\n",
    "    transforms.Normalize((0.4914, 0.4822, 0.4465), (0.2023, 0.1994, 0.2010)),])\n",
    "\n",
    "    testset = torchvision.datasets.CIFAR10(root=root_data_dir, train=False, download=False, transform=transform_test)\n",
    "    testloader = torch.utils.data.DataLoader(testset, batch_size=batch_size, shuffle=False, num_workers=4)\n",
    "    classes = ('plane', 'car', 'bird', 'cat', 'deer', 'dog', 'frog', 'horse', 'ship', 'truck')\n",
    "\n",
    "    criterion = criterion\n",
    "\n",
    "    net.eval()\n",
    "    test_loss = 0\n",
    "    correct = 0\n",
    "    total = 0\n",
    "    for batch_idx, (inputs, targets) in enumerate(testloader):\n",
    "        inputs, targets = inputs.to(device), targets.to(device)\n",
    "        with torch.no_grad():\n",
    "            outputs = net(inputs)\n",
    "            loss = criterion(outputs, targets)\n",
    "            test_loss += loss.item()\n",
    "            _, predicted = outputs.max(1)\n",
    "            total += targets.size(0)\n",
    "            correct += predicted.eq(targets).sum().item()\n",
    "    num_val_steps = len(testloader)\n",
    "    val_acc = correct / total\n",
    "    print(\"Test Loss=%.4f, Test accuracy=%.4f\" % (test_loss / (num_val_steps), val_acc))\n",
    "    return test_loss / (num_val_steps), val_acc\n",
    "# a few arguments, do NOT change these\n",
    "DATA_ROOT           = \"./data\"\n",
    "CORRUPTED_DATA_ROOT = \"/data/usr/ft42/nobackup/CIFAR_10_C/CIFAR-10-C/\"\n",
    "# the folder where the trained model is saved\n",
    "CHECKPOINT_FOLDER   = \"savedFiles/saved_models/\"\n",
    "# the folder where the figures are saved\n",
    "FIGURES_FOLDER      = \"savedFiles/saved_figures/\"\n",
    "# the folder where the csvs are saved\n",
    "CSVS_FOLDER         = \"savedFiles/saved_csvs/\"\n",
    "# name of the saved model\n",
    "MODELNAME_REG       = \"resnet20_mixup\"\n",
    "Alpha_i             =0.2\n",
    "\n",
    "\n",
    "# a few arguments, do NOT change these\n",
    "DATA_ROOT           = \"./data\"\n",
    "CORRUPTED_DATA_ROOT = \"/data/usr/ft42/nobackup/CIFAR_10_C/CIFAR-10-C/\"\n",
    "# the folder where the trained model is saved\n",
    "CHECKPOINT_FOLDER   = \"savedFiles/saved_models/\"\n",
    "# the folder where the figures are saved\n",
    "FIGURES_FOLDER      = \"savedFiles/saved_figures/\"\n",
    "# the folder where the csvs are saved\n",
    "CSVS_FOLDER         = \"savedFiles/saved_csvs/\"\n",
    "# name of the saved model\n",
    "MODELNAME_REG       = \"resnet20_mixup\"\n",
    "\n",
    "# hyperparameters, do NOT change right now\n",
    "TRAIN_BATCH_SIZE = 256    # training batch size\n",
    "VAL_BATCH_SIZE   = 100    # validation batch size\n",
    "INITIAL_LR       = 0.1    # initial learning rate\n",
    "MOMENTUM         = 0.9    # momentum for optimizer\n",
    "REG              = 1e-3   # L2 regularization strength\n",
    "EPOCHS           = 200    # total number of training epochs\n",
    "DECAY_EPOCHS     = 20     # parameter for LR schedule (decay after XX epochs)\n",
    "DECAY            = 0.5    # parameter for LR schedule (decay multiplier)\n",
    "MIXUP_ALPHA_RANGE = [0.2,0.4,0.6,0.8,1.0,2.0]\n",
    "MODELNAME = MODELNAME_REG+'_{}'.format(Alpha_i)\n",
    "criterion = nn.CrossEntropyLoss().to(device)\n",
    "\n",
    "# start the training/validation process\n",
    "best_val_acc          = 0\n",
    "current_learning_rate = INITIAL_LR\n",
    "epochs                = np.linspace(1,EPOCHS,EPOCHS)\n",
    "\n",
    "    #-------------- Testingh---------------#\n",
    "test_data_list=[]\n",
    "test_acc_list =[]\n",
    "test_loss_list=[]\n",
    "\n",
    "\n",
    "net = ResNet(ResBlock, [3, 3, 3]).to(device)\n",
    "net = net.to(device)\n",
    "net.load_state_dict(torch.load(CHECKPOINT_FOLDER+MODELNAME+'.pt',map_location='cuda:0'))\n",
    "\n",
    "\n",
    "EPS_Test = [0.0,0.01,0.02,0.03,0.04,0.05,0.06,0.07,0.08,0.09,0.10]\n",
    "Mixup_FGSM_acc_list = []\n",
    "test_CIFAR10(net,criterion,root_data_dir=DATA_ROOT,batch_size=VAL_BATCH_SIZE)\n",
    "for roi_EPS in EPS_Test:\n",
    "    test_loss,test_acc=test_CIFAR10_adversarial(net,criterion,root_data_dir=DATA_ROOT,batch_size=VAL_BATCH_SIZE,eps=roi_EPS)\n",
    "    Mixup_FGSM_acc_list.append(test_acc)\n",
    "    print(\"Attack Epsilon: {}; Whitebox Accuracy: {}\".format(roi_EPS, test_acc))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "be0e0bee",
   "metadata": {},
   "source": [
    "# Cutout"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "7f18665e",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-12-15T09:11:27.093380Z",
     "start_time": "2022-12-15T09:11:00.143016Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Run on GPU...\n",
      "Test Loss=0.2753, Test accuracy=0.9169\n",
      "Attack Epsilon: 0.0; Whitebox Accuracy: 0.9169\n",
      "Attack Epsilon: 0.01; Whitebox Accuracy: 0.6233\n",
      "Attack Epsilon: 0.02; Whitebox Accuracy: 0.4187\n",
      "Attack Epsilon: 0.03; Whitebox Accuracy: 0.317\n",
      "Attack Epsilon: 0.04; Whitebox Accuracy: 0.2659\n",
      "Attack Epsilon: 0.05; Whitebox Accuracy: 0.2308\n",
      "Attack Epsilon: 0.06; Whitebox Accuracy: 0.2111\n",
      "Attack Epsilon: 0.07; Whitebox Accuracy: 0.1948\n",
      "Attack Epsilon: 0.08; Whitebox Accuracy: 0.1838\n",
      "Attack Epsilon: 0.09; Whitebox Accuracy: 0.1743\n",
      "Attack Epsilon: 0.1; Whitebox Accuracy: 0.1693\n"
     ]
    }
   ],
   "source": [
    "# import necessary dependencies\n",
    "import argparse\n",
    "import os, sys\n",
    "import time\n",
    "import datetime\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.nn.functional as F\n",
    "import torchvision\n",
    "import torchvision.transforms as transforms\n",
    "import torch.nn as nn\n",
    "import torch.optim as optim\n",
    "from tqdm import tqdm_notebook as tqdm\n",
    "from torchvision.transforms import InterpolationMode\n",
    "from torch.utils.data import DataLoader\n",
    "from torchvision import datasets\n",
    "from torchsummary import summary\n",
    "from torch.autograd import Variable\n",
    "import PIL\n",
    "from PIL import Image\n",
    "from torch.utils.data import Subset\n",
    "from torch.utils.tensorboard import SummaryWriter\n",
    "import random\n",
    "import pandas as pd\n",
    "random.seed(10)\n",
    "import warnings\n",
    "warnings.filterwarnings(\"ignore\", category=DeprecationWarning)\n",
    "warnings.simplefilter('ignore')\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.nn.functional as F\n",
    "import attacks\n",
    "\n",
    "\n",
    "\n",
    "# GPU check\n",
    "device = 'cuda' if torch.cuda.is_available() else 'cpu'\n",
    "if device =='cuda':\n",
    "    print(\"Run on GPU...\")\n",
    "else:\n",
    "    print(\"Run on CPU...\")\n",
    "\n",
    "\n",
    "# Residual block\n",
    "class ResBlock(nn.Module):\n",
    "    def __init__(self, in_channels, out_channels, stride=1, downsample=None):\n",
    "        super(ResBlock, self).__init__()\n",
    "        self.conv1 = nn.Conv2d(in_channels, out_channels, kernel_size=3, stride=stride, padding=1, bias=False)\n",
    "        self.bn1 = nn.BatchNorm2d(out_channels)\n",
    "        self.conv2 = nn.Conv2d(out_channels, out_channels, kernel_size=3, stride=1, padding=1, bias=False)\n",
    "        self.bn2 = nn.BatchNorm2d(out_channels)\n",
    "        self.downsample = downsample\n",
    "\n",
    "    def forward(self,x):\n",
    "        identity = x\n",
    "        out = F.relu(self.bn1(self.conv1(x.to(device))))\n",
    "        out = self.bn2(self.conv2(out))\n",
    "        if self.downsample:\n",
    "            identity = self.downsample(x)\n",
    "        out += identity\n",
    "        out = F.relu(out)\n",
    "        return out\n",
    "\n",
    "\n",
    "# ResNet\n",
    "class ResNet(nn.Module):\n",
    "    def __init__(self, block, numBlocks, num_classes=10):\n",
    "        super(ResNet, self).__init__()\n",
    "        self.in_channels = 16\n",
    "        self.conv1   = nn.Conv2d(3, 16, kernel_size=3, stride=1, padding=1, bias=False)\n",
    "        self.bn1     = nn.BatchNorm2d(16)\n",
    "        self.layer1  = self.make_layer(block, 16, numBlocks[0], stride=1)\n",
    "        self.layer2  = self.make_layer(block, 32, numBlocks[1], stride=2)\n",
    "        self.layer3  = self.make_layer(block, 64, numBlocks[2], stride=2)\n",
    "        self.avgpool = nn.AvgPool2d(8)\n",
    "        self.fc1     = nn.Linear(64, num_classes)\n",
    "\n",
    "    def make_layer(self, block, out_channels, numBlocks, stride):\n",
    "        downsample = None\n",
    "        if (stride != 1) or (self.in_channels != out_channels):\n",
    "            downsample = nn.Sequential(\n",
    "                nn.Conv2d(self.in_channels, out_channels, kernel_size=1, stride=stride, bias=False),\n",
    "                nn.BatchNorm2d(out_channels))\n",
    "        layers = []\n",
    "        layers.append(block(self.in_channels, out_channels, stride, downsample))\n",
    "        self.in_channels = out_channels\n",
    "        for i in range(1, numBlocks):\n",
    "            layers.append(block(out_channels, out_channels))\n",
    "        return nn.Sequential(*layers)\n",
    "\n",
    "    def forward(self,x):\n",
    "        out = F.relu(self.bn1(self.conv1(x)))\n",
    "        out = self.layer1(out)\n",
    "        out = self.layer2(out)\n",
    "        out = self.layer3(out)\n",
    "        out = self.avgpool(out)\n",
    "        out = out.view(out.size(0),-1)\n",
    "        out = self.fc1(out)\n",
    "        return out\n",
    "\n",
    "def test_CIFAR10_adversarial(net,criterion,root_data_dir,batch_size,eps):\n",
    "    transform_test = transforms.Compose([\n",
    "    transforms.PILToTensor(),\n",
    "    transforms.ConvertImageDtype(torch.float),\n",
    "    transforms.Normalize((0.4914, 0.4822, 0.4465), (0.2023, 0.1994, 0.2010)),])\n",
    "\n",
    "    testset = torchvision.datasets.CIFAR10(root=root_data_dir, train=False, download=False, transform=transform_test)\n",
    "    testloader = torch.utils.data.DataLoader(testset, batch_size=batch_size, shuffle=False, num_workers=4)\n",
    "    classes = ('plane', 'car', 'bird', 'cat', 'deer', 'dog', 'frog', 'horse', 'ship', 'truck')\n",
    "\n",
    "    criterion = criterion\n",
    "\n",
    "    net.eval()\n",
    "    test_loss = 0\n",
    "    correct = 0\n",
    "    total = 0\n",
    "    for batch_idx, (inputs, targets) in enumerate(testloader):\n",
    "        inputs, targets = inputs.to(device), targets.to(device)\n",
    "        inputs  = attacks.FGSM_attack(model=net, device=device, dat=inputs, lbl=targets, eps=eps)\n",
    "        with torch.no_grad():\n",
    "            outputs = net(inputs)\n",
    "            loss = criterion(outputs, targets)\n",
    "            test_loss += loss.item()\n",
    "            _, predicted = outputs.max(1)\n",
    "            total += targets.size(0)\n",
    "            correct += predicted.eq(targets).sum().item()\n",
    "    num_val_steps = len(testloader)\n",
    "    val_acc = correct / total\n",
    "    #print(\"Test Loss=%.4f, Test accuracy=%.4f\" % (test_loss / (num_val_steps), val_acc))\n",
    "    return test_loss / (num_val_steps), val_acc\n",
    "\n",
    "def test_CIFAR10(net,criterion,root_data_dir,batch_size):\n",
    "    transform_test = transforms.Compose([\n",
    "    transforms.PILToTensor(),\n",
    "    transforms.ConvertImageDtype(torch.float),\n",
    "    transforms.Normalize((0.4914, 0.4822, 0.4465), (0.2023, 0.1994, 0.2010)),])\n",
    "\n",
    "    testset = torchvision.datasets.CIFAR10(root=root_data_dir, train=False, download=False, transform=transform_test)\n",
    "    testloader = torch.utils.data.DataLoader(testset, batch_size=batch_size, shuffle=False, num_workers=4)\n",
    "    classes = ('plane', 'car', 'bird', 'cat', 'deer', 'dog', 'frog', 'horse', 'ship', 'truck')\n",
    "\n",
    "    criterion = criterion\n",
    "\n",
    "    net.eval()\n",
    "    test_loss = 0\n",
    "    correct = 0\n",
    "    total = 0\n",
    "    for batch_idx, (inputs, targets) in enumerate(testloader):\n",
    "        inputs, targets = inputs.to(device), targets.to(device)\n",
    "        with torch.no_grad():\n",
    "            outputs = net(inputs)\n",
    "            loss = criterion(outputs, targets)\n",
    "            test_loss += loss.item()\n",
    "            _, predicted = outputs.max(1)\n",
    "            total += targets.size(0)\n",
    "            correct += predicted.eq(targets).sum().item()\n",
    "    num_val_steps = len(testloader)\n",
    "    val_acc = correct / total\n",
    "    print(\"Test Loss=%.4f, Test accuracy=%.4f\" % (test_loss / (num_val_steps), val_acc))\n",
    "    return test_loss / (num_val_steps), val_acc\n",
    "# a few arguments, do NOT change these\n",
    "DATA_ROOT           = \"./data\"\n",
    "CORRUPTED_DATA_ROOT = \"/data/usr/ft42/nobackup/CIFAR_10_C/CIFAR-10-C/\"\n",
    "# the folder where the trained model is saved\n",
    "CHECKPOINT_FOLDER   = \"savedFiles/saved_models/\"\n",
    "# the folder where the figures are saved\n",
    "FIGURES_FOLDER      = \"savedFiles/saved_figures/\"\n",
    "# the folder where the csvs are saved\n",
    "CSVS_FOLDER         = \"savedFiles/saved_csvs/\"\n",
    "# name of the saved model\n",
    "\n",
    "\n",
    "\n",
    "# a few arguments, do NOT change these\n",
    "DATA_ROOT           = \"./data\"\n",
    "CORRUPTED_DATA_ROOT = \"/data/usr/ft42/nobackup/CIFAR_10_C/CIFAR-10-C/\"\n",
    "# the folder where the trained model is saved\n",
    "CHECKPOINT_FOLDER   = \"savedFiles/saved_models/\"\n",
    "# the folder where the figures are saved\n",
    "FIGURES_FOLDER      = \"savedFiles/saved_figures/\"\n",
    "# the folder where the csvs are saved\n",
    "CSVS_FOLDER         = \"savedFiles/saved_csvs/\"\n",
    "# name of the saved model\n",
    "\n",
    "\n",
    "# hyperparameters, do NOT change right now\n",
    "TRAIN_BATCH_SIZE = 256    # training batch size\n",
    "VAL_BATCH_SIZE   = 100    # validation batch size\n",
    "INITIAL_LR       = 0.1    # initial learning rate\n",
    "MOMENTUM         = 0.9    # momentum for optimizer\n",
    "REG              = 1e-3   # L2 regularization strength\n",
    "EPOCHS           = 200    # total number of training epochs\n",
    "DECAY_EPOCHS     = 20     # parameter for LR schedule (decay after XX epochs)\n",
    "DECAY            = 0.5    # parameter for LR schedule (decay multiplier)\n",
    "criterion = nn.CrossEntropyLoss().to(device)\n",
    "\n",
    "# start the training/validation process\n",
    "best_val_acc          = 0\n",
    "current_learning_rate = INITIAL_LR\n",
    "epochs                = np.linspace(1,EPOCHS,EPOCHS)\n",
    "\n",
    "    #-------------- Testingh---------------#\n",
    "test_data_list=[]\n",
    "test_acc_list =[]\n",
    "test_loss_list=[]\n",
    "\n",
    "\n",
    "net = ResNet(ResBlock, [3, 3, 3]).to(device)\n",
    "net = net.to(device)\n",
    "net.load_state_dict(torch.load('/data/usr/ss1305/spie23_emphysemaSegmentation/sharedFolder/ECE661/Project/savedFiles/saved_models/resnet20_cutout_1_2.pt',map_location='cuda:0'))\n",
    "\n",
    "\n",
    "EPS_Test = [0.0,0.01,0.02,0.03,0.04,0.05,0.06,0.07,0.08,0.09,0.10]\n",
    "Cutout_FGSM_acc_list = []\n",
    "test_CIFAR10(net,criterion,root_data_dir=DATA_ROOT,batch_size=VAL_BATCH_SIZE)\n",
    "for roi_EPS in EPS_Test:\n",
    "    test_loss,test_acc=test_CIFAR10_adversarial(net,criterion,root_data_dir=DATA_ROOT,batch_size=VAL_BATCH_SIZE,eps=roi_EPS)\n",
    "    Cutout_FGSM_acc_list.append(test_acc)\n",
    "    print(\"Attack Epsilon: {}; Whitebox Accuracy: {}\".format(roi_EPS, test_acc))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "acdd34a8",
   "metadata": {},
   "source": [
    "# Noregularizer\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "4c407a17",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-12-15T09:11:53.896475Z",
     "start_time": "2022-12-15T09:11:27.095867Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Run on GPU...\n",
      "Test Loss=0.2729, Test accuracy=0.9183\n",
      "Attack Epsilon: 0.0; Whitebox Accuracy: 0.9183\n",
      "Attack Epsilon: 0.01; Whitebox Accuracy: 0.6244\n",
      "Attack Epsilon: 0.02; Whitebox Accuracy: 0.4329\n",
      "Attack Epsilon: 0.03; Whitebox Accuracy: 0.3336\n",
      "Attack Epsilon: 0.04; Whitebox Accuracy: 0.2748\n",
      "Attack Epsilon: 0.05; Whitebox Accuracy: 0.2405\n",
      "Attack Epsilon: 0.06; Whitebox Accuracy: 0.2145\n",
      "Attack Epsilon: 0.07; Whitebox Accuracy: 0.1962\n",
      "Attack Epsilon: 0.08; Whitebox Accuracy: 0.184\n",
      "Attack Epsilon: 0.09; Whitebox Accuracy: 0.1725\n",
      "Attack Epsilon: 0.1; Whitebox Accuracy: 0.1621\n"
     ]
    }
   ],
   "source": [
    "# import necessary dependencies\n",
    "import argparse\n",
    "import os, sys\n",
    "import time\n",
    "import datetime\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.nn.functional as F\n",
    "import torchvision\n",
    "import torchvision.transforms as transforms\n",
    "import torch.nn as nn\n",
    "import torch.optim as optim\n",
    "from tqdm import tqdm_notebook as tqdm\n",
    "from torchvision.transforms import InterpolationMode\n",
    "from torch.utils.data import DataLoader\n",
    "from torchvision import datasets\n",
    "from torchsummary import summary\n",
    "from torch.autograd import Variable\n",
    "import PIL\n",
    "from PIL import Image\n",
    "from torch.utils.data import Subset\n",
    "from torch.utils.tensorboard import SummaryWriter\n",
    "import random\n",
    "import pandas as pd\n",
    "random.seed(10)\n",
    "import warnings\n",
    "warnings.filterwarnings(\"ignore\", category=DeprecationWarning)\n",
    "warnings.simplefilter('ignore')\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.nn.functional as F\n",
    "import attacks\n",
    "\n",
    "\n",
    "\n",
    "# GPU check\n",
    "device = 'cuda' if torch.cuda.is_available() else 'cpu'\n",
    "if device =='cuda':\n",
    "    print(\"Run on GPU...\")\n",
    "else:\n",
    "    print(\"Run on CPU...\")\n",
    "\n",
    "\n",
    "# Residual block\n",
    "class ResBlock(nn.Module):\n",
    "    def __init__(self, in_channels, out_channels, stride=1, downsample=None):\n",
    "        super(ResBlock, self).__init__()\n",
    "        self.conv1 = nn.Conv2d(in_channels, out_channels, kernel_size=3, stride=stride, padding=1, bias=False)\n",
    "        self.bn1 = nn.BatchNorm2d(out_channels)\n",
    "        self.conv2 = nn.Conv2d(out_channels, out_channels, kernel_size=3, stride=1, padding=1, bias=False)\n",
    "        self.bn2 = nn.BatchNorm2d(out_channels)\n",
    "        self.downsample = downsample\n",
    "\n",
    "    def forward(self,x):\n",
    "        identity = x\n",
    "        out = F.relu(self.bn1(self.conv1(x.to(device))))\n",
    "        out = self.bn2(self.conv2(out))\n",
    "        if self.downsample:\n",
    "            identity = self.downsample(x)\n",
    "        out += identity\n",
    "        out = F.relu(out)\n",
    "        return out\n",
    "\n",
    "\n",
    "# ResNet\n",
    "class ResNet(nn.Module):\n",
    "    def __init__(self, block, numBlocks, num_classes=10):\n",
    "        super(ResNet, self).__init__()\n",
    "        self.in_channels = 16\n",
    "        self.conv1   = nn.Conv2d(3, 16, kernel_size=3, stride=1, padding=1, bias=False)\n",
    "        self.bn1     = nn.BatchNorm2d(16)\n",
    "        self.layer1  = self.make_layer(block, 16, numBlocks[0], stride=1)\n",
    "        self.layer2  = self.make_layer(block, 32, numBlocks[1], stride=2)\n",
    "        self.layer3  = self.make_layer(block, 64, numBlocks[2], stride=2)\n",
    "        self.avgpool = nn.AvgPool2d(8)\n",
    "        self.fc1     = nn.Linear(64, num_classes)\n",
    "\n",
    "    def make_layer(self, block, out_channels, numBlocks, stride):\n",
    "        downsample = None\n",
    "        if (stride != 1) or (self.in_channels != out_channels):\n",
    "            downsample = nn.Sequential(\n",
    "                nn.Conv2d(self.in_channels, out_channels, kernel_size=1, stride=stride, bias=False),\n",
    "                nn.BatchNorm2d(out_channels))\n",
    "        layers = []\n",
    "        layers.append(block(self.in_channels, out_channels, stride, downsample))\n",
    "        self.in_channels = out_channels\n",
    "        for i in range(1, numBlocks):\n",
    "            layers.append(block(out_channels, out_channels))\n",
    "        return nn.Sequential(*layers)\n",
    "\n",
    "    def forward(self,x):\n",
    "        out = F.relu(self.bn1(self.conv1(x)))\n",
    "        out = self.layer1(out)\n",
    "        out = self.layer2(out)\n",
    "        out = self.layer3(out)\n",
    "        out = self.avgpool(out)\n",
    "        out = out.view(out.size(0),-1)\n",
    "        out = self.fc1(out)\n",
    "        return out\n",
    "\n",
    "def test_CIFAR10_adversarial(net,criterion,root_data_dir,batch_size,eps):\n",
    "    transform_test = transforms.Compose([\n",
    "    transforms.PILToTensor(),\n",
    "    transforms.ConvertImageDtype(torch.float),\n",
    "    transforms.Normalize((0.4914, 0.4822, 0.4465), (0.2023, 0.1994, 0.2010)),])\n",
    "\n",
    "    testset = torchvision.datasets.CIFAR10(root=root_data_dir, train=False, download=False, transform=transform_test)\n",
    "    testloader = torch.utils.data.DataLoader(testset, batch_size=batch_size, shuffle=False, num_workers=4)\n",
    "    classes = ('plane', 'car', 'bird', 'cat', 'deer', 'dog', 'frog', 'horse', 'ship', 'truck')\n",
    "\n",
    "    criterion = criterion\n",
    "\n",
    "    net.eval()\n",
    "    test_loss = 0\n",
    "    correct = 0\n",
    "    total = 0\n",
    "    for batch_idx, (inputs, targets) in enumerate(testloader):\n",
    "        inputs, targets = inputs.to(device), targets.to(device)\n",
    "        inputs  = attacks.FGSM_attack(model=net, device=device, dat=inputs, lbl=targets, eps=eps)\n",
    "        with torch.no_grad():\n",
    "            outputs = net(inputs)\n",
    "            loss = criterion(outputs, targets)\n",
    "            test_loss += loss.item()\n",
    "            _, predicted = outputs.max(1)\n",
    "            total += targets.size(0)\n",
    "            correct += predicted.eq(targets).sum().item()\n",
    "    num_val_steps = len(testloader)\n",
    "    val_acc = correct / total\n",
    "    #print(\"Test Loss=%.4f, Test accuracy=%.4f\" % (test_loss / (num_val_steps), val_acc))\n",
    "    return test_loss / (num_val_steps), val_acc\n",
    "\n",
    "def test_CIFAR10(net,criterion,root_data_dir,batch_size):\n",
    "    transform_test = transforms.Compose([\n",
    "    transforms.PILToTensor(),\n",
    "    transforms.ConvertImageDtype(torch.float),\n",
    "    transforms.Normalize((0.4914, 0.4822, 0.4465), (0.2023, 0.1994, 0.2010)),])\n",
    "\n",
    "    testset = torchvision.datasets.CIFAR10(root=root_data_dir, train=False, download=False, transform=transform_test)\n",
    "    testloader = torch.utils.data.DataLoader(testset, batch_size=batch_size, shuffle=False, num_workers=4)\n",
    "    classes = ('plane', 'car', 'bird', 'cat', 'deer', 'dog', 'frog', 'horse', 'ship', 'truck')\n",
    "\n",
    "    criterion = criterion\n",
    "\n",
    "    net.eval()\n",
    "    test_loss = 0\n",
    "    correct = 0\n",
    "    total = 0\n",
    "    for batch_idx, (inputs, targets) in enumerate(testloader):\n",
    "        inputs, targets = inputs.to(device), targets.to(device)\n",
    "        with torch.no_grad():\n",
    "            outputs = net(inputs)\n",
    "            loss = criterion(outputs, targets)\n",
    "            test_loss += loss.item()\n",
    "            _, predicted = outputs.max(1)\n",
    "            total += targets.size(0)\n",
    "            correct += predicted.eq(targets).sum().item()\n",
    "    num_val_steps = len(testloader)\n",
    "    val_acc = correct / total\n",
    "    print(\"Test Loss=%.4f, Test accuracy=%.4f\" % (test_loss / (num_val_steps), val_acc))\n",
    "    return test_loss / (num_val_steps), val_acc\n",
    "# a few arguments, do NOT change these\n",
    "DATA_ROOT           = \"./data\"\n",
    "CORRUPTED_DATA_ROOT = \"/data/usr/ft42/nobackup/CIFAR_10_C/CIFAR-10-C/\"\n",
    "# the folder where the trained model is saved\n",
    "CHECKPOINT_FOLDER   = \"savedFiles/saved_models/\"\n",
    "# the folder where the figures are saved\n",
    "FIGURES_FOLDER      = \"savedFiles/saved_figures/\"\n",
    "# the folder where the csvs are saved\n",
    "CSVS_FOLDER         = \"savedFiles/saved_csvs/\"\n",
    "# name of the saved model\n",
    "\n",
    "\n",
    "\n",
    "# a few arguments, do NOT change these\n",
    "DATA_ROOT           = \"./data\"\n",
    "CORRUPTED_DATA_ROOT = \"/data/usr/ft42/nobackup/CIFAR_10_C/CIFAR-10-C/\"\n",
    "# the folder where the trained model is saved\n",
    "CHECKPOINT_FOLDER   = \"savedFiles/saved_models/\"\n",
    "# the folder where the figures are saved\n",
    "FIGURES_FOLDER      = \"savedFiles/saved_figures/\"\n",
    "# the folder where the csvs are saved\n",
    "CSVS_FOLDER         = \"savedFiles/saved_csvs/\"\n",
    "# name of the saved model\n",
    "\n",
    "\n",
    "# hyperparameters, do NOT change right now\n",
    "TRAIN_BATCH_SIZE = 256    # training batch size\n",
    "VAL_BATCH_SIZE   = 100    # validation batch size\n",
    "INITIAL_LR       = 0.1    # initial learning rate\n",
    "MOMENTUM         = 0.9    # momentum for optimizer\n",
    "REG              = 1e-3   # L2 regularization strength\n",
    "EPOCHS           = 200    # total number of training epochs\n",
    "DECAY_EPOCHS     = 20     # parameter for LR schedule (decay after XX epochs)\n",
    "DECAY            = 0.5    # parameter for LR schedule (decay multiplier)\n",
    "criterion = nn.CrossEntropyLoss().to(device)\n",
    "\n",
    "# start the training/validation process\n",
    "best_val_acc          = 0\n",
    "current_learning_rate = INITIAL_LR\n",
    "epochs                = np.linspace(1,EPOCHS,EPOCHS)\n",
    "\n",
    "    #-------------- Testingh---------------#\n",
    "test_data_list=[]\n",
    "test_acc_list =[]\n",
    "test_loss_list=[]\n",
    "\n",
    "\n",
    "net = ResNet(ResBlock, [3, 3, 3]).to(device)\n",
    "net = net.to(device)\n",
    "net.load_state_dict(torch.load('/data/usr/ss1305/spie23_emphysemaSegmentation/sharedFolder/ECE661/Project/savedFiles/saved_models/resnet20_original.pt',map_location='cuda:0'))\n",
    "\n",
    "\n",
    "EPS_Test = [0.0,0.01,0.02,0.03,0.04,0.05,0.06,0.07,0.08,0.09,0.10]\n",
    "noregularizer_FGSM_acc_list = []\n",
    "test_CIFAR10(net,criterion,root_data_dir=DATA_ROOT,batch_size=VAL_BATCH_SIZE)\n",
    "for roi_EPS in EPS_Test:\n",
    "    test_loss,test_acc=test_CIFAR10_adversarial(net,criterion,root_data_dir=DATA_ROOT,batch_size=VAL_BATCH_SIZE,eps=roi_EPS)\n",
    "    noregularizer_FGSM_acc_list.append(test_acc)\n",
    "    print(\"Attack Epsilon: {}; Whitebox Accuracy: {}\".format(roi_EPS, test_acc))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "c86f9d3f",
   "metadata": {},
   "source": [
    "# Cutout=1,2 and Mixup=0.2 "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "b752e9fe",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-12-15T09:12:20.364792Z",
     "start_time": "2022-12-15T09:11:53.898332Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Run on GPU...\n",
      "Test Loss=0.3196, Test accuracy=0.9152\n",
      "Attack Epsilon: 0.0; Whitebox Accuracy: 0.9152\n",
      "Attack Epsilon: 0.01; Whitebox Accuracy: 0.6547\n",
      "Attack Epsilon: 0.02; Whitebox Accuracy: 0.5205\n",
      "Attack Epsilon: 0.03; Whitebox Accuracy: 0.4628\n",
      "Attack Epsilon: 0.04; Whitebox Accuracy: 0.4324\n",
      "Attack Epsilon: 0.05; Whitebox Accuracy: 0.4106\n",
      "Attack Epsilon: 0.06; Whitebox Accuracy: 0.3976\n",
      "Attack Epsilon: 0.07; Whitebox Accuracy: 0.3839\n",
      "Attack Epsilon: 0.08; Whitebox Accuracy: 0.373\n",
      "Attack Epsilon: 0.09; Whitebox Accuracy: 0.3638\n",
      "Attack Epsilon: 0.1; Whitebox Accuracy: 0.3574\n"
     ]
    }
   ],
   "source": [
    "# import necessary dependencies\n",
    "import argparse\n",
    "import os, sys\n",
    "import time\n",
    "import datetime\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.nn.functional as F\n",
    "import torchvision\n",
    "import torchvision.transforms as transforms\n",
    "import torch.nn as nn\n",
    "import torch.optim as optim\n",
    "from tqdm import tqdm_notebook as tqdm\n",
    "from torchvision.transforms import InterpolationMode\n",
    "from torch.utils.data import DataLoader\n",
    "from torchvision import datasets\n",
    "from torchsummary import summary\n",
    "from torch.autograd import Variable\n",
    "import PIL\n",
    "from PIL import Image\n",
    "from torch.utils.data import Subset\n",
    "from torch.utils.tensorboard import SummaryWriter\n",
    "import random\n",
    "import pandas as pd\n",
    "random.seed(10)\n",
    "import warnings\n",
    "warnings.filterwarnings(\"ignore\", category=DeprecationWarning)\n",
    "warnings.simplefilter('ignore')\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.nn.functional as F\n",
    "import attacks\n",
    "\n",
    "\n",
    "\n",
    "# GPU check\n",
    "device = 'cuda' if torch.cuda.is_available() else 'cpu'\n",
    "if device =='cuda':\n",
    "    print(\"Run on GPU...\")\n",
    "else:\n",
    "    print(\"Run on CPU...\")\n",
    "\n",
    "\n",
    "# Residual block\n",
    "class ResBlock(nn.Module):\n",
    "    def __init__(self, in_channels, out_channels, stride=1, downsample=None):\n",
    "        super(ResBlock, self).__init__()\n",
    "        self.conv1 = nn.Conv2d(in_channels, out_channels, kernel_size=3, stride=stride, padding=1, bias=False)\n",
    "        self.bn1 = nn.BatchNorm2d(out_channels)\n",
    "        self.conv2 = nn.Conv2d(out_channels, out_channels, kernel_size=3, stride=1, padding=1, bias=False)\n",
    "        self.bn2 = nn.BatchNorm2d(out_channels)\n",
    "        self.downsample = downsample\n",
    "\n",
    "    def forward(self,x):\n",
    "        identity = x\n",
    "        out = F.relu(self.bn1(self.conv1(x.to(device))))\n",
    "        out = self.bn2(self.conv2(out))\n",
    "        if self.downsample:\n",
    "            identity = self.downsample(x)\n",
    "        out += identity\n",
    "        out = F.relu(out)\n",
    "        return out\n",
    "\n",
    "\n",
    "# ResNet\n",
    "class ResNet(nn.Module):\n",
    "    def __init__(self, block, numBlocks, num_classes=10):\n",
    "        super(ResNet, self).__init__()\n",
    "        self.in_channels = 16\n",
    "        self.conv1   = nn.Conv2d(3, 16, kernel_size=3, stride=1, padding=1, bias=False)\n",
    "        self.bn1     = nn.BatchNorm2d(16)\n",
    "        self.layer1  = self.make_layer(block, 16, numBlocks[0], stride=1)\n",
    "        self.layer2  = self.make_layer(block, 32, numBlocks[1], stride=2)\n",
    "        self.layer3  = self.make_layer(block, 64, numBlocks[2], stride=2)\n",
    "        self.avgpool = nn.AvgPool2d(8)\n",
    "        self.fc1     = nn.Linear(64, num_classes)\n",
    "\n",
    "    def make_layer(self, block, out_channels, numBlocks, stride):\n",
    "        downsample = None\n",
    "        if (stride != 1) or (self.in_channels != out_channels):\n",
    "            downsample = nn.Sequential(\n",
    "                nn.Conv2d(self.in_channels, out_channels, kernel_size=1, stride=stride, bias=False),\n",
    "                nn.BatchNorm2d(out_channels))\n",
    "        layers = []\n",
    "        layers.append(block(self.in_channels, out_channels, stride, downsample))\n",
    "        self.in_channels = out_channels\n",
    "        for i in range(1, numBlocks):\n",
    "            layers.append(block(out_channels, out_channels))\n",
    "        return nn.Sequential(*layers)\n",
    "\n",
    "    def forward(self,x):\n",
    "        out = F.relu(self.bn1(self.conv1(x)))\n",
    "        out = self.layer1(out)\n",
    "        out = self.layer2(out)\n",
    "        out = self.layer3(out)\n",
    "        out = self.avgpool(out)\n",
    "        out = out.view(out.size(0),-1)\n",
    "        out = self.fc1(out)\n",
    "        return out\n",
    "\n",
    "def test_CIFAR10_adversarial(net,criterion,root_data_dir,batch_size,eps):\n",
    "    transform_test = transforms.Compose([\n",
    "    transforms.PILToTensor(),\n",
    "    transforms.ConvertImageDtype(torch.float),\n",
    "    transforms.Normalize((0.4914, 0.4822, 0.4465), (0.2023, 0.1994, 0.2010)),])\n",
    "\n",
    "    testset = torchvision.datasets.CIFAR10(root=root_data_dir, train=False, download=False, transform=transform_test)\n",
    "    testloader = torch.utils.data.DataLoader(testset, batch_size=batch_size, shuffle=False, num_workers=4)\n",
    "    classes = ('plane', 'car', 'bird', 'cat', 'deer', 'dog', 'frog', 'horse', 'ship', 'truck')\n",
    "\n",
    "    criterion = criterion\n",
    "\n",
    "    net.eval()\n",
    "    test_loss = 0\n",
    "    correct = 0\n",
    "    total = 0\n",
    "    for batch_idx, (inputs, targets) in enumerate(testloader):\n",
    "        inputs, targets = inputs.to(device), targets.to(device)\n",
    "        inputs  = attacks.FGSM_attack(model=net, device=device, dat=inputs, lbl=targets, eps=eps)\n",
    "        with torch.no_grad():\n",
    "            outputs = net(inputs)\n",
    "            loss = criterion(outputs, targets)\n",
    "            test_loss += loss.item()\n",
    "            _, predicted = outputs.max(1)\n",
    "            total += targets.size(0)\n",
    "            correct += predicted.eq(targets).sum().item()\n",
    "    num_val_steps = len(testloader)\n",
    "    val_acc = correct / total\n",
    "    #print(\"Test Loss=%.4f, Test accuracy=%.4f\" % (test_loss / (num_val_steps), val_acc))\n",
    "    return test_loss / (num_val_steps), val_acc\n",
    "\n",
    "def test_CIFAR10(net,criterion,root_data_dir,batch_size):\n",
    "    transform_test = transforms.Compose([\n",
    "    transforms.PILToTensor(),\n",
    "    transforms.ConvertImageDtype(torch.float),\n",
    "    transforms.Normalize((0.4914, 0.4822, 0.4465), (0.2023, 0.1994, 0.2010)),])\n",
    "\n",
    "    testset = torchvision.datasets.CIFAR10(root=root_data_dir, train=False, download=False, transform=transform_test)\n",
    "    testloader = torch.utils.data.DataLoader(testset, batch_size=batch_size, shuffle=False, num_workers=4)\n",
    "    classes = ('plane', 'car', 'bird', 'cat', 'deer', 'dog', 'frog', 'horse', 'ship', 'truck')\n",
    "\n",
    "    criterion = criterion\n",
    "\n",
    "    net.eval()\n",
    "    test_loss = 0\n",
    "    correct = 0\n",
    "    total = 0\n",
    "    for batch_idx, (inputs, targets) in enumerate(testloader):\n",
    "        inputs, targets = inputs.to(device), targets.to(device)\n",
    "        with torch.no_grad():\n",
    "            outputs = net(inputs)\n",
    "            loss = criterion(outputs, targets)\n",
    "            test_loss += loss.item()\n",
    "            _, predicted = outputs.max(1)\n",
    "            total += targets.size(0)\n",
    "            correct += predicted.eq(targets).sum().item()\n",
    "    num_val_steps = len(testloader)\n",
    "    val_acc = correct / total\n",
    "    print(\"Test Loss=%.4f, Test accuracy=%.4f\" % (test_loss / (num_val_steps), val_acc))\n",
    "    return test_loss / (num_val_steps), val_acc\n",
    "# a few arguments, do NOT change these\n",
    "DATA_ROOT           = \"./data\"\n",
    "CORRUPTED_DATA_ROOT = \"/data/usr/ft42/nobackup/CIFAR_10_C/CIFAR-10-C/\"\n",
    "# the folder where the trained model is saved\n",
    "CHECKPOINT_FOLDER   = \"savedFiles/saved_models/\"\n",
    "# the folder where the figures are saved\n",
    "FIGURES_FOLDER      = \"savedFiles/saved_figures/\"\n",
    "# the folder where the csvs are saved\n",
    "CSVS_FOLDER         = \"savedFiles/saved_csvs/\"\n",
    "# name of the saved model\n",
    "\n",
    "\n",
    "\n",
    "# a few arguments, do NOT change these\n",
    "DATA_ROOT           = \"./data\"\n",
    "CORRUPTED_DATA_ROOT = \"/data/usr/ft42/nobackup/CIFAR_10_C/CIFAR-10-C/\"\n",
    "# the folder where the trained model is saved\n",
    "CHECKPOINT_FOLDER   = \"savedFiles/saved_models/\"\n",
    "# the folder where the figures are saved\n",
    "FIGURES_FOLDER      = \"savedFiles/saved_figures/\"\n",
    "# the folder where the csvs are saved\n",
    "CSVS_FOLDER         = \"savedFiles/saved_csvs/\"\n",
    "# name of the saved model\n",
    "\n",
    "\n",
    "# hyperparameters, do NOT change right now\n",
    "TRAIN_BATCH_SIZE = 256    # training batch size\n",
    "VAL_BATCH_SIZE   = 100    # validation batch size\n",
    "INITIAL_LR       = 0.1    # initial learning rate\n",
    "MOMENTUM         = 0.9    # momentum for optimizer\n",
    "REG              = 1e-3   # L2 regularization strength\n",
    "EPOCHS           = 200    # total number of training epochs\n",
    "DECAY_EPOCHS     = 20     # parameter for LR schedule (decay after XX epochs)\n",
    "DECAY            = 0.5    # parameter for LR schedule (decay multiplier)\n",
    "criterion = nn.CrossEntropyLoss().to(device)\n",
    "\n",
    "# start the training/validation process\n",
    "best_val_acc          = 0\n",
    "current_learning_rate = INITIAL_LR\n",
    "epochs                = np.linspace(1,EPOCHS,EPOCHS)\n",
    "\n",
    "    #-------------- Testingh---------------#\n",
    "test_data_list=[]\n",
    "test_acc_list =[]\n",
    "test_loss_list=[]\n",
    "\n",
    "\n",
    "net = ResNet(ResBlock, [3, 3, 3]).to(device)\n",
    "net = net.to(device)\n",
    "net.load_state_dict(torch.load('/data/usr/ft42/nobackup/Project/savedFiles/saved_models/resnet20_mixup02_cutout12_0.2.pt',map_location='cuda:0'))\n",
    "\n",
    "\n",
    "EPS_Test = [0.0,0.01,0.02,0.03,0.04,0.05,0.06,0.07,0.08,0.09,0.10]\n",
    "cutout_mixup_FGSM_acc_list = []\n",
    "test_CIFAR10(net,criterion,root_data_dir=DATA_ROOT,batch_size=VAL_BATCH_SIZE)\n",
    "for roi_EPS in EPS_Test:\n",
    "    test_loss,test_acc=test_CIFAR10_adversarial(net,criterion,root_data_dir=DATA_ROOT,batch_size=VAL_BATCH_SIZE,eps=roi_EPS)\n",
    "    cutout_mixup_FGSM_acc_list.append(test_acc)\n",
    "    print(\"Attack Epsilon: {}; Whitebox Accuracy: {}\".format(roi_EPS, test_acc))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "6770c52f",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-12-15T09:19:22.546003Z",
     "start_time": "2022-12-15T09:19:22.528313Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>epsilon</th>\n",
       "      <th>original_acc</th>\n",
       "      <th>cutout_acc</th>\n",
       "      <th>mixup_acc</th>\n",
       "      <th>aux_rot_acc</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.00</td>\n",
       "      <td>0.9183</td>\n",
       "      <td>0.9169</td>\n",
       "      <td>0.9169</td>\n",
       "      <td>0.9231</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.01</td>\n",
       "      <td>0.6244</td>\n",
       "      <td>0.6233</td>\n",
       "      <td>0.6553</td>\n",
       "      <td>0.7105</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.02</td>\n",
       "      <td>0.4329</td>\n",
       "      <td>0.4187</td>\n",
       "      <td>0.5271</td>\n",
       "      <td>0.6019</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.03</td>\n",
       "      <td>0.3336</td>\n",
       "      <td>0.3170</td>\n",
       "      <td>0.4696</td>\n",
       "      <td>0.5198</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0.04</td>\n",
       "      <td>0.2748</td>\n",
       "      <td>0.2659</td>\n",
       "      <td>0.4307</td>\n",
       "      <td>0.4590</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>0.05</td>\n",
       "      <td>0.2405</td>\n",
       "      <td>0.2308</td>\n",
       "      <td>0.4070</td>\n",
       "      <td>0.4100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>0.06</td>\n",
       "      <td>0.2145</td>\n",
       "      <td>0.2111</td>\n",
       "      <td>0.3892</td>\n",
       "      <td>0.3739</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>0.07</td>\n",
       "      <td>0.1962</td>\n",
       "      <td>0.1948</td>\n",
       "      <td>0.3770</td>\n",
       "      <td>0.3441</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>0.08</td>\n",
       "      <td>0.1840</td>\n",
       "      <td>0.1838</td>\n",
       "      <td>0.3671</td>\n",
       "      <td>0.3187</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>0.09</td>\n",
       "      <td>0.1725</td>\n",
       "      <td>0.1743</td>\n",
       "      <td>0.3581</td>\n",
       "      <td>0.2985</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>0.10</td>\n",
       "      <td>0.1621</td>\n",
       "      <td>0.1693</td>\n",
       "      <td>0.3511</td>\n",
       "      <td>0.2792</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    epsilon  original_acc  cutout_acc  mixup_acc  aux_rot_acc\n",
       "0      0.00        0.9183      0.9169     0.9169       0.9231\n",
       "1      0.01        0.6244      0.6233     0.6553       0.7105\n",
       "2      0.02        0.4329      0.4187     0.5271       0.6019\n",
       "3      0.03        0.3336      0.3170     0.4696       0.5198\n",
       "4      0.04        0.2748      0.2659     0.4307       0.4590\n",
       "5      0.05        0.2405      0.2308     0.4070       0.4100\n",
       "6      0.06        0.2145      0.2111     0.3892       0.3739\n",
       "7      0.07        0.1962      0.1948     0.3770       0.3441\n",
       "8      0.08        0.1840      0.1838     0.3671       0.3187\n",
       "9      0.09        0.1725      0.1743     0.3581       0.2985\n",
       "10     0.10        0.1621      0.1693     0.3511       0.2792"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "Attack_info_df=pd.DataFrame(list(zip(EPS_Test,noregularizer_FGSM_acc_list,Cutout_FGSM_acc_list,\n",
    "                                     Mixup_FGSM_acc_list,Auxiliary_Rotation_FGSM_acc_list)),\n",
    "                            columns=['epsilon','original_acc','cutout_acc','mixup_acc','aux_rot_acc'])\n",
    "Attack_info_df.to_csv('Whitebox_Adversarial_Attack_Accuracy_final.csv',index=False,encoding='utf-8')\n",
    "Attack_info_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "864433e0",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-12-15T09:20:00.660052Z",
     "start_time": "2022-12-15T09:20:00.264060Z"
    }
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAjQAAAI0CAYAAAAKi7MDAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8/fFQqAAAACXBIWXMAAAsTAAALEwEAmpwYAADCH0lEQVR4nOzdd3wVVdrA8d9JryQkgdAJvfcOggiIFBFQF0WKgMpaXnft69rb6qprW9cCKiAgICpVQKmhI0VqKCG0EEJJAgnp9bx/zGVyExJIyC1JeL588mHOmTMzz53c3Dw5c2aO0lojhBBCCFGRuTg7ACGEEEKIspKERgghhBAVniQ0QgghhKjwJKERQgghRIUnCY0QQgghKjxJaIQQQghR4UlCIwBQSr2hlIq6Tpu+SimtlKpjg+OFK6W+Let+HKkk56gisHwPx5aifZhlm1vsGZc9lPa1ipuHUmqGUmq1VdluP99KKRel1G6l1L2l2MZfKXVOKdXOHjFVRpLQVCJKqUlKqWyllH+h+r3XqJ9WikNsAWoCsZbtb7H8wggra+zOoJRaoZTKVUoNdXYsDlYT+NleO1dKHbSc11ZFrItSSr1RqK5cvo+UUvdZXsdPzo7F2SyfFfOLWVdLKZWjlHrY0XGV0d+BvzjoWBMBBfxS0g201snAx8BH9gqqspGEpnJZA7gBfa5UKKWqAa2Bs0XUtwFWU0Ja6yyt9TmtdZ7NInYSyy/PvsB/gMlODaYQpZS7nfbrAWD5HmbY6Rh9gCDgO8rZeb0BfwXeB+5USlV3djCWv/JdnXT4qcBwy+dGYROBNGCuY0MqG611ktb6koMO9zQwVZf+SbYzgFuVUq1tH1LlIwlNJaK1PgUcA/pbVfcDDgCLi6hXGEmQSSk1XCl1WCmVarks1MRqnXnJyZIQbLSsOmGpD7dqe79Sao9SKkMpdVIp9bFSyrdQyC5KqX8rpeKVUpeVUlOVUl5W+3C3rD+jlMqy/OX/gGWdUkotU0rtuJIAWD7wVyulNpTgg/9hYDnwGXCHUqp2ofPgpZT6SimVpJS6pJT6CvC0Wn+75a/3OoW2u08plaaUqmIph1q6tuOUUslKqc2WX/qFz+lQpdQmpVQG8LBSqopSarqlyzlTKXVaKfVxoeOHK6UuWmJcr5TqWigWrZT6m1JqjlIqCZhlVT/Wqt3fLd+rFMvx5imlal7n/BVnMvADRkIzrtD3MxxoBLxuieFKr0yR7yOlVEdLL9oFS2w7lFKDCr1GN6XU60qpY5bzdEYp9XlxwSmlxlq+D9f8y9zyvu+J8ddxOMYv7cJtOimlfrO8d1OUUtuVUt2s1g9QSm20vB+ufI8aWdYVuNxhFZu2Kr+hjB6t+5RSh4EsoGlZz4vl2CuLeD1rlVLfFXNKZgM5wIOFtlHAQ8APWutUS93DSqlDyvjZv2j5eSzVZWqlVGOl1C9KqUTLz99KpVQbq/UTlNErNEApFWE51h9KqfZWba73M3TV96CIOB5UxudOllIqRin1jlLKzWp9uFLqW6XUq5bjXFRKzVRK+Vm1aQ+0AhYV2re3Uup9ZXw+ZimlYpVSr1i30VpfwOgZl8umJaG1lq9K9AVMAfZalacCnwB3F1G/36r8BpAK/AZ0AtoBu4CNVm36AhqoA7gCd1nKXYAaQJCl3QTgEjAOaIjRM7QPmGW1r3DgMvAN0AIYBlwAPrFq8yGQgNEt3BR4CcgD+lvWV8O4/PUfS/llS/u61zlHbpbthlnKK4DXCrX5xBLPcKA5Rk/OZSDKst4FiAH+UWi75cAcy7I3cBCjm7kz0NgSYybQotA5PWw5Bw0s5/e/wF6gG1AP45frI1bHGQmMApphfFh+C1wEgq3aaMv5+D+MRKKJVf1Yq3Z/BwZYjt0D4wN0vdX6MMs2t1znvAYBGUAbS/kQML7Q+hOWc1nD8nWt91FfjPdSK8v3/x0sv9St9vm95fs0zvIauwNPFzoHYy3LLwCJwG0l+Dn6EPjFsnwfEAUoq/WtMH5e5lq+t02A0UAPy/oBQC7wKcbPUnOMX/zNLetnAKsLHXMsoAv9TKYB6y3vg6aAf1nPi+V7nAc0sGrf2FLX7RrnZBpwuFDd7ZZz3MFS7oSR+IwH6mP0Aj8M1CnFZ1gocA74yrJ9M+BzjPdyNavPmDzgT+BWoC3wK3AG8La0ud7PUIHvgeV8R1mVh1q+h/+0nOf7MD7X3i70OZaI8XnRHBiI8XNo3ebvQEwRr3MVsM2yTZjltbYpot0HwB8lPX8385fTA5AvG39DjV9yeUCIpRyF8Qsj2PJBY13/idV2b1jWV7Oqu8+yLy9Lua/lw6uOpXyLpRxWKIaTwKOF6vpY2la1lMMt7Vyt2kzG+IXoC/hg/OJ/vNB+FgJrrcq3WeJ+HcgGRpTgHI20fGC6Wcr3A6cAF0vZ1xLHI4W221noA+/fwAGrcqglljss5QkYSY9bof2sBT4tdE7HFWqzGJhRiu+7i+XDdoxVnQa+K6JtgYSmiPUdLG1qW8phlCyheRrYZVV+EdhUqE0U8EahuiLfR8UcYy/wsmW5sWW7e6/RXmP8cv0MI4ltV4JjeGAkA1cSXi+MX1oDrNrMssTiUsw+NgK/XuMYMyhZQpMH1LPDedkHvGNVfg+rP3iK2aabZb99rOp+BHYU+tlKAqqU9L1bxHHeALYVqlMYvc9PWcoTLLH0t2pTFUgBHrKUr/kzVPh7wNUJzUZgfqFt/g6kAx6Wcnjh84aRiG21Kn9KoYQE43MrE0vyfp3z8Tcg7kbP5830JZecKp+1lv/7KaXqY/wyWq+1TsC49HSlvhGFLjcBsVrrOOsyxgdJiccPKOMae33gY0t3eIpSKgWjFwSMD9srtmutc63KmzEu6zSytPMANhQ6xHqMv0wB0Fqvw7gs8AbwrdZ6UQnCnIzRRZ5jKS8CAoDBlnIjSxxbCm23qVD5e6CVUqqjpTwG4xfhlW7sKz0OiYXORW+Mv+itbS9U/hK4Vyl1QCn1mVJqsFLK/HlVSjVQSs2yXJK4jNF7FIBx7q+136so47LX75Yu+WSr11l4X9fzCMYviStmAz1UEYODS0IpVU0p9aUyLoEmWs5dK6u4rpz3qy6fFPIO8ADQU2u9twSHHomRSKwA0MZ4ox8xxtRc0QlYo4sfT9apBHGVxHmtdbR1hY3OyxRgolLK1XIJZQJGb2mxtNZ/YCRCj1jiCAFGWPZ1xSrgOMblw3lKqcmWdqXRBehU6GcmGeOzrPDPzVar+C5h9Apeeb9d82eoBFpR9OePF8ZnxBWF31OxGH/cXOGN8QeStS6AOxBt/TqVUl8XEUeGZR/iOtyu30RUJFrreKXUXozxMn7An1rrJMvqdVb1ORg/nNayCu/O8n9pPgSutP275XiFxZRiX9eljLEyvTC6hhsppZS2/FlTTPswjC7egUqpv1utcsVIdJaV9Nha60NKqZ0YPQB/Wv6fbZWkuWB8wI4sYvO0QuXUQvv+XSlVD7gDoxdnNrBfKdXfsv9fgXjgCeA0xvduE0YSWOx+C7McYzlGj8Nbln3WwUjKCu/rWvvpg3Hp8BOl1CdWq1wwzuvfi9zw2mZgXCp4AeNSVTowrzRxWazG6G28H6NX7Xr+ipHEZxhDRAAjsc9VSlXXxriGssqz7NNaUYPBi/r+zaDs52UWxoDnoRjfowCM99j1TAX+o5T6G8Z4mkysBgNrrVOUUp0xfiYHAI8CH1jet7tKGJsLxh9b/1fEuqQi6opUgp8hWynqc9P6MzMO43JXYWnk94ZecbmIdkGWfYjrkB6aymkNRuLSn/weG8hPaPpjdIEml/E4V36QzQG4WuvzGL9gm2mto4r4sv5LpYsqOHi3J8YH5DGMSxOZWN2ZZXErRk/TFW9g9Ob0ArpifMhfy8MYSUY7oL3V1/3AUGUMDj5meW2FP4R6FbG/74HRll6adsBMq3U7McYQXS7iPMReJ0601he11nO11n/F+MVzK9BSKRUMtAT+rbX+XWt9EOOvuBu5E6cLxl9/T2mtN2utj1Dwr8uSmozx13n7Ql/PUHBwcBZW7xerOoqo7wN8qbVeorXej3GnXkOr9X9a/h94ndjWAkOAV5RSr16roTIGA/fFGHNm/TraYVyWvDI4eBfQ/xp/8e+6TlwXgFqF6joW1bAIZT4vWuvLGEnQI5avn7TWiSU49myMZGwchQYDW+07V2u9QWv9GkZP1VmMHrKS2onROxJTxM9N4V/s3a8sKKUCMZLqg1axFPkzVMI4Iij68ycd4zOipP4EGquCdy/uwLi0HVbo9RWVLLfBOCfiepx9zUu+bP+FcelEY2T7A63qAzF6Mi4Dbxba5g2srh9b6gqMbeDqMTShlv09ifHLNMBSPw7jl9TLGLeMN8PSNW2173BLHF9jfAgNxRjX8plVmw+49qDgWzHGzVwZszLKctyuxZyXK4OBXy1incLoPXrNUv4MOI8x/qiZJRZzULDVdsGWY+7GavyIZZ0XRvK1g/yBf90wBhmOKOqcWm37L4xfqs0wutk/x+h2D8D4Q+QCsMByXnpgXO9PxWp8CsWMlbGuxxhMmQe8gjEoeATGAGUN9LW0CeMaY2jIHww8roh1/pZ14y3lZRgJRj0gxPJainsf7cTodWqDkVQswfgLfYbV/mdbzsVYjMsAXYC/F/Nae1m+h29d42fnw8LfY6t172IZHGyJ6cqtyp0tx/4L+YOCB5I/KLit5fs4ASPRB6P3QmP0sDXCSCrOc/UYmqtiscV5sbTpgtFTmwP0KsXnyzSMga8ay2Bgq3XDMcZSdbJ8j0dScFxLbcv7a+Q19h+K8XP6O8bl2TCMz6J/YVw2hPxBwTsxko42lvNwFvC53s+QZf0Mrj2GZojle/gixs/ZKIoeFPxtofhfAU5alatiJEG9C7VbRX6y1xAjietVqI3C+ANxQkm/Pzfzl9MDkC87fFONzD8Lo4fDp9C6nZYPosI/XFd9eHKdhMZS9wLGnQW5QLhV/QiM69tpGL9E9mB1J5Hlg2Aa+XcyJWPcqeNt1cYd4xLBGcvrOQg8YFkXZPlB/0+hmKdi/PXkX8R5GWmJv1kx5+0TLIODMXotpmD8okiy7Pe9wufIst1Cy37/XsS6YIxBgldewxlL+w7FnVNL/asYyVCK5fjrsUooMJK5vRjJwhHgHgoNuKUECY2lfOWyVTrGL8pBlC6hedoSR5EDQS2vd5NluTPGX6zphd5bV72PMH5JbbG0PQk8jnH5aEah98jblvVZGEnpp9d4rd0wBvj+u4g4rwwGfq+Y19HOsr8BlnJXSzypGO/fbVgl0xiXOrZa4k/C6CFtaLX+ZctrTsFIjJ6gZAlNmc+LVbvdQEQpP1+uDA7eUcS6PhgJa5zlPXEUeNFq/ZX30oTrHKM+xu3/cRifY6cwkrQGlvUTMBKxgRg9rpkY48U6luJnaAbXSGgsdQ9a9n/lZ/dfWA3ypwQJjaVuOsZzaKzrvDEu+135Hp3l6rstb8NIonyudb7ky/hSlpMmhBDiJmK5BHIS+EBr/ZmTwykVpdQEjESiQowDVUo1xuipbaVLcLnZarvlGDd1vG+34CoRGUMjhBA3EWU8gLI6xqUUX4zeA2FHWusojMHmDUq6jTKmqtmK0XMsSqBCZLdCCCFsph7G3VFngUnaGCAs7ExrXeRcWNdon4xx2VCUkFxyEkIIIUSFJ5echBBCCFHhSUIjhBBCiAqv0oyhCQkJ0WFhYXbZd2pqKr6+hSeKFvYi59vx5Jw7lpxvx5Lz7Vj2PN+7du2K11pXK2pdpUlowsLC2LnTPg9TDA8Pp2/fvnbZt7ianG/Hk3PuWHK+HUvOt2PZ83wrpU4Vt04uOQkhhBCiwpOERgghhBAVniQ0QgghhKjwJKERQgghRIVXaQYFCyGEqJiys7OJiYkhIyPDLvsPCAjg0KFDdtm3uFpZz7eXlxd16tTB3d29VNtJQiOEEMKpYmJi8Pf3JywsDKWUzfefnJyMv7+/zfcrilaW8621JiEhgZiYGBo0KPHUV4BcchJCCOFkGRkZBAcH2yWZERWLUorg4OAb6q2ThEYIIYTTSTIjrrjR94IkNEIIIYSo8CShEUIIIWzs3Xff5eGHH7Z52+tRShEVFWWTfVU0MihYCCGEuI4ZM2bw0UcfcezYMapUqcLIkSN57733CAwMLLL9Sy+9VOJ9l6atKJ700AghhBDX8NFHH/GPf/yDDz/8kKSkJLZt28apU6e4/fbbycrKuqp9Tk6OE6IUktAIIYQQxbh8+TKvv/46n3/+OYMGDcLd3Z2wsDDmz5/PyZMnmT17Nm+88Qb33nsvY8eOpUqVKsyYMYM33niDsWPHmvuZOXMm9evXJzg4mLfffpuwsDBWr14NUKDtyZMnUUrx/fffU69ePUJCQvjXv/5l7mf79u306NGDwMBAatasyf/93/8VmVTdjOSSkxBCiPJljm3veLrmE1Ee0NfcdsuWLWRkZHD33XcXqPfz82PIkCGsWrWKZs2asXjxYn766SdmzpxJZmYm77//vtn24MGDPP744/z222907dqVl156iTNnzlzzuJs2beLIkSNERkbStWtX7r77blq0aIGrqyuffPIJnTt3JiYmhsGDB/Pll1/y1FNPXecsVH7SQyOEEEIUIz4+npCQENzcrv77v2bNmsTHxwPQo0cPRowYgYuLC97e3gXa/fzzzwwbNoxbbrkFDw8P3nrrrevemvz666/j7e1Nu3btaNeuHXv37gWgU6dOdO/eHTc3N8LCwvjrX//K+vXrbfRqKzbpoRFCCCGKERISQnx8PDk5OVclNWfPniUkJASAunXrFruP2NjYAut9fHwIDg6+5nFr1KhRoH1KSgoAkZGRPPPMM+zcuZO0tDRycnLo1KlTqV9XZSQJjRBCiPLlOpeBSqssj+Lv0aMHnp6eLFiwgFGjRpn1KSkprFixgnfffZeYmJhr9rjUrFmTI0eOmOX09HQSEhJuKJ7HHnuMDh06MHfuXPz9/fn000/5+eefb2hflY1cchJCCCGKERAQwOuvv86TTz7Jb7/9RnZ2NidPnmTUqFHUqVOHcePGXXcf9957L0uXLmXLli1kZWXxxhtvoPWNJW3JyclUqVIFPz8/Dh8+zFdffXVD+6mMJKG5jtTUVM6cOcOFCxecHYoQQggneOGFF3j33Xd57rnnqFKlCt26daNu3bqsWbMGT0/P627fqlUrPv/8c+6//35q1qyJn58f1atXL9G2hf3nP/9hzpw5+Pv788gjj3DffffdyEuqlNSNZonlTefOnfXOnTttus9t27axatUq8vLy6NatG4MGDbLp/kXRwsPD6du3r7PDuKnIOXcsOd8FHTp0iBYtWtht/+Vttu2UlBQCAwM5evRoqWeUrghscb6Le08opXZprTsXtY300FxD9erVycvLA2D//v3k5uY6OSIhhBAV0dKlS0lLSyM1NZXnnnuONm3aEBYW5uywKhVJaK4hrG4t/H09AEhLS+PYsWNOjkgIIURFtHjxYmrVqkWtWrU4evQo8+bNkxnGbUwSmmuI3PYNmYnJZnnfvn1OjEYIIURF9e2335KYmEhSUhJr1qyhWbNmzg6p0pHbtq9hzoqq7PnfKiJrRdKqVStcXV3JyMjAy8vL2aEJIYQQwor00FxDYnYUS5OXcuTIEQ4ePEhubi4HDx50dlhCCCGEKEQSmmt4dFL+7XDHjh0jJyfHfPy0EEIIIcoPSWiuoUXz5gQGhAKYD1OKjo7m0qVLTo5MCCGEENYkobkGpRSd+vYzy5GRkYAMDhZCCCHKG0loruPhRyeay0eOHEFrzb59+274sdVCCCGEsD1JaK7j7n634u5mTAWflJTEhQsXuHjxIjExMU6OTAghhKPMmTOHzp074+fnR82aNRk8eDCbNm267nZhYWGsXr3aJjGEh4dTp04dm+yrMnJYQqOUClJKLVRKpSqlTimlHiimXaBS6nul1AXL1xuOirEoHh4e1GvV3izLZSchhLi5fPzxxzz11FO89NJLnD9/nujoaB5//HEWL17s7NCEFUf20HwBZAGhwBjgK6VUqyLafQL4AGFAV2CcUmpiEe0cpuPg28zlKwnNgQMHyMnJcVZIQgghHCApKYnXXnuNL774grvvvhtfX1/c3d0ZNmwYH374IRMmTOCVV14x21v3oowbN47o6GiGDRuGn58fH3zwAQBLliyhVatWBAYG0rdvXw4dOmRur5QiKirKLF/Zf2pqKoMHDyY2NhY/Pz/8/PyIjY110FmoGByS0CilfIF7gFe11ila603AEqCoedeHAR9ordO01ieB74BJjoizOMN7dkdhPKL69OnTpKamkpGRwdGjR50ZlhBCVErKxl9V/P2LXXc9W7duJSMjg5EjR5b6dcyaNYt69eqxdOlSUlJSeOGFF4iMjGT06NF8+umnxMXFMWTIEIYNG0ZWVtY19+Xr68uKFSuoVasWKSkppKSkUKtWrVLHVJk5qoemKZCjtY60qtsLFNVDAwXfZwpoba/ASqK2vz8hdZub5SuJjFx2EkKIyi0hIYGQkBDc3GzzYP0ff/yRoUOHcvvtt+Pu7s5zzz1Heno6W7Zsscn+b2aOmvrAD7hcqC4JKGp+8d+AF5VSD2JcnpqEcQnqKkqpycBkgNDQUMLDw20VbwEpKSk0uq0bcTONbsHIyEjat2/PkSNHWLVqFe7u7nY57s0qJSXFbt9LUTQ5544l57uggIAAkpPz583Dv6hfDfZR4LhF8Pb2Jj4+nkuXLhWZ1GRnZ5OZmWnuJy0tDa21WdZak5aWZpZPnTpFzZo1Cxy3Vq1aREVF0alTJ8B4f1xZb73/wvsur3Jzc8scY0ZGRql/RhyV0KQAVQrVVQGKesV/Az4HjgIJwFxgdFE71VpPBaYCdO7cWfft29dG4RYUHh7O5KcfZ9vMGQBEHY0iJycHNzc3AgIC6Nq1q12Oe7MKDw/HXt9LUTQ5544l57ugQ4cO4W+VxNj6oRjJyckF9l/AdZKn/v374+npyZo1a7j33nuvWh8YGEhubq65/8uXL6OUMssuLi74+PiY5fr167N//36zrLUmNjaWxo0b4+/vj4+PDy4uLub6hIQEGjRogL+/P76+vgX2XV5d83yXkJeXFx06dCjVNo665BQJuCmlmljVtQMiCjfUWl/UWo/RWtfQWreyxLjdQXEWa3S7zvj4VwcgKzuL6OhoQC47CSFEZRYQEMBbb73FE088waJFi0hLSyM7O5sVK1bwwgsv0L59e5YvX87Fixc5d+4cn376aYHtQ0NDOX78uFkeNWoUy5YtY82aNWRnZ/PRRx/h6elJz549AWjfvj1z5swhNzeX3377jfXr1xfYV0JCAklJSQ557RWNQxIarXUqsAB4Synlq5TqBQwHZhVuq5RqpJQKVkq5KqUGY1xSescRcV6Ll1LUvqWXWT5y+AgAZ86cIT4+3llhCSGEsLNnn32Wjz/+mHfeeYdq1apRt25d/ve//zFixAjGjRtHu3btCAsLY+DAgdx3330Ftv3nP//JO++8Q2BgIP/5z39o1qwZs2fP5sknnyQkJISlS5eydOlSPDw8APjss89YunQpgYGB/PDDD4wYMcLcV/PmzRk9ejQNGzYkMDBQ7nIqxFGXnAAeB6YBFzAuJT2mtY5QSvUGVmit/SztOgGfAoEYPTtjtNZX9eQ4w60ThnF0xUIAIiMiGTR4EEop9u3bR79+/a6ztRBCiIpqzJgxjBkzpsh1P/74Y4Hy008/bS4PHz6c4cOHF1g/cuTIYu+a6ty5MxERxf/KmzZtWklDvuk47Dk0lktJI7TWvlrrelrrOZb6jVbJDFrr+VrrWlprH611e631746K8XomD/sLbq5eAFxKvWT2zMhUCEIIIYRzydQHpdDJ24/ANt3McuQh4y70pKQkTp065aywhBBCiJueJDSl4AI0HmX11OA9+Y/V2bt3rxMiEkIIIQRIQlNqf7n/Lq489y/6YjRpaWkAHDx4kOzsbCdGJoQQQty8JKEppbvD2uNf23hqsEZz8sgJALKysjhy5IgzQxNCCCFuWpLQlFKYUgQM7m+WD/2RP6mYXHYSQgghnEMSmhvQfXz+LdpHz0eRm5sLwLFjx0hJSXFWWEIIIcRNSxKaGzCqcy+8fUMAyNAZJJwybt/WWrN//35nhiaEEELclCShuQH9vKvjeUv+ZaeIjflJjEyFIIQQN5dHH32Ut99+29lh3PQkobkBwUC18QPN8oGTh3B1dQXg3LlznD9/3kmRCSGEsKWwsDA8PDyumuKmQ4cOKKU4efIkX3/9Na+++qqTIhRXSEJzg4YN6IyrqzH3RoJOQCfmPylYemmEEKLyaNCgAXPnzjXL+/fvNx/ZIcoPSWhu0KDAMPxb5j81eP/a/Duc9u3bR15enjPCEkIIYWPjxo1j5syZZvn7779n/PjxZnnChAm88sorALz//vt069aNnJwcAL766itatWpFRkYG4eHh1KlTp8C+w8LCWL16NQBvvPEG9957L/fddx/+/v507NhR7p4tBUdOTlmp3OJRhdz7R8D+jQDs3LuLLqO7kZaWRkpKCidOnKBRo0bODVIIISqgN9WbDjvW6/r167bp3r07s2bN4tChQzRt2pR58+axefNmM4mx9vzzz7Ns2TLeeecdxowZw0svvcTatWvx8vIqUTyLFy9m7ty5zJ49m88++4wRI0YQGRmJu7t7qV/bzUZ6aG6QN9D2L93N8qm8aKq6VDXLctlJCCEqjyu9NKtWraJFixbUrl27yHYuLi7MnDmT//73v9x111288MILdOjQocTH6dSpE/feey/u7u4888wzZGRksG3bNlu9jEpNEpoyGFKjOlVqNAUgjzz2/55/t9OhQ4fIzMx0VmhCCCFsaNy4ccyZM4cZM2YUuNxUlLCwMG677TZOnjzJE088Uarj1K1b11x2cXGhTp06xMbG3lDMNxu55FQGA7zr8MGQoTDNmKRy08aNPDTiYeLi4sjOzubQoUO0b9/euUEKIUQFU5LLQKWRnJyMv79/mfZRv359GjRowPLly/nuu++u2XbZsmVs3bqV/v378/zzzzNlyhQAfH19Cwwmzs3NJS4ursC2p0+fNpfz8vKIiYmhVq1aZYr9ZiE9NGXQyc2L3IdHmOVDGYepV7WeWZbLTkIIUXl89913rF27Fl9f32LbxMfH8/DDD/Ptt9/y/fffs3TpUpYvXw5A06ZNycjIYNmyZWRnZ/POO+9c1ZO/a9cuFixYQE5ODp9++imenp507969qEOJQiShKQNXYEDL6nh5BwGQTjqRqyLN9SdOnCApKclJ0QkhhLClRo0a0blz52u2mTx5MsOHD2fIkCEEBwfz3Xff8fDDD5OQkEBAQABffvklDz/8MLVr18bX1/equ56GDx/Ojz/+SNWqVZk1axYLFiyQAcElJJecyuh2V2/W9+xHxpqfAVi1YhUPfPAAx48fB4znFdxyyy3ODFEIIcQNOnnyZJH1bm5uaG08f2zGjBlm/YIFCwq0Gzx4cIExMBMmTGDChAlm+bnnnivQ3svLi9mzZ5ct6JuU9NCU0QCfOqQ8dL9Z3nVuF83qNzPL+/btM9/0QgghhLAPSWjKqImLK9UGdcHVxegSjCOO2M2xZhdhXFwcZ8+edWaIQgghRKUnCU0ZKWCgaxr+zbqYdYvmLaJFixZmWZ70KIQQ4nreeOMNudxUBpLQ2MAAt0Cy7xtpljcf2EzrFq3N8oEDB8jNzXVGaEIIIcRNQRIaG+jvHUrqpPvM8vG84yRH5D/3IC0tjWPHjjkrPCGEEKLSk4TGBmooRetghX81Y+6mPPL45dtfaNOmjdlGLjsJIYQQ9iMJjY0MyE7B9Y4hZnnVxlW0bdvWLB85coSMjAxnhCaEEEJUepLQ2Eh/r1AuPzraLEekRpB3Po+aNWsCxiOuIyIinBWeEEIIUalJQmMjt3pWhW6d8PQMACCNNJZMWVKgl0amQhBCCCHsQxIaG/EHemTE4NvtNrNu6ZKltG7dGqUUANHR0Vy6dMlJEQohhLgRmzZtomfPngQEBBAUFESvXr3YsWMHWVlZPPvss9SpUwc/Pz/CwsJ46qmnzO3CwsLw9vbGz8+PGjVqMGHCBFJSUsz1EyZMwMPDAz8/P4KCgrj99ts5fPiwE15h5SAJjQ0NyMshZeIos7wzZicumS40btzYrJNeGiGEqDguX77MnXfeyZNPPsnFixc5c+YMr7/+Op6enrz33nvs3LmT7du3k5ycTHh4OB07diyw/dKlS0lJSWHPnj3s3r2b9957r8D6F154gZSUFGJiYqhevXqBaRFE6UhCY0MDfOqQde8wXJQxRdZ5zrNh1oYCl5327t0rUyEIIUQFERlpTDg8evRoXF1d8fb2ZuDAgbRt25YdO3YwcuRIatWqhVKKsLAwxo8fX+R+atSowR133MGePXuKXO/j48MDDzzAgQMH7PVSKj2ZnNKGurn54OeVhnvjjlw6uh2AX2b9wpdPfImnpyeZmZlcunSJmJgY6tat6+RohRCifHrzzTcddqzXX3/9muubNm2Kq6srDz74IPfffz/du3enatWqAHTv3p2PP/4YDw8PevfuXWCIQWExMTGsWLGCfv36Fbk+JSWFH374gQ4dOpTtBd3EpIfGhtyBPmkxZP9lhFm3Yd8GXHChZcuWZp08k0YIISqGKlWqsGnTJpRSPPLII1SrVo277rqL8+fP889//pN//OMf/PDDD3Tu3JnatWvz/fffF9h+xIgR+Pv7U7duXapXr35Vsvaf//yHwMBAGjduTEpKSoGZu0XpSEJjYwOUGymP5N++fSznGIdXH6Zdu3ZmXUREBDk5Oc4ITwghRCm1aNGCGTNmEBMTw4EDB4iNjeWpp57C1dWVJ554gs2bN5OYmMjLL7/MpEmTOHTokLntokWLzPE1hw8fJj4+vsC+n3vuORITEzl37hxLliyhUaNGjn55lYZccrKxAb51wd8d/6Awki+eJJdcfvr6J15f9DqBgYEkJiaSkZHB0aNHC0xgKYQQwnC9y0CllZycPxVNWTVv3pwJEyYwZcqUAvXe3t488cQTvP766xw8ePCqz/dbb72VCRMm8Nxzz7Fo0SKbxCIKkh4aG2vt4k71rEu4DbjDrFu5fiVKqasGBwshhCjfDh8+zEcffURMTAwAp0+fZu7cuXTv3p1PP/2U8PBw0tPTycnJ4fvvvyc5ObnYcTBPPfUUq1atks9/O5GExsYUMCD9HJcfG2PW7U/aT9yRuAIJzdGjR0lLS3NChEIIIUrK39+fP/74g27duuHr60v37t1p3bo1H330ET4+Pjz77LPUqFGDkJAQvvjiC3755RcaNmxY5L6qVavG+PHjeeuttxz8Km4OcsnJDvq7+zOnd1M8PfzJzEomhRSWTlnKpI8nUadOHWJiYsjLy+PAgQN07drV2eEKIYQoRu3atZk/f36R6yZPnszkyZOL3fbkyZNX1X311VfmsgwAti3pobGDAd61wdUV3059zLolC5cAyFQIQgghhB1IQmMH9ZSiSfpZ0ibcZ9ZtP7mdjKQMWrVqhYuLcdrPnDlz1Yh3IYQQQpSeJDR2MiArkYz7h+OiXAE4y1m2zNuCj48PTZs2NdtJL40QQghRdpLQ2MkAzxCoUoWABu3Nup9m/ARQ4Jk0+/btk6kQhBBCiDKShMZObvOqhtJ55I64y6xbv2s9ebl5NGnSBG9vbwCSkpI4deqUs8IUQgghKgVJaOykKtAp7TSXH33ArDuafZSoDVG4urrSqlUrs16eSSCEEEKUjSQ0djQgJx2aNMYvsA4AOeTw01dXX3Y6ePAg2dnZTolRCCGEqAwkobGjAd61APC4Lf+pwb+v+R0wnm0QFBQEQFZWFocPH3Z8gEIIIUQlIQmNHfXyqIJXbgaX/5p/2WnPxT1cOnkJpdRVg4OFEEKI8uSHH35g4MCBzg6jRCShsSMv4Ja00+T074OHmy8AySSzbMoyoOBD9o4dO0ZKSoozwhRCCHENc+bMoXPnzvj5+VGzZk0GDx7Mpk2bSrRtWFgYq1evtkkc4eHh1KlTxyb7mjBhAkopFi9eXKD+6aefRillPsV4zJgxrFy50ibHtDdJaOxsAICbG34dept1i35eBEBgYCD169cHQGvN/v37HR+gEEKIYn388cc89dRTvPTSS5w/f57o6Ggef/zxqxKB8uDkyZOEhYWVuH3Tpk2ZOXOmWc7JyWH+/Pk0atTIDtHZnyQ0dtbfuy4AGePuNeu2HdtGVmoWIFMhCCFEeZWUlMRrr73GF198wd13342vry/u7u4MGzaMDz/8EDB6Ol555RVzG+telHHjxhEdHc2wYcPw8/Pjgw8+AGDJkiW0atWKwMBA+vbty6FDh8ztlVJERUWZ5Sv7T01NZfDgwcTGxuLn54efnx+xsbFlen3Dhg1j06ZNXLp0CYDffvuNtm3bUqNGDbPNjBkzuOWWWwDYsmULISEhnD59GjDu0K1atao5BrS42K3Py7vvvktISAhhYWH88MMPZYq/MElo7KyDmxdVsy+TNu4elOV0n9Fn+OOnPwBo2bIlbm7GHKHnzp3j/PnzTotVCCHKA6WUTb+qVKlS7Lpr2bp1KxkZGYwcOfKGXsesWbOoV68eS5cuJSUlhRdeeIHIyEhGjx7Np59+SlxcHEOGDGHYsGFkZWVdc1++vr6sWLGCWrVqkZKSQkpKCrVq1bqhuK7w8vJi+PDhzJs3D4CZM2cyfvz4Ytv37NmTv/71rzz44IOkp6czduxY3n77bZo3b16i4507d474+HjOnDnD999/z+TJkzly5EiZXoM1hyU0SqkgpdRCpVSqUuqUUuqBYtp5KqW+VkqdV0pdVEotVUrVdlSctuYK9EuLhcBAAuq1MevnTzNmb/Xy8qJZs2ZmvTyTRgghyoeEhARCQkLMPzpt4ccff2To0KHcfvvtuLu789xzz5Gens6WLVtsdozSGD9+PDNnziQxMZH169czYsSIa7Z/4403SEpKomvXrtSuXZsnnniiVMd7++238fT05NZbb2Xo0KHFzmR+IxzZQ/MFkAWEAmOAr5RSrYpo93egB9AWqAVcAj53VJD2MMDVy1i4a5hZt277OnPKA+u7nfbv309eXp5D4xNCCHG14OBg4uPjycnJsdk+Y2NjzbGTAC4uLtStW5czZ87c0P7mzJlDYGAggYGBtG3blujoaLMcGBhIdHT0Nbe/5ZZbiIuL41//+hd33nmn+RT74ri7uzNhwgQOHDjAs88+e91eLmtVq1bF19fXLNevX7/Ml82sOSShUUr5AvcAr2qtU7TWm4AlwLgimjcAftdan9daZwA/AkUlPhXGAB9jHE3i42PMusjMSE5uOwlAo0aNzG9ySkoKJ06ccHiMQghRXmitbfp1+fLlYtddS48ePfD09GTRokXFtvH19SUtLc0snzt3rsD6wr/wa9WqVWC6G601p0+fpnZt40KEj49PsfsrKnl44IEHSExMJDExkX379lGvXj2znJiYSL169a75GgHGjh3LRx99dM3LTVecOXOGN998k4kTJ/Lss8+SmZlprrtW7ACXLl0iNTXVLEdHR5f5spk12/WjXVtTIEdrHWlVtxe4tYi23wGfKaVqAYkYvTkritqpUmoyMBkgNDSU8PBwG4acLyUlpUz71kDdLs043aI5fv61SEmOJZts/vfa/xj2stFrExgYaH6jV65cSYsWLWwQecVU1vMtSk/OuWPJ+S4oICCA5ORku+0/Nzf3hvbv4uLCyy+/zOOPP052djb9+vXD3d2ddevWsXHjRnP8yOeff87f//53srOz+eijj9Bam8cLCQnh4MGDdOvWDYChQ4fy73//m6VLl9KrVy+++uorPDw8aNOmDcnJybRp04YZM2bw2muvsXbtWtavX2+u8/PzIyEhgZiYGAICAq6KNyUlpcCxryU7O5vMzEySk5OZOHEinTp1okOHDiQnJ5Obm0tGRgbJyclkZGSY509rzbhx4xg3bhxvvvkmI0eO5B//+Advv/02gBn7yy+/zIIFCwrEfiXR+ec//8nrr7/Ozp07+fXXX3nhhReKjDcjI6P0PyO2zoSLyYB7A+cK1T0ChBfRNgCYh5EH5AC7gaDrHaNTp07aXtatW1fmfUxKPKjRWgcPHa8tr033rdbXXB8bG6vfeOMN/cYbb+h33nlHZ2RklPmYFZUtzrcoHTnnjiXnu6CDBw/adf+XL18u0/azZ8/WnTp10j4+Pjo0NFQPGTJEb968WWutdXp6uh41apT29/fXbdq00R9//LGuXbu2ue2iRYt03bp1dUBAgP7www+11lovWLBAt2jRQlepUkX36dNHHzhwwGy/Y8cO3bJlS+3n56fHjh2r77//fv3yyy+b6ydOnKiDgoJ0QECAPnPmTIE4T5w4oevXr1+i1/Tggw8W2K+1Xr166enTp2uttZ4+fbru1auX1lrrTz/9VLdt21ZnZmZqrbU+c+aMDgkJ0Rs2bLhu7OvWrdO1a9fW77zzjg4ODtZ169bVM2fOLDa+4t4TwE5dTB6g9HW63GxBKdUB2Ky19rGqexboq7UeVqjtbMAXeAhIBV4A7tRad7vWMTp37qx37txp89jBuN2sb9++ZdrH3LSzPOBTE/elv5N91yAAAggg+kw0VWpVQWvN119/zYULFwAYPnw47du3L2PkFZMtzrcoHTnnjiXnu6BDhw7ZtVc6OTkZf39/u+1fFFTU+Q4PD2fs2LHExMSUaB/FvSeUUru01p2L2sZRg4IjATelVBOrunZARBFt2wMztNYXtdaZGAOCuyqlQuwfpv308zbu688e1A93N2PQVRJJrJhqXE1TSskzaYQQQogb5JCERmudCiwA3lJK+SqlegHDgVlFNN8BjFdKBSil3IHHgVitdbwjYrWXUKVom3oa3N2p0rqnWb9w/kJzuU2b/Nu6T5w4QVJSkkNjFEIIISoqR962/TjgDVwA5gKPaa0jlFK9lVLWkxg9B2QAR4E4YAhwY081KmcG5BgvM3NM/lODtxzZQk6GcUtglSpVaNiwoblOpkIQQghRGfTt27fEl5tulMMSGsslpBFaa1+tdT2t9RxL/UattZ9VuwSt9RitdXWtdaDW+hat9XZHxWlPA7yMy04pE/+Cwrj97nTeaXYt3mW2sb7stHfv3uveViiEEEIImfrAoXp7VsUtLxuCgwmsnf9onXnfzDOXW7Rogbu7OwDx8fGcPXvW4XEKIYSjyR9v4oobfS9IQuNAfkCPVMtTG4cMNevXbl1rfgM9PDxo2bKluU6mQhBCVHZeXl4kJCRIUiPQWpOQkICXl1ept3XUg/WExQCdw0Yg6Ykx8M37ABxOO8yZPWeo08GYobVt27ZmInPgwAEGDhyIq6urs0IWQgi7qlOnDjExMcTFxdll/xkZGTf0C1LcmLKeby8vL3PG8tKQhMbBBvjU4XUgr21r/HyrkZIaRxZZ/PS/n3j6u6cBCAsLw9/f33y6YlRUVIEJLIUQojJxd3enQYMGdtt/eHg4HTp0sNv+RUHOOt9yycnBurj54p+TCkrh2WuAWb9sxTJz2cXFRZ5JI4QQQpSCJDQO5g70TTNuXUuZ/IBZ/+fZP0mNy5+0yzqhOXLkCOnp6Q6LUQghhKhoJKFxgv7KuIspc9hAPFw8AbjEJVZOW2m2qV69OjVr1gSMidUOHjzo+ECFEEKICkISGicY4FvXWPDwILBFV7P+lzm/FGgnl52EEEKIkpGExglaurhTI/MiAOn35z81eHPEZnKzc81ymzZtUMp4AF90dDQXL150bKBCCCFEBSEJjRMoYEDGeQCSJ99vqYFTuafYuyz/uTO+vr40btzYLEsvjRBCCFE0SWicZICbZbaH6tUJrWbcrqjRzJsyr0C7du3amcv79u2TB08JIYQQRZCExkn6++Q/NChv8DBzec3mNQXaNW3aFE9Py8DhS5fsPrmXEEIIURFJQuMkdZSiWVosABeffNCsj0iO4FzEObPs7u4uUyEIIYQQ1yEJjRMNyE4EILdTewK8AgHIJJOfv/i5QDvry04RERHk5OQ4KkQhhBCiQpCExokGeFYzFpQiuHMPs/7XX38t0K5evXoEBgYCxhwZkZGRjgpRCCGEqBAkoXGivl7VcNHGbdpn//qQWb/z9E4ykjLMslJKnkkjhBBCXIMkNE4UCHRJjQYg/d6heLh4AJBAAqtnrC7Q1jqhOXr0KGlpaY4KUwghhCj3JKFxsv65mcaClxf1GrQw63+eVXAcTXBwsDmdel5eHgcOHHBYjEIIIUR5JwmNkw3wrmUu59491FzeuHcjebl5Bdpa99LI3U5CCCFEPklonKyHRxW8c43xMiee+j+z/kTOCSJWRRRo27p1a1xcjG9ZbGws8fHxjgtUCCGEKMckoXEyL6C3ZRwNtWpSN8iYuFKj+fHrHwu09fb2pmnTpmZZemmEEEIIgyQ05cAAy1xOANX75M++vTJ85VVtrZ9Js3//fpkKQQghhEASmnJhgE9dczn6yYfN5f1J+4k/VvCyUpMmTfD29gYgKSmJU6dOOSZIIYQQohyThKYcaOfmRXBWEgBxt91BoEcAABlk8Mv/finQ1tXVlVatWplluewkhBBCSEJTLrgA/dKNeZ1QimZtWpvrli5eelV768tOBw8eJDs7294hCiGEEOWaJDTlxABXb3NZjelvLv9x8g+yUrMKtK1duzbBwcEAZGVlcfjwYccEKYQQQpRTktCUE9bjaPY99H+4K3cA4nU862atK9BWpkIQQgghCpKEppxo6OJKg/TzAKRVqUar+s3MdT9///NV7a0TmmPHjpGcnGz/IIUQQohyShKacmRAVoK5XOeOjuby+j/Xo/MK3p4dGBhI/fr1AdBay1QIQgghbmqS0JQj/d0DzeVzf73bXD6WdYzIjZFXtZepEIQQQgiDJDTlSD/vmuby7nZDaVC1HgB55DH3i7lXtW/ZsiVubm4AnD9/nvPnzzsmUCGEEKKckYSmHKmmFO1TjQfl5bq40bZbG3PdyrVXPzXYy8uL5s2bm2XppRFCCHGzkoSmnBmQk2ou+zzY01zem7CXxJjEq9pbX3bav38/eXl5V7URQgghKjtJaMqZAV41zOV9A+8kwPLU4DTSWPi/hVe1b9SoEb6+vgCkpKRw/PhxxwQqhBBClCOS0JQzt3gG4ZFrPEgvIqgtPVq3N9ctWrDoqvYuLi60aZN/aUqeSSOEEOJmJAlNOeML9EiLNstNRuRfUvoj6g9yMnKu2sb6stOhQ4fIzMy0a4xCCCFEeSMJTTk0QOeayxfv64WbstzJpM+z6cdNV7WvUaMG1atXByAnJ4dDhw45JlAhhBCinJCEphwa4FPHXF5fuycd6uX3wPz43Y9XtS88FYLc7SSEEOJmIwlNOdTZzZcq2SkAxPjWpVf//Nm1w3eEo7W+ahvrcTQnT54kKSnJ/oEKIYQQ5YQkNOWQG3BbWoxZDhqX3/tyNOMoJ7afuGqbKlWq0LBhQ7Msg4OFEELcTCShKacGuLiby7ubtqB+gDFvUy65zPvfvCK3adcuvydn3759RfbkCCGEEJWRJDTlVH+fuuby2pBuDOjVyywvX7m8yG2aN2+Ou7uRCMXHx3P27Fn7BimEEEKUE5LQlFPNXT2olRkPQJJHIF3ub2mu23NhDykXUq7axsPDg5Yt89vJ4GAhhBA3C0loyikFDMi4YJbjeoXh7+4PQCqpLPpyUZHbWd/tdODAAXJzc4tsJ4QQQlQmktCUYwPc/M3ltX71uKXNLWZ50fxFRW4TFhaGv7+xXVpaGlFRUXaNUQghhCgPJKEpx/pbPY9mc3BXht+TP1nlliNbyM2+uvfFxcWlQC/Nrl27ZHCwEEKISk8SmnKsllK0sNy+neXqSehddXDFFYCzeWfZtmBbkdtZ3+109OhRuYVbCCFEpScJTTk3ICv/AXlb/X1oW9fqqcHfXP3UYIBq1arRoUMHs7x8+XIuXrxovyCFEEIIJ5OEppwb4FXdXF7t34Q7hww2y2u2ril2u0GDBhEUFARAVlYWCxYskAHCQgghKi2HJTRKqSCl1EKlVKpS6pRS6oFi2q1QSqVYfWUppfY7Ks7y5lavarjmGTNs767ajjvH5k9xcCTtCNF7oovczsPDg3vuuQcXF+NbfObMGdavX2//gIUQQggncGQPzRdAFhAKjAG+Ukq1KtxIaz1Ya+135QvYAvzkwDjLlQCga+opALRyIbqOpo6/MVg4l1zm/bfopwYD1KpVi379+pnljRs3cvLkSXuGK4QQQjiFQxIapZQvcA/wqtY6RWu9CVgCjLvOdmFAb2Cm3YMsxwbkZZnLq1096d+rv1le/lvRTw2+omfPnjRo0MAsL1y4kPT0dNsHKYQQQjiRcsQtvUqpDsBmrbWPVd1zwK1a62HX2O41oJ/Wum8x6ycDkwFCQ0M7zZtXfG9FWaSkpODn52eXfZfE/iru/K2jMfVBo+Qo/u/T+Tz92ssA+OPPz4t/xqOKR7HbZ2ZmsnPnTnJyjEtX1apVo0WLFiil7B/8DXD2+b4ZyTl3LDnfjiXn27Hseb5vu+22XVrrzkWtc7PLEa/mB1wuVJcE+BfR1tp44J3iVmqtpwJTATp37qz79u1bhhCLFx4ejr32XRI9gBdz0klz8+aYf2PunNiRV97yJTUnlWSSSTyYyKgXR11zH/Xq1WP+/PkAxMXF0aNHjwJ3QpUnzj7fNyM5544l59ux5Hw7lrPOt6PG0KQAVQrVVQGSi9tAKXULUAP42Y5xVQieQJ+0/MG/4TkX6dky/yF7C+YuuO4+WrRoQceOHc3yihUrSEhIsGmcQgghhLM4KqGJBNyUUk2s6toBEdfY5kFggdb66lkYb0IDyL88tNo9kLtH322WN0VsIi8377r7uOOOOwgODgYgOztbbuUWQghRaTgkodFapwILgLeUUr5KqV7AcGBWUe2VUt7AKGCGI+KrCAb41DOX1wZ3ZtS4PrhYvn1ncs+w69dd191H4Vu5Y2NjWbdunX0CFkIIIRzIkbdtPw54AxeAucBjWusIpVRvpVThXpgRQCIgv20t2rh5US3rEgBxXtU5rY/RulZrc/3cr+eWaD81a9akf//8u6Q2b97MiRMnbBusEEII4WAOS2i01he11iO01r5a63pa6zmW+o2W581Yt52rta6vZVZFkwvQL/2sWV6dk8yQQUPM8prNxT81uLAePXrQsGFDs7xw4ULS0tJsEqcQQgjhDDL1QQUywNXbXF7tVYNxf8t/jM/B5IOcPXy2qM2uopRixIgR+PgYd9EnJyezdOlSmZVbCCFEhSUJTQViPY5mQ0g3GoUpavrWBCCHnGs+Nbgwf39/7rrrLrN8+PBh/vzzT9sFK4QQQjiQJDQVSJiLK40sl53S3HzZlnSQ/t3zx8Ms+3VZqfbXrFkzOnfOfz7R77//Tnx8vG2CFUIIIRxIEpoKZkDmRXN5jc7h/kfuN8s7Tu8gMzmzVPsbOHAg1apVA4xbuX/55RfzicJCCCFERSEJTQUzwKOqubzaN4yBI/rh7WKMrbnMZVZ8t6JU+3N3d+eee+7B1dUVgHPnzrF27VrbBSyEEEI4gCQ0Fcxt3jVR2niI3vagTqQl76d78+7m+p9nlf7ByqGhoQwYMMAsb926lePHj5c9WCGEEMJBJKGpYIKVokOqMQ1Crosb61NPMPIvI831G/dvROeV/m6lbt260bhxY7Mst3ILIYSoSCShqYAG5KSay6uVG6OfGG0+NTg6O5p9q/aVep9KKYYPH27eyp2SksKSJUvkVm4hhBAVgiQ0FdAA75rm8pqAFoRU9aZ5aHOzbs6Xc25ov35+fowYMcIsHzlyhF27rj+lghBCCOFsktBUQLd4BuGZa9zNdDCgJbHx2xl8+2Bz/coNK294302aNKFr165m+ffffycuLu7GgxVCCCEcQBKaCsgb6JV6yiyvyThX4KnBEYkRxJ+48efJ3H777VSvXh2AnJwcuZVbCCFEuScJTQXV33KnE8BqNz/adm5LqHcoANlkM++zkj81uDA3N7cCt3KfP3+eNWtKPleUEEII4WiS0FRQA3zrmsurgzpCZjx9u/Q165YuWVqm/VevXp2BAwea5W3bthEVFVWmfQohhBD2IglNBdXJzZeA7GQAYn1qczhhB/dNus9cv+PkDrLTs8t0jC5dutCkSROzvGjRIlJTU6+xhRBCCOEcktBUUK5Av7QYs7wmO4mho4fi5eIFwCV9iZUzbnxwMOTfyu3r6wtAamoqixcvllu5hRBClDuS0FRgA1zczeXVnsF4eHjQpXEXs+6nmT+V+Ri+vr4FbuU+evQoO3bsKPN+hRBCCFuShKYCG+BTz1xeF9yNnJTjjLhnhFm3/s/1NulNady4Md2750+vsHLlSi5cuFDm/QohhBC2IglNBdbE1YM6GcYzYi57BLDz0j7GPDkGhQLgZNZJDm08ZJNj9e/fn9BQ4y6q3NxcfvnlF7KzyzZGRwghhLAVSWgqMAUMyMjvKVmdl0lozVCahjQ16374/AebHOvKrdxubm4AXLhwgdWrV9tk30IIIURZSUJTwQ1wr2Iur/apAzqPO/rdYdYtX7mcvNy8ojYttWrVqnHHHfn73r59O5GRkTbZtxBCCFEWktBUcP196pjLW4M6k3ppPw/+/UGzbs/lPfzwkm16aQA6depEs2bNzPLixYtJSUmx2f6FEEKIGyEJTQVXQylap0YDkOXqyabkSDr27Ej3pvmDeF/76DWSzybb5HhKKe666y78/PwASEtLk1u5hRBCOJ0kNJXAgOzL5vJqZQwI/vbnb3GxfHtP5p7kjVFv2Ox4Pj4+jBw50ixHRUXxxx9/2Gz/QgghRGlJQlMJ9Peqbi6v9m8CuZm0atOKifdONOu/3fQth363zR1PAA0bNqRHjx75x129mnPnztls/0IIIURpSEJTCdzqVR3XPGM27D1V2xGXsBOAj779iCqexqDhy1zmuXHPkZuda7Pj9u/fn5o1awLGrdwLFiyQW7mFEEI4hSQ0lYA/0D31pFlel3YagICAAN55+x2zfmXcSha9vshmx3V1deXuu+/G3d14YnFcXBwrV5ZtugUhhBDiRkhCU0kMyMsyl1e7epnLjz/zOM1qG3cl5ZDDmx+8yeWYy1dtf6NCQkIK3Mq9c+dOjhw5YrP9CyGEECUhCU0lMcA7//bt1VXbQlYSYPSiTJk9xVy3P3c/n4z9xKbH7tixIy1atDDLixcvJjnZNndVCSGEECUhCU0l0dWjCr45aQCc8GvI8YTt5rpb+97KXQPuMstfr/+aI8tt14uilGLYsGH4+/sDkJ6ezqJFi+RWbiGEEA4jCU0l4QHcankeDcDqzPgC6/837X94uHoAcI5zvDHhDXIycmx2fG9v7wK3ch8/fpytW7fabP9CCCHEtUhCU4kMUPnfztUeVQusq1u3Ls8/87xZXhK3hN/e+s2mx2/QoAG9evUyy2vWrOHs2bM2PYYQQghRFEloKpEBPvXM5bVBnclLO1Ng/ctvvkyNqjUASCON9z54j0vHL9k0httuu41atWoBkJeXxy+//EJWVtZ1thJCCCHKRhKaSqS1mxfVs4wEJcErhL0XdxdY7+3tzadffWqWt+Vu47tJ39l0rEvhW7kTEhL4/fffbbZ/IYQQoiiS0FQiChiQFmuWV+dcPWnkqFGj6N7RmOcpjzymrJ/C4cWHbRpHcHAwgwcPNst//vknhw7Z7inFQgghRGGS0FQy/d18zOXV3jWgUO+LUoqvvvsKF8t4myii+Hjyx2Sl2vayUPv27WnZsqVZXrp0KZcv2+75N0IIIYQ1SWgqGetxNBuDu5Jx+erbs9u3b8/EB/PneVoQt4C1b621aRxKKe68806qVDGmXpBbuYUQQtiTJDSVTD0XV5pYLjulu/mwNelgke3e++A9/H2M58Zc5CKf/OcT4g/HF9n2Rnl7e3P33Xeb5RMnTrBlyxabHkMIIYQASWgqpQFZF83lNbroySKrVavGW/96yyyH54Uz95G5Nu9BqV+/Pr179zbLa9euJTY29hpbCCGEEKUnCU0lNMAjyFxe4d8EMuKKbPfEE0/QpGETALLIYvqm6UT8GGHzeG699VZq164NyK3cQggh7EMSmkqon08tPCyTVf4Z1JGIM0uLbOfu7s7nX35ulnezm2//71syL2faNJ4rt3J7eBhPKr548SIrVqyw6TGEEELc3CShqYQCgeFWD9WbrvNA5xXZ9o477mDo4KFmeUHCAta+ZtsBwgBBQUEMGTLELO/Zs4eDB4se3yOEEEKUliQ0ldRE75rm8qzaw8g+V3yS8ul/P8XN1Q2A05zmu/9+x/l9520eU9u2bWndurVZXrp0KUlJSTY/jhBCiJuPJDSV1EBXL2plJQJwwTuUFRd3FNu2cePGPP3M02Z5pV7JgskL0Hm2HSCslGLo0KEEBAQAkJGRwcKFC8nLK7r3SAghhCgpSWgqKVfgwbz82bSnBbSEtOLvLnrllVeoFlINgGSSmffHPPZ8v8fmcXl5eXH33XejlALg1KlTbN682ebHEUIIcXORhKYSm+AVYi4vqzWECyfnFtu2SpUqfPDhB2Z5M5uZ/+x80i+m2zyuevXq0adPH7O8bt06YmJibH4cIYQQNw9JaCqxpkAvyy3bOS7uzM5NBatem8LGjx9Pp46dAMgllyWXlrDm5TV2ia1Pnz7UrVsXAK01CxYsIDPTtndXCSGEuHlIQlPJTfIINJen1b0bHVv87dIuLi58/r/827gPcpCfv/6ZMzvOFLvNjXJxcWHkyJF4enoCcOnSJbmVWwghxA2ThKaS+4uLOz65xjNpIgJbs/P8tXtcevTowdixY83yb/zGkkeXkJdr+4G7VatWZejQ/FvG9+7dy4EDB2x+HCGEEJWfJDSVnD/wl7z8SznT/ZtByslrbvPvf/8bXx9fAM5znmV/LmPX1F12ia9Nmza0bdvWLP/6669kZGTY5VhCCCEqL4clNEqpIKXUQqVUqlLqlFLqgWu07aiU2qCUSlFKnVdK/d1RcVZGE939zeU5YaNJPz79mu1r167NSy+/ZJbXspZfX/yV1AupdolvyJAhBAYGApCZmcmhQ4fkVm4hhBCl4sgemi+ALCAUGAN8pZRqVbiRUioE+A2YAgQDjYGVDoyz0ukDNMpOASDJI5BFGech99pzKT3zzDOE1Q8DIJ10fr/8O6teWGWX+Dw9PbnnnnvMW7kvX77M+vXr7XIsIYQQlZNDEhqllC9wD/Cq1jpFa70JWAKMK6L5M8DvWusftNaZWutkrfUhR8RZWSlggqu3WZ5e9244s/ia23h5efHRxx+Z5e1sZ9X3qzi18ZRdYqxTpw633nqrWd6wYQM7d+60y7GEEEJUPo7qoWkK5GitI63q9gJX9dAA3YGLSqktSqkLSqmlSql6DomyEnvQxRWljSf/rq4xgOjoBdfdZuTIkfTr1w8AjeY3fmPZY8vIzc61S4y9e/emQYMGZnnZsmXs27fPLscSQghRuSitbft4+yIPolRv4CetdQ2rukeAMVrrvoXaRgLVgduB/cAHQCetda8i9jsZmAwQGhraad68eXaJPyUlBT8/P7vs25H+0aY524ONb8Fb+15l4IUmpLtdO1c8fvw4jzz8CHmWyS3v536GPDaEuqPq2iXGnJwcdu/eTVpamlnXqlUrQkJCrrGVKKvK8h6vKOR8O5acb8ey5/m+7bbbdmmtOxe1zlEJTQdgs9bax6ruWaCv1npYobZ7gT+11hMt5WAgHgjUWhc7k2Hnzp21vS5RhIeH07dvX7vs25HmAaMtyw2Tj3H06Fe4dPzPdbf7v//7P7744gsAqlKVp32f5u9H/k6V2lXsEueqVauIioriwoULALi6ujJ69GgaNWpkl+OJyvMeryjkfDuWnG/Hsuf5VkoVm9A46pJTJOCmlGpiVdcOiCii7T7AOsuyf8Z1kxgBBOZlA3DcvxEbkiMh5/pTG7z55ptUrVoVgEtcYkPqBlY+Y79x2u7u7owbN46goCAAcnNzmTdvHtHR0XY7phBCiIrNIQmN1joVWAC8pZTyVUr1AoYDs4poPh0YqZRqr5RyB14FNl2rd0aUjBfwgHIzy9Pr3gOnf77udsHBwbz11ltmeQMb2DZ/G8dXH7dHmAD4+fkxbtw4qlQxeoFycnKYM2cOsbHFT7AphBDi5uXI27YfB7yBC8Bc4DGtdYRSqrdSKuVKI631WuAlYJmlbWOg2GfWiNKZaLk1GuDnevdy+fjMEm336KOP0qqVMYY7iyxWs5rlTywnJ7P4uaHKKjAwkPHjx+PrazzkLzMzk9mzZ5uXooQQQogrHJbQaK0vaq1HaK19tdb1tNZzLPUbtdZ+hdp+pbWurbWuqrUeprU+7ag4K7tOQGvLZac0N1/m+9WHS9e/k8jNzY3PPvvMLO9lL3sj97LlP1vsFSpg9A6NGzcOLy8vANLT05k1axYXL16063GFEEJULDL1wU1GAZNc3M3y9IYTIWpKibbt378/I0aMMMsrWMH6d9aTeDLRtkEWEhoaytixY/Hw8ACMEfQzZ87k8uXLdj2uEEKIikMSmpvQWMDNchv2lmq9OBK/FbJTrr2RxUcffWQmFmc4w+6M3fz299/sFaqpdu3ajB49Gjc3YwxQUlISM2fOJDXVPtMxCCGEqFgkobkJVQPuJH8szfR6o+DU3BJt27BhQ5599lmzvIpV7FuyjyNLj9g6zKuEhYUxatQoXFyMt21CQgKzZs0iPf36d2oJIYSo3EqU0Cil2tk7EOFYk6wGB89sMJ6cqKlQwmcSvfTSS9SsWROAFFLYyEZ++9tvZKdl2yVWa02aNCkw79P58+eZM2cOWVnXnptKCCFE5VbSHprVSqm9SqnnlFI17RqRcIjBQKjlstNZn1qs9KoGF0v2YEI/Pz/ef/99s7yVrRw/eZyN7220R6hXadmyJXfddZdZjomJYd68eeTk2O+OKyGEEOVbSROamsBrQDfgqFJqpVJqrFLK5zrbiXLKDRin8r/90xpOgqNfl3j7MWPG0K1bNwByyWUlK9nywRYSIhNsHWqR2rdvz+DBg83yiRMn+Omnn8jNtc88U0IIIcq3EiU0WuscrfVirfVfgNrAfOAF4LxSaqblQXmigplotbyk9l3En10JWYkl2tbFxYX//ve/Zvkwh4nMimT5/y3HEdNpAHTt2tWcPBMgMjKShQsXkpeX55DjCyGEKD9KNShYKeWH8QT9+4E6GNMDHQV+UEp9YfPohF21BLpZko9sVw/m1B0JJ4p6eHPRunbtyoMPPmiWf+M3jq46ysGfD9o61GL17t2bW265xSxHRETw66+/OiypEkIIUT6UdFDwUKXUPOAMcB/wLVBLa/2I1vptoCPw4LX2Icon6ycHT2s0CaK+LvHgYID33nvPnFU1jjh2spPfn/qdzORMm8danH79+tGlSxezvHv3bn7//XdJaoQQ4iZS0h6afwO7gOZa6yFa63la64wrK7XWF4Gn7BCfsLP7AS/LL/69Vduz28UD4jaVePuaNWvyyiuvmOV1rON87HnWv7ne1qEWSynF4MGDadcu/2a8P/74g/DwcIfFIIQQwrlKOoamjdb6Q6312Wu0+dZ2YQlHCQDuseqlmd5wYqkGBwM89dRTNGrUCIAMMljHOrZ9uo0LBxw355JSirvuuosWLVqYdRs2bGDz5s0Oi0EIIYTzlPSS0wKlVO9Cdb2VUtefqlmUe9aDg38IG0PmmSWQEVfi7T09Pfn444/N8k52ci73HMseX+bQyz4uLi7cc889NG7c2KxbvXo1O3eW7HZ0IYQQFVdJLzndChSehXArcJttwxHOcBtQ37J80TOYJTUHwfEZpdrHsGHDuP322wHQaFawglMbT7Fv1vUnvrQlV1dXRo0aRf369c26ZcuWsW+fY+MQQgjhWCVNaDIA30J1foD9Hw0r7M4FmGBVnt7IMmGlLvntz0opPv30U1xdXQE4yUkOcYhVz68i/ZJjpyZwd3dn9OjR1KpVy6xbtGgRhw4dcmgcQgghHKekCc3vwBSlVBUAy///A+w/K6FwCOtb1H6vcQdnctPh3JpS7aNly5Y88cQTZnklK0m8kMjaV9baKMqS8/T0ZOzYsVSvXh0ArTW//PILx44dc3gsQggh7K+kCc2zQBXgolLqAnARYzzpU3aKSzhYA+DKI+ryXFyZ2WC8cQt3Kb3xxhsEBwcDkEgiW9nKzq92Ersr1nbBlpC3tzfjxo0jKCgIgNzcXObNm0d0dLTDYxFCCGFfJb3L6ZLWeihQFxgK1NFaD9NaJ9ozOOFY1oODpzeciI5ZDGmlS0SqVq3KO++8Y5Y3spHL+jLLHltGXq7jn+Dr5+fH+PHjqVKlCgA5OTnMmTOH2FjHJ1hCCCHsp1RPCrbctr0TuKCUclFKlWp7Ub7djdENB3C0SlM2h3SHY9+Vej+PPPIIbdu2BSCbbFazmtgdsfz57Z+2C7YUAgICGD9+PL6+xjCwzMxMZs+ezYULjrutXAghhH2V9LbtWkqphUqpBCAHYzDwlS9RSfhgPGjviukNJ8KxqZBXulmsXV1d+eyzz8zyPvZxmtOs+ecaUuNSbRNsKQUHBzNu3Di8vLwASE9PZ9asWVy8eNEp8QghhLCtkvawTAGygP5ACsZUB0uAR+0Ul3AS68tO8+uNIiXrEsSuKPV++vbty7333muWV7CCtEtprH5xtQ2ivDGhoaGMHTsWDw8PAFJSUpg5cyaXL192WkxCCCFso6QJTU9gktZ6D6C11nuBhzAGC4tKpBvQ3LKc4u7Pz3XvvaHBwQAffvghnp6eAMQSy172smfaHqI3O29Qbu3atRk9ejRubm4AJCUlMXPmTFJTndNzJIQQwjZKmtDkYlxqAkhUSlUDUoHadolKOI0CJlmVpzecaPTQpJws9b7CwsJ4/vnnzfJqVpNBBssfX05ejuMHCFvHNWrUKFxcjLd/QkICs2bNIj3dsc/LEUIIYTslTWj+AIZYln8HfgQWYAwQFpXMOMDVsrwh9Fai/BrCsW9uaF8vvvgitWsbeW8qqWxkI+f3nWf7F9ttE+wNatKkCffccw/KMo/V+fPnmTNnDpmZjpslXAghhO2UNKEZB1yZPvkpYC1wAHjADjEJJ6sBDLYqz2g4wbjbKTer1Pvy9fXlgw8+MMtb2UoCCax7dR3JsclljrUsWrZsyV133WWWY2JimDdvHjk5pRsELYQQwvmum9AopVyBzzAuMaG1Ttdav6O1/se1Zt8WFZv1ZafvGzxIbmYcnFl8Q/saPXo0vXr1AiCPPH7nd7KSs1j53EobRFo27du3Z/Dg/PTt5MmT/PTTT+Tm5joxKiGEEKV13YRGa50LDAScN+hBONxQIMSyHONblzWh/eHojQ0OVkrx2WefmZd3IokkiigOzD3AibUnbBNwGXTt2pV+/fqZ5cjISBYuXEhenrzlhRCioijpJadPgDeVUu72DEaUHx7AWKvytEaT4PxauHzkhvbXqVMnJk7Mvyn8N34jl1yWP7Gc3Czn94b07t2bW265xSxHRETw66+/orV2YlRCCCFKqqQJzZPA80CyUuq0Uir6ypcdYxNOZn3ZaVGdEVxyD4SoqTe8v3fffRd/f38A4olnBzuIPxzP1o+3li1QG+nXrx9dunQxy7t37+b333+XpEYIISqAkiY0Y4EBwB2W5XFWX6KSagN0sixnunoxN2w0HJ8BOTd2e3NoaCivvfaaWV7HOlJJZcPbG0g8lVjWcMtMKcXgwYNp3769WffHH38QHh7utJiEEEKUTEknp1xf3Je9AxTOZf3k4GkNJ0HWRTj98w3v729/+xtNmjQBIJNM1rKW7LRsfn/q9zJGahtKKYYNG0bLli3Nug0bNrB582YnRiWEEOJ6SjqX01vFfdk7QOFcowFPy/Ku4M7sD2h9w4ODATw8PPjkk0/M8i52cZazHF50mKPLj5YtWBtxcXHh7rvvpnHjxmbd6tWr2blTHrskhBDlVUkvOdUt9NUFeA5oZKe4RDkRBIywKk9vNBHit8ClfTe8zyFDhjBo0CCz/Bu/odGseHIF2enlY75TV1dXRo0aRf369c26ZcuWsW/fjb9uIYQQ9lPSS04TC30NBu4mfzoEUYlZX3aaHTaWLBd3iJpyw/tTSvHJJ5+Y8ymd4hQHOcil45fY9O9NZYzWdtzd3Rk9ejS1atUy6xYtWsShQ4ecGJUQQoiilLSHpigrKfjHu6ikBgB1LMtxXtVZVmsonJgF2Sk3vM/mzZvz5JNPmuWVrCSLLDa/v5n0M+VnTiVPT0/Gjh1L9erVAdBa88svv3Ds2DEnRyaEEMJaScfQNCz01Rp4Bzht3/BEeeAKPGhVnt5wIuQkw6m5Zdrva6+9RrVq1QBIIoktbCE3M5ejnx0tV7dKe3t7M27cOIKCggDIzc1l3rx5REfLUwuEEKK8KGkPTRRw1PJ/FLAN6E3B33OiEptgtby81hDOeYXC0a+gDIlHYGAg//rXv8zyJjaRRBKXdlzi0C/l67KOn58f48ePp0qVKgDk5OQwZ84cYmNjnRyZEEIIKPkYGhettavlfxettZ/WurfWepe9AxTlQ2Ogj2U518WNWQ3GwaXdcLFsd/5MmjSJDh06AJBDDqtYBcDCcQvZOWVnueqpCQgIYPz48fj6+gKQmZnJ7NmzuXDhgpMjE0IIUdJLTu2VUnUL1dVVSrWzT1iiPLIeHDy94UQ0lOkWbjDuJvrss8/M8gEOcIpT5GTksOzRZfx8389kJGWU6Ri2FBwczLhx4/Dy8gIgPT2dWbNmcfHiRSdHJoQQN7eSXnKaDRSex8kDmGXbcER5di/ga1k+FNCS7cFdjXE0WYll2m/v3r257777zPJKj5XkWeZCPfjTQaZ0mMKZHWfKdAxbCg0NZezYsXh4eACQkpLCzJkzuXz5spMjE0KIm1dJE5p6Wuvj1hVa62NAmM0jEuWWH3CfVXlaw0mQm27c8VRGH3zwAd7e3gCcyTpDfJ94c13iiUSm9ZzGlo+2oPPKxyWo2rVrM3r0aPPW86SkJGbOnElqaqqTIxNCiJtTSROaGKVUR+sKS1lGRN5krC87zat/P2mu3hD1dZkGBwPUq1ePf/zjH2b5601fk3BfAu7+RsdgXk4eq55bxdy75pIWn1amY9lKWFgY9913Hy4uxo9RQkICM2bMID4+/jpbCiGEsLWSJjSfAIuVUk8qpYYopZ4EFgIf2y80UR71AppYli97BLCw7khIOghxZX8g3vPPP0+jRsbDp/Py8vj8x89Z2WYlvu19zTZHlx3l6/Zfc2rDqTIfzxYaN27MPffcg1IKgPj4eL755hsOHjzo5MiEEOLmUtK7nL4BngGGAh9a/n9Waz3VjrGJckhRxISVUObBwQA+Pj6sX7+etm3bmnXrt6znw9gP8bjHw6xLPpPM97d9z/q31pOXm1fm45ZVy5Ytueeee8zLT1lZWfz000+sWrWKvDznxyeEEDeDEj8pWGv9k9Z6kNa6leX/G59yWVRo48l/46yt0Z+TvvWNGbgz4sq879q1a/Pxxx/z6quvmr0eFy5c4OUFL3Nu1Dk8gozERudpwl8PZ9bts0g+m1zm45ZVq1ateOihh6hatapZt2XLFmbNmiXjaoQQwgFKetv2f5VSPQvV9VRKfWqXqES5VhsYaFX+vsGDkJcFx2fYZP+urq689dZbrFq1itDQUMCYcuDr+V+zpNES/Lr6mW1PrjvJ1+2+Jur3KJscuyxq1KjBI488QtOmTc26kydPMmXKFGJiYpwYmRBCVH4l7aEZDRR+gtou4AHbhiMqiklWy9MbTiQPZUxYqW13iaV///7s3buX22+/3azbtmMb70a9i9t9bsb1LyAtLo0fBv3A6hdXk5uda7Pj3whvb2/uv/9+brvtNrMuOTmZ6dOns2PHjnL1oEAhhKhMSprQ6CLaupZie1HJ3AUEWZZP+YURHtoXUo7BuTU2PU5oaCi//fYb7777Lq6urgBcvHiRV358hei7o/EK9TLbbn5/MzNunUHiqUSbxlBaSin69OnDmDFjzFvR8/LyWL58OYsXLyY7O9up8QkhRGVU0oRkI/COUsoFwPL/m5Z6cRPypGD33PSGlqHCUWUfHFyYi4sL//znPwkPD6dOnTpm/bRfpvFTzZ/w7+1v1sVsjWFK+ykcWuj8uaAaN27M5MmTqVmzplm3d+9evvvuO3mysBBC2FhJE5q/AwOAs0qp7cBZS/lJewUmyj/ru51+rnsvSe5VIGYxpNnn8US33HILe/bs4c477zTrdu3ZxZt738R1jCvK1bgGlZGYwfy757P8yeXkZOTYJZaSCgwMZNKkSbRv396sO3/+PN988w2RkZHOC0wIISqZkt62HQN0BIZj3Lb9F2AdsL2kB1JKBSmlFiqlUpVSp5RSRY6/UUq9oZTKVkqlWH01LOlxhON0AK5M5pXh5s2P9e4DnQvHvrPbMYODg1myZAkff/wx7u7GQ/cuX77Mqz+8StRdUfjU8zHb7vjfDr7r8R0JkQl2i6ck3NzcGD58OHfeead52SwjI4O5c+eybt06ubVbCCFsoDRjYIKBbsBLGMlMR4yem5L6AsgCQoExwFdKqVbFtP3RMqP3la/jxbQTTlT4mTTTG1lKx6ZCnv16RpRSPP3002zatIkGDRqY9bMWzmKW/yyqDKhi1p3bc44pHaewb/Y+u8VTUp06dWLixIlUqZIf34YNG5g7dy7p6elOjEwIISq+ayY0Sil3pdQ9SqmlwBngr8ACIBEYpbX+qSQHUUr5AvcAr2qtU7TWm4AlwLiyBC+cbwz5s5ZuC+nBoSrNIS0GYlfY/dhdu3blzz//5N577zXr9kfs59Wtr6LHa1w9jN6Q7NRsFo5byOJJi8lKzbJ7XNdSu3Zt/vrXv9KwYX6nY1RUFFOnTuXs2bNOjEwIISq26/XQnAemAEeA7lrrllrrtzF6WkqjKZCjtbYeNLAXKK6HZphS6qJSKkIp9VgpjyUcKATjjqcr7Dk4uCiBgYHMnz+fL7/8Ek9PTwBSU1N5c+ab7L9jP76N8qdN2DN9D990+Ybz+887JLbi+Pj4MGbMGG655RazLjExkWnTprF7924nRiaEEBWXutZzMZRS4cAtwFZgNjBfa31JKXUWaKe1vlCigyjVG/hJa13Dqu4RYIzWum+hti0xeoDOY1zi+gV4Rms9t4j9TgYmA4SGhnaaN29eScIptZSUFPz8/K7f8Ca1NSiIlyzTFYSmn+P0orq46Vz+qD6HDLca19n6ajd6vqOionjrrbc4ffq0WVevbj3G1R6H+zZ3s87Fw4VGTzSi5rCa5tOInSU+Pp7Dhw+Tm5v//JyaNWvSuHFjc9JLR5D3uGPJ+XYsOd+OZc/zfdttt+3SWncuat01ExoApVR9jKfdjwfqASuBW4EWWuszJQlAKdUB2Ky19rGqexboq7Uedp1tXwS6aK3vuVa7zp076507Cz/7zzbCw8Pp27evXfZdGeRgvDGuXDBZsn4Yw878Cq1egnb/KvX+ynK+U1JSeOyxx5g9e7ZZ5+XlxfP3P4/nj57kpOeP7Wk1qhV3Tr0TrwCvonblMAkJCcyfP58LF/L/PqhVqxajRo0iICDAITHIe9yx5Hw7lpxvx7Ln+VZKFZvQXPdPQK31Ka3121rrJkB/jN9becBepdQHJYwhEnBTSjWxqmsHRJRgW435TFhRHrlhZLtXmJedjn0HuY4ds+Ln58fMmTOZPn06Pj5G/pyRkcHbM95m12278G+Z/8yaiPkRTO04lTM7SpSX201wcDAPPfQQrVu3NutiY2OZOnUqx4/LeHghhCiJUvVpa603aa0nAzUwnkHTpoTbpWIMJn5LKeWrlOqFcQv4rMJtlVLDlVJVlaEr8DdgcWniFI5nfbfT0trDiPMMgYzzcMbx3zqlFBMmTGDHjh0FkoSFyxfyZcaXBNyT3+tx6fglpvWaxtaPtzp1WgIPDw/uvvtuBg0aZF5qSktLY/bs2WzatEmmTBBCiOu4oYv0WusMrfVcrfXgUmz2OOANXADmAo9prSOUUr2VUilW7e4HooBkYCbwvtb6+xuJUzhOM6CHZTnHxZ3ZYWONwlHHDA4uSsuWLfnjjz945JFHzLpjx4/xj6X/IGNCBh7+xszdedl5rHx2JXOHzSUtPs1Z4aKUolu3bjz44IPm9WetNWvWrGH+/PlkZGQ4LTYhhCjvHDbqUGt9UWs9Qmvtq7Wup7WeY6nfqLX2s2o3WmsdbHn+THOt9X8dFaMomwITVjaaiAY4vxYuH3FSRMYdRVOnTmXOnDlmkpCVlcW/Z/ybzd03E9Ahv7fm6LKjfN3+a05tOOWscAGoV68ekydPpl69embd4cOH+eabbwqMsxFCCJFPJpcUNjMKowsOYH9gW/6s2tEoRE11Vkim0aNHs3v3bjp27GjWLVu1jE8SPiFgdH5Sk3wmme9v+571b68nL9d5T/D19/dn/PjxdO/e3ay7ePEi3377Lfv373daXEIIUV5JQiNspgpwr1V5WiNLn83xGZDj/CfhNm7cmC1btvC3v/3NrIuOjua5+c+RMiEFzyDjOTY6TxP+WjizB84m+Wyys8LF1dWVO+64g3vuucec5iE7O5sFCxbw22+/FbjVWwghbnaS0Aibsr7sNCdsLBkunpB1EU7/7LSYrHl6evLZZ5+xcOFCAgMDAcjNzeU/M/7DmrZrqNq9qtn2xNoTfN3ua6J+j3JStIbWrVvz8MMPExwcbNb98ccfzJw5k+Rk5yVcQghRnkhCI2yqD3BldqVEjwAW1xluFJw4OLgoI0aMYM+ePQUu6awOX8370e8TMD7AfFBAWlwaPwz6gdUvriY323k9ItWrV+eRRx6hefPmZl10dDRTp07l1CnnjvkRQojyQBIaYVMuwASr8rRGDxkL8VvgkvMniLRWv359NmzYwAsvvGDWxcbG8uzsZ0kcn4hPaP7M3Zvf38yMW2eQeCrRCZEaPD09GTVqFAMGDDCfcJySksLMmTPZtm2b3NothLipSUIjbO5B8p+EuKrGAE771DEKUVOcFVKx3N3def/991m+fDkhISEA5OXl8en3n/Jr418J7pt/mSdmawxT2k/h0MJDzgoXpRS9evVi3Lhx5oMD8/Ly+P3331mwYAFZWc6dfFMIIZxFEhphc/UxHikNoJULMxtYniN8YhZkpxS3mVMNHjyYPXv20KdPH7Nuw+YNvBXxFv6T/FGuRoqWkZjB/Lvns/zJ5eRk5BS3O7tr0KABkydPpnbt2mbdgQMH+Pbbb0lISHBaXEII4SyS0Ai7sH5y8PTGk41n0uQkw6mr5hgtN2rXrs2aNWt47bXXzEs6cXFxPDvtWc4/cB6/uvmTre343w6+6/EdCZHOSx4CAgKYMGECnTp1Muvi4uKYOnUqhw8fdlpcQgjhDJLQCLsYCVx5ussx3/psrNbbKBz9CsrxWA83NzfefPNNVq9eTY0a+TOFfznrS36u+TMhd4SYdef2nGNqp6ns+8F5Y4Pc3Ny48847GT58OG5uboDx4MAff/yR1atXk5fnvGfpCCGEI0lCI+zCGxhtVZ7e6GFj4dJuuGifWdFtqV+/fuzdu5eBAweaddu2b+PV7a/iO9kXVw9XALJSslg4diGLJy0mK9V541fat2/PpEmTzFvRATZv3szs2bNJTU11WlxCCOEoktAIu7G+7DS//n0ku1ku2ZSzW7iLU716dVasWMF7772Hq6uRwFy6dInnpz5P9L3RVGlcxWy7Z/oevunyDbG7Yp0VLjVr1mTy5Mk0btzYrDtx4gRTp07lzBnnziguhBD2JgmNsJsuQCvLcpqrJz/V+4tRODUXshKdFFXpuLi48OKLL7J+/Xrq1q1r1n8z5xt+8P+B6sOrm3Xxh+L5pvM3zL9nPuf2nnNGuHh7e/PAAw9w6623mnWXL19m+vTp7Ny5U27tFkJUWpLQCLtRFBoc3PT/jIXcdDgx2xkh3bBevXqxe/duhg0bZtbt2r2LF9e9iPdj3rj7uJv1hxYcYkr7KU5LbJRS9O3blwceeAAvLy/AeBrysmXLWLJkCdnZ2Q6PSQgh7E0SGmFXYwFXy/KmoI5E+jcxClFfl+vBwUUJDg5m8eLFfPLJJ+bcSpcvX+YfX/2DyGGRNBreqEB7Zyc2TZo0YfLkyQUGN+/Zs4dp06Zx6dIlh8cjhBD2JAmNsKtQ4E6r8ozGjxgLSREQt9kZIZWJUoqnnnqKLVu20LBhQ7P++x+/54PjH9Dth260uLtFgW2cmdhUrVqVSZMm0a5dO7Pu3LlzTJ06laNHjzo0FiGEsCdJaITdWV92+r7Rw+Qqy9suqmIMDi5K586d+fPPPxk1apRZt3//fgaPGcwHpz7A62Uv6g+rX2AbZyU27u7uDB8+nKFDh+LiYpz7jIwM5syZw/r162VcjRCiUpCERtjdEODK0NlYj6qsrGG5FTr6J8iId1ZYZRYQEMC8efP4+uuv8fT0NOt37drFi/96kcdXP87BYQdRfRSa/KTBGYmNUorOnTszceJEqlTJvzsrPDycuXPnyrgaIUSFJwmNsDt3YJxVeXqLZ42FvCw4McMJEdmOUoq//vWv7Nq1iwceeKBAYpOens78pfN5fcPrzA6bTVTrKFLIn/rBGYlNnTp1mDx5MmFhYWbd0aNH2bVrF0eOHHFIDEIIYQ+S0AiHsL7stDj0NhI8gozC0SmgK/7TbFu1asUPP/xAbGws//3vf2nTpk2B9cdOHmP2gdl86vYpK2qt4ChHycN43Y5ObHx9fRk3bhy9evUy6zIzM5k3bx7z588nOTnZ7jEIIYStSUIjHKIVxnNpALKUK3MaPWQUUqLg/FpnhWVzQUFBPPnkk+zdu5ft27czefJk/Pzy54DKycnhj9g/+IEf+ML7C9axjkQSAccmNi4uLgwYMIBRo0aZs3YDHDp0iC+++IIdO3bI2BohRIUiCY1wmElWy9ObPplfqCBPDi4NpRRdunRhypQpnD17lmnTptGzZ88CbRLSE1jPej7jM2Yzm4McJIcchyY2LVq04Iknnihwa3dmZibLly9n2rRpXLhwwa7HF0IIW5GERjjM/YCXZXm3b132BFpuJY5ZBGnOmzLA3vz8/Jg4cSKbN28mIiKCZ555huDgYHO9RhNFFPOZz8d8zEpWEkecwxIbHx8fmjVrxoMPPlggrpiYGKZMmcKaNWtk0LAQotyThEY4TCDGLNxXTG/9irGgc+H4NCdE5HgtW7bko48+4syZM/z444/cfvvtBdankcYWtvAFXzCNaexhD3sX7HVIYhMWFsajjz5Knz59zNu78/Ly2LRpE19//TXHjx+327GFEKKsJKERDmV92emH2sPIcrFMGRA1FfJynRKTM3h6ejJq1ChWrlzJ8ePHefXVV6ldu3aBNtFEs4hFfMRH/MqvrFmwhq/bf23XxMbNzY3bbruNRx99lHr16pn1Fy9eZNasWSxcuFBm7xZClEuS0AiH6gdc+TWZ4OrJ0vpjjELaaTi7wllhOVWDBg146623OHXqFMuWLWPkyJG4ubmZ6zPJZCc7mcpUpjCF7xd8z6ftP7VrYlOtWjUmTJjAnXfeac4HBbBv3z6++OIL9uzZI4OGhRDliiQ0wqFcgAetytNavpBfqISDg0vD1dWVIUOGsGDBAk6fPs37779PkyZNCrQ5xzmWs5yP+Ih/LfgX/2z/T368+0e7JDZKKTp16sQTTzxBq1atzPr09HQWL17MrFmzSEhIsPlxhRDiRkhCIxxugtXyb1WaE+td0yjELofUU84IqdypUaMGL7zwAkeOHGH9+vWMGzeuQE9JDjnsYx8zmMHjCx9nXPtxTB061S6JjZ+fH/feey8PPPAAAQEBZv2JEyf46quv2LBhA7m5N8/lQiFE+SQJjXC4hkBfy3KeUsxq87alpCHqG+cEVU4ppejTpw8zZ87k7NmzfPHFF7Rv375Am4tcZDWreWz5Y9zW/jZe7PkiMbtibB5LkyZNePzxx+nRowdKKQByc3NZt24dU6ZMITo62ubHFEKIkpKERjiF9ZODp9UflT/T0bFvUTrHCRGVf4GBgTz++OPs3r2bXbt28dhjj+Hv52+uzyOPwxzm/a3v07JzS+5pfg/bl2+3aQweHh4MHDiQRx55hFq1apn1cXFxTJ8+nV9//ZWMjAybHlMIIUpCEhrhFPcAV34VR7r7s7X2XUYh4zwhGZudFVaF0bFjR7788kvOnT/HjBkz6Nqha4H1ySSz4MgCug3tRtvQtkz5YAqZmZk2O37NmjV56KGHuOOOO3B3dzfrd+3axRdffEFERIQMGhZCOJQkNMIpfIH7rMrTW79mLtdKXeLweCoqHx8fHnzwQf748w8OHTrE4w8+ThXPKgXa7L+wn0f/8SjVqlTjr+P+SkREhE2O7eLiQvfu3XniiSdo2rSpWZ+SksLPP//M3LlzSUxMtMmxhBDieiShEU5jfdlpXlAHUt2MOY+qZv0JlyOdE1QF1rx5c76Y8QVxl+P49qNvaR/avsD65Kxkps6eSuvWrenSrgvTpk0jJSWl6J2VQkBAAPfffz9/+ctfCsxbdfToUb788ku2bt1KXl7Fn4BUCFG+SUIjnKYH0MyynKJc+KXt2/kro6Y6I6RKwcPDg4eeeYjd53azY8UO7m1xLwEEFGizc99OHnroIWqE1mDy5Mns3r2brKysGz6mUoqWLVvyxBNP0LlzZ7M+OzublStX8u2333L27Nkb3r8QQlyPJDTCaRQFe2mmNxiXXzj6JVza6+iQKp3Ogzrz08GfiNgVwT97/ZOWtMTF6sc+NS2Vb775hmeeeYagoCDuuusuvvzyyxue5sDLy4uhQ4cyadIkqlevbtafPXuWb775ht9//71MiZMQQhRHEhrhVOPIfxOGewZzvMYgo5CbDhvvhqxLzgqtUqndsTbvbnqXNXvW8PWQrxnIQEIIKdAmNTWVpUuX8sQTT9CoUSOaNm3K3/72N5YtW1bq6Q7q1q3L5MmT6devn/nUY60127Zt48svvyQyUi4pCiFsSxIa4VS1gMFW5Rk9Z5KjvI1CynHYMh60jL+wlRrtavDIskf4fs/3/G/k/5jEJDrTmUACr2p79OhRPv/8c+68806CgoK4/fbb+eijjzhw4ECJ7mBydXWld+/ePPbYYzRo0MCsT0pKYu7cufz0008kJyfb8uUJIW5iktAIp7O+7PS9VzUOBf4jvyL2VzjwL4fHVNnVaFeD+xbcx7/2/IvXJr/G80HP8yRPMpjBNKEJbrgVaJ+VlcXq1at57rnnaNOmDXXr1uXhhx/m559/5tKla/eiBQUFMW7cOEaMGIGPj49Zf/DgQb744gt27twpt3gLIcrM7fpNhLCvYUAwkABEAytrjaBNjVQ49KHRYP/rENwZag0ufifihtRoV4NhU4bhd58fTf2bcmTJESKXRBKzL4ZoookiimMc4wIXCmx35swZvvvuO7777jvz9u1BgwZxxx130KlTJ1xdXQu0V0rRrl07mjRpwqpVq9izZw8AmZmZLFu2jH379nHnnXcWGHcjhBClIQmNcDoPYCzwmaW8vEYNnq32LlzcCefXARq2jIFBu8CvQfE7EjdMuShqd6lN7S616fd2PxJPJnJkqZHcnAw/yaWcSxzjGFFEcZzjZJD/NOC8vDy2bNnCli1beO211wgODmbgwIEMGjSIgQMHUqNGDbOtj48Pw4cPp23btvz6669cvHgRgNOnTzNlyhR69epFnz59Csw2LoQQJSGfGqJcmEh+QrMpJIREF1cCe82DFR0h/YwxOHjj3XD7FnDzdmaoN4XAsEC6PdmNbk92IyMpg6jfoohcEsnR5UdJTUzlDGeIsvyLJbbAtgkJCcydO5e5c+cC0L59ewYNGsSgQYPo0aMHHh4eNGjQgMcee4wNGzawefNm8vLyyMvLY+PGjURERHDnnXcWGHcjhBDXIwmNKBfaAR2A3UCWqyvzgEe9qkPvn2F1H8jLhkt7YMdj0H06WCZHFPbnFeBF6/ta0/q+1uRm5xK9KZojS45wZPEREk8kkkoqxzluJjipFLwjas+ePezZs4d///vf+Pv7069fP/PyVL9+/WjdujW//vorp0+fBuDixYvMnDmTdu3aMXDgwALjboQQojiS0IhyYxLwpGX5C0vZI6Q7dPqvkcgAnPgeQrpDk0edE+RNztXdlQa3NaDBbQ244+M7iDsYlz/u5o8Y8nQe5zlvJjenOU0e+XepJScns3jxYhYvXgxA06ZNzeSmadOmbNq0yZxzau/evRw9epSBAwfStm1bc4ZvIYQoiiQ0otx4AHgeyAAOAM8B/wVo/FeI32YkMwC7/gZV2xuJjXAapRTVW1Wneqvq9P5nb1LOpRC5LJLIJZHUXVWX3um9ySCDk5w0E5xEEgvsIzIyksjISP773//i6elJr169qFOnDn5+flSrVo20tDQWLVrE3r17zdvHhRCiKJLQiHIjCHgXeMZS/hzoBdynFHT5ChL3Gped8rJh470w+E/wkrtiygu/Gn50fKgjHR/qSHZaNsfXHOfIkiMELw2m+fnmaDQJJJjJzUlOkkOOuX1mZiZr1641y4GBgTRs2JDGjRuTnp7O6dOn6dOnDz179rzqLiohhJCERpQrTwEL4+LYWK0aAA9jjK9p7uYNvRfAb52MAcLpZ2DTfdBvFbjI27i8cfdxp9mwZjQb1gydpzmz44x5aSrkQAjd6U422eat4VFEEUdcgX0kJiby559/8ueff6KUom7duqxdu5YuXbrw2GOPUb9+fSe9OiFEeSS/CUS5ooAXDh/mbLVqRAEpwL3AH4CvXwPo+QOEDwU0XAiHvS9Bhw+cGLG4HuWiqNOtDnW61aH/v/pz6fgl85Zwj/UeNMptxB3cQRJJZnJznONkkmnuQ2tNdHQ00dHRrF27ls8//5yuXbsycuRI2rVrR6tWrQgJCblGFEKIyk4SGlHu+OXm8jPQHWM8TQTwGPA9oGoNhjZvGA/bA+Phe8Fdod69TopWlFbVhlXp/vfudP97d9IvpRe4JTzgcgCd6EQuucQQYz77pvCt4WlpaYSHhxMeHm7WVa9enVatWtGqVStatmxpLgcHBzv4FQohnEESGlEutcO40+khS3kW0Bt4BKD1K5Cww5gWAWDbRAhoBQEtnBCpKAvvqt60Gd2GNqPbkJuVy6mNp8xLU64nXalPffrRjxRSjFvD3aM45nqM1IyrJ8u8cOECFy5cYN26dQXqQ0NDCyQ4V75kgLEQlYskNKLcmgRsAqZbyk8CnYCOygV6zoLfOkPKMchJMR66d8d2cPd3WryibFw9XGnYvyEN+zdk0KeDuHDggpncnNl+hra0pW12W/Ky8zjX4Bwnap/g/OXzxMXFERcXR05OTpH7PX/+POfPny8y0Smc5LRs2VISHSEqKEloRLn2P2AXsA/IxBhPswuo6hFoDBJe2R1y0+HyYaOn5paf5KF7lYBSitA2oYS2CaXPy31IPptM5K/GLeHHVx+n1ola1DpVC1oCwyGveh6JiYnExcVx4cIF4uLiSDiXQNzFOLJysoo8xpVEx/rOKoAaNWpcleS0atWKqlWrOuCVCyFulMMSGqVUEPAdMBCIB/6ptZ5zjfYewF7AX2tdxzFRivLGB/gZo2cmGTgBTAAWAapqW+g6FbaOMxqf/gUO/QdaPu+UWIX9+Nf0p9Mjnej0SCeyUrM4vvo4R5cf5fSm08R9HYdLIxeCbgkiqFkQzZo1M7fLy8sj8XAiCTsTyFSZJPokEpsey/GY42RkZBR5rHPnznHu3DnWrFlToL5mzZpXjc9p1aoVgYGB9nzpQogScmQPzRdAFhAKtAeWKaX2aq0jimn/PBAHyDWEm1wTjMtOV4b9LgH+g/EGocFYSPgDIv9nrNz7IgR1ghr9nBCpcAQPXw+aD29O8+HNAUi/mE7MthiiN0dzbNcxzvmeQzfToMDFxYWglkEEtQwy/ozaAhyEvNw8XJu5kt0gm8v+lzmbdZajp45y+PDhYhOds2fPcvbsWVavXl2gvlatWlclOS1btpRERwgHc0hCo5TyBe4BWmutU4BNSqklwDjgxSLaN8CYgPkZ4BtHxCjKt3swnlHzqaX8T6Ab0Aegw0dw8U+I3wI6Dzbfb8zM7VvXKbEKx/IO8qbJkCY0GdKE/vQnNzuXyC2RbNmyhTMZZ9Au2mgYAtwF3AYu21zQOzVuR9wIsvzrVrMbtYfUxrWZK5erXuZsxlkOHT5EREQEhw8fNqdkKCw2NpbY2NgiE52iLl0FBATY94QIcZNSWmv7H0SpDsBmrbWPVd1zwK1a62FFtP8V4/LUJWB2cZeclFKTgckAoaGhnebNm2eP8ElJScHPz88u+xZXK+58ZyvF0+3bE2H5hRCcmcnUnTsJys7GIzeeznGT8ci7BMBl9xbsDvkUrTwcGntFVVnf41lZWcTExBB7JpbcvNyCKzOBncA2jOuZhbh4uODf3J8qravg19KPtKA0YhJiOHnypPkVHR1NdnZ2qWIKCQmhbt26NG3alAYNGtCwYUPq16+Ph4e8V+2lsr6/yyt7nu/bbrttl9a6c1HrHJXQ9AZ+0lrXsKp7BBijte5bqO1IYLLWerBSqi/XSGisde7cWe/cudOmcV8RHh5O3759r9tO2Ma1zvdpoCPG1QOA24BVgCvAhQ2wph9oyy+uxo9C16/sHG3lUNnf45mZmezatYtt27aRnFwoe8kDlwgX8sLzIOHa+wlpHkKdnnWo16sedXvVJaBhACdOnCAiIoKDBw8SERFh9uhkZRU9GLkorq6uNGnShDZt2hT4atCgAS4uLjfwioW1yv7+Lm/seb6VUsUmNI4aQ5MCVClUV4VCfxdZLk19AAxxUFyigqkL/AAMAjSwDngdeAegeh/o8CH8aZkNKuprCOkGDSc4JVZRfnh6etKzZ0+6du3K/v372bJlC/HxlrTYBfLa5EEbCPUMxS/Kj4vrLnLp+KWr9hN/OJ74w/HsmbYHAO9gb+r2rEvdnnUZ22cstZ6phbu3Ozk5ORw7dqxAkhMREcGRI0eKTHRyc3M5fPgwhw8f5qeffjLrfX19ad269VWJjjwVWYirOSqhiQTclFJNtNZHLXXtMB4Ca60JEAZsVMattx5AgFLqHNBda33SMeGK8mwgRhLzhqX8L6AHMBSg2VMQ/wdE/2is3P4oBLaFoI4Oj1OUP25ubnTo0IH27dtz5MgRNm/eTExMjLn+fOZ5ztc9T/1X63Nri1txO+1GzNYYTm8+zdk/z5KXnVdgf+kJ6UQujSRyaSQALu4u1OxYk7q9jCTn9l63M3LkSLN9Tk4OUVFR/Pjjj+Tl5bF//37279/PsWPHKKq3PDU1lT/++IM//vijQH2NGjWuSnJatmyJt7e3LU+XEBWKQxIarXWqUmoB8JZS6mGMu5yGAz0LNT2A8Uf4FT0xHkXSEQrNXCduaq8AmzEuN4Exunw3UF8p6PYtJB2ApAjIy4SN98CgneApj8AXBqUUzZs3p1mzZkRHR7N582aOHj1qrj916hSnTp2ievXq9HqwFwM+GEBeVh6xO2M5vfk0p7cYX+kJ6QX2m5edx5k/znDmjzNs+3gbAIENAs1LVHV71qVpq6bceuutBbrkU1NTiYiIMBOcK19xcUV/7F25tXzVqlVmnYuLS5GXrRo2bCiXrcRNwZG3bT8OTAMuYFypfkxrHWEZX7NCa+2ntc4Bzl3ZQCl1EcjTWp8rco/ipuWKcempA3AGY/T4X4CNgKe7n2Vm7s6QkwypJ2HLGLh1Gbi4Oi9oUe4opahfvz7169fnwoULbNmyhf3795OXZ/TEXLhwgYULF7J27Vq6d+9Ox24dqd/bmOVba01CZEJ+grP5NPGH4686RuKJRBJPJLJv9j4APKt44tXAi5QeKYQ0DzG/unTuQteuXQtse/78+auSnIiICNLT0686Tl5eHkeOHOHIkSP8/PPPZr2Pjw+tWrUyE5y2bdvSpk0bqllmtBeisnDIoGBHkEHBlUdpzvcW4FbgykPv/w/4/MrK04tgY353P61fhbZv2SjKykXe4/mSkpLYunUrf/7551V3MHl7e9OlSxe6deuGj4/PVdumJaQRs9V4Jk7MlhjObD9DTkbRUzIU5u7jTnCz4AJJTkjzEIKaBOHu7W62y83N5fjx4+zbt69AohMVFVXkZavihIaGFnnZqqjXVdHJ+9uxnDUoWBKaEpAfBscq7fn+FHjaqjwXuP9KYc9LcPC9/JW3LoXad5Y1xEpH3uNXS09PZ/v27Wzfvp20tLQC666MxenRo8c1p0TIzcrl3J5zRG+ONnpyNp8m5VxK6QJRULVBVUKahxDcvGDC4xPig2W8IWlpaRw8ePCqHp3z58+X/FBK0bhx46t6cxo2bIira8Xt3ZT3t2NJQlNGktBUHqU93xrjctMvlrIvsANoAZCXC+GD4JzloWfuAcZ4Gv/GNoy44pP3ePGys7PZs2cPW7ZsITExscA6pRStWrWiV69e1KhRo+gdWNFakxSdxJq5a6jlUcu8ayr+cDxpcWnX3b4w7yBvQlqEXNWrExgWiIubMW7mwoULRV62KpykXfM43t7mwwEbN25c4KsizHEl72/Hquy3bQthNwrjKYx7gSggFWOahO2Ar4sr9JwLv3WCtGjITjJm5h64DdwqX9e6sD13d3e6dOlCp06dOHjwIJs3b+bcOWNYn9aaAwcOcODAARo1akSvXr0ICwsze00KU0oRWD+Q4O7B9Ojbo8C6tIS0AglOwuEE4g/Hc+n4JXRe0X94pl9MN3t+rLl6uBLUJIhqLaoR3DyYas2rMar3KB5/+HE8/DzIy8vj+PHjVyU6R48eNccPFThOejo7d+6kqD8ag4KCaNy4MY0aNboq2alWrVqx50IIW5OERlQKARiTWHYHMoCDwKPATEB5hUDvX2BVL8jLgsT9sH0y9JglM3OLEnNxcaF169a0atWK48ePs3nzZk6cOGGuP3bsGMeOHaNWrVr06tWL5s2bl+ruIp9gH+r1qke9XvUK1Odk5nDx6MUCyc6Vr+zUop9SnJuVS1xEHHERV98lVaVOFbNXp07zOrS/pT0hD4fgV9OPjIwMDh06xP79+wuM0bmSwBXl4sWL5qW5wvz9/YtNdmrWrCl3XwmbkoRGVBrtgC+BSZbybKA3lrkxgjtD5y9g+yPGypM/QHB3aPZ/TohUVGRKKRo1akSjRo2IjY1l8+bNHDp0yByQGxsby08//URQUBA9e/akXbt2uLnd+Eetm6cb1VtXp3rr6gXqtdYkn0km7lDcVb06ybFFzOVgcTnmMpdjLnN81fEC9R7+HoQ0D6Fai2o0bN6Qrrd0JeThEIIaBf1/e/cdH9dV5338c2Y06t2SZcuymmW5O5JrbMctPZBG2CS00Elg6fAEliwtWSAEWGDZAAuEhQU2pMESk2I7TTi24y65ypZlq1qS1XufOc8f52o0I8lyyWikkX7vvO5Lmtvm3Bt59NU9jabWJvcTnOLiYq9lpB5XA9ra2sjPzyc/P3/YtrCwMHfQGRp4Zs+eHdBtdsT4kEAjJpWPADsx4wMAfBZYbi1kfdzMzH3mCbPx0BchPhcS141DScVkkJyczN13301jYyO7d++moKAAp9NMvdHY2MgLL7zAG2+8werVq1m5ciWhoaE+e2+lFNEp0USnRDPnhjle23pae6g/VU99ofcTncbTjbj6h1cpAfS29VK1v4qq/VXe72NXxM+JJ2F+AunZ6eSk5xC3KY7Y9Fhi0mKob6nnzJkzw4JOcXExra2tFyx/V1eXu7puKIfDQWZmpjvgeAae9PR0HA7HCGcUU50EGjHpPI6Zc/AI0ItpMHwQiANY8Z/QVACNB0D3w8674eZDEHbxBp1CXEh8fDy33normzZtYu/evRw4cIDu7m7ADJr3+uuvs3PnTpYvX87VV1895uUJiQ5h1spZzFo5y2u9s89Jc0kz9SfrqSuscz/RqSuso6dl5NnEtdOMt9NQNPJEVxFJEcSmxxKXEcfm9M3cdd1dxH4slpj0GPrC+yirLBsx7DQ0XHjirL6+PveYOkPZ7XbS0tJGDDuZmZk+DY0isEgvp0sgLeT9yxf3+zTmqczAg/fbgL8BNoCOMtNIuMf6QE1cD9e9Brap+1ef/Iz7Vk9PD4cOHeKtt94aNhmmzWYjMTGRW265hdTU1AnRaFZrTUdth3mSM+SpTktZy5WfWEFUchRxGeaJTmxGrPsrcVDXXUdJWcmwsDNam51R304pUlJSvKqv5syZQ2NjI+9+97uJi4ubEPd7spNu22+TBJrJw1f3+y+Y3k4DHgO+MvCi+hXTnVtbj9/nfQGW/+Rtv2egkp/xseF0Ojl69Ci7du0anAzTQ3x8PDk5OeTk5BAVFTUOJby43o5eGk83UldYR9PZJjPycakZ/bilvOWCVViXQtkVMbNjhoUdx3QHTaqJqpYqSkq8A09FRcXFT3wBERERpKWlkZqa6l48X8+aNUuqs3xAAs3bJIFm8vDl/f4SMBBT7MDrwIaBjccfhcMPDe689s+Q/h6mIvkZH1taa4qKiti1a9eIv5AHBrTLzc0lOzs7YBrEupwu2s610VTS5A45nl9bK1sv2OX8UtgcNmLTvMNOaHIobcFt1PXWca7hnFf7ndLS0hG7nV/y+9lsJCcnjxh2Bl7HxMRc8fmnChmHRogx8BiwFzNFghO4FzOJ5QyAhV81jYQrnzc77/0YxC42ixA+pJRi3rx5zJs3j3PnzvHiiy/S2NhIT49pt6K15vTp05w+fZrw8HCWLl1Kbm4u06dPv8iZx5fNbiMmNYaY1BgzB8kQzl4nLRUtw8LOQABqrx591GRXn4vG4kYaixtH3B4UFsTC9IWsTV9L7M2xRKRE0BXZRYOrgfNd5ymvLqe4uJjjx49TX19PR0fH6O/nclFZWUllZSW7d+8ecZ/o6OhRA8/MmTPfVq82ceXkrotJzQE8jZnEsh4z8+n7gO1AkLLB1f8D21ZC22lwdppB927aD8HyV5gYG7NmzSI7O5t169ZRWFhIfn4+paWl7u2dnZ3s2bOHPXv2kJycTG5uLosXLw7Ixq72YDvxc+KJnxM/4va+rj5aylvcYaepxLtKq7N+9NGM+7v6TZufwuHVeQAzomYwP30+t8y6hfR3pGNPtNMR1kELLTT2NlLbWktFZQVlZWWUl5dTXV190fmwWltbL9g7C0yj5VmzZg0LPJ7fT9TqxUAngUZMeinAk8BNmGkS3gC+BXwXTHBZ/1fYttoEmrbTsOdDZp2SQb/E2HE4HCxdupSlS5fS1NREQUEBBQUFXl2dq6qqqKqqYtu2bSxYsIDc3NxRRyIONI4wBwnzEkiYlzDi9t723sGgM8JTngv1zHIf39ZL7dFaABp2D+9VFWWLYnXyam5MvZHo9dGEzwqnJ6qHtqA2mnUzDV0NnKs9R0VFBeXl5ZSVlY067g6YdlPl5eWUl5ezc+fOEfeJjY0dtS2PDDp4ZSTQiCnhBkyI+bb1+nvAWuCdYKqYVv8Wdr/XbKx8Hk48Bou+5v+CiikpLi6OzZs3s3HjRkpKSsjPz+fkyZPuMW36+/vdo/bGxsa6GxJP9vYcwZHBIw4qOKC7ufuCYae5tPmCIykP0C7tHmhwJDZsZIRlkJOaQ8zsGKLWRKESFB1hHbTaWmnsbaSuvY7Kqkp34LmUyUCbm5tpbm7m8OHDI253OBykpKSQkpJCYmLiqEtCQgLBwcEXfc+pQAKNmDK+jmlLs916fR9wCEgH0xi4YS+c+qnZeOTrEL8CZt7g93KKqctms7lHIe7s7OTo0aPk5+d7/ZJsbm4mLy+PvLw85syZQ05ODvPnz5+S7TZCY0OZmTuTmbkzh23TWtNZ30lzSTO7X95NSnQKLeUttJa30lLRQkt5Cx3nR29TA6Zaq+FUAw2nRh43J5ZYkhOSuSH1BqKvjiYsOYzuqG7aHG206BYauhuoaapxP7UpLy93t526kL6+PkpKSrym1hhNTEyMO+BMnz79oiEoEKsvL8XU+xcgpiw7ZjqEZUAl0IQZdG8nEAKQ+wNoOgS1O0x37t3vhZsPQkTauJVZTF3h4eGsXr2a1atXU11dTX5+PkePHnUP2AeD80eFhoayZMkScnNzmTlz+C/3qUgpRURiBBGJESR2Jg6bDBTMPFmtla0m6FSYr0O/723vveh7ddZ30lnfSfWh6mHbQgghIyiDpbOWEjM7hugV0bimuegM7zRPefrMU56q2ip34KmrGz4H12haWlpoaWmhuLj4kvaPjIy8YNgZKRBFRERcVnnGiwQaMaUkAs9gum73Y0YU/jJmdGFsDlj3NGxdBl3VZuC9N98NN+wE++T8i0YEhpkzZzJz5kxuvPFGTp48SUFBAWfOnHFv7+7uZv/+/ezfv58ZM2aQk5PDkiVLCA+XGeVHExQSNGqjZa01PS09JtxYT3U8A09rhamuuthYPK5+Fy1lLRccpDCRRGZFzuK62dcRsyyGkJkhdEV10R3aTZe9iw5XB219bbR0tlDfWE9dXZ17qa+vv+yu6u3t7bS3t1/yE6CwsLCLPvXxDEPjNRyMBBox5awBfgR8wXr9c2Ad8F4wUyBc8xy8utFMjdB4EA58Flb/ZnwKK4SHoKAgFi9ezOLFi2lpaXE3JG5ubnbvU1NTw9atW3nllVeYP38+OTk5ZGZmSiPTK6CUIjQ2lNDYUJKWJo24j8vpor2mffCpjhV8PKu2OutG760FpgH0aD22Qq3/0uLSiEyKJCIpgshFkYRND8MV7aI3tJeuoC46VScdrg5ae1ppaGrwCj8DS39//2Xdh66uLvfTo0vhcDjYtGkT27dvv/jOPiSBRkxJnwPexIwmDPAJIAdYAJC4Fpb9BA5+1mw88wRMW20mtxRigoiJiWHjxo1s2LCB0tJS8vPzKSwsdP+ycjqdHD9+nOPHjxMdHe1uSBwXFzfOJZ9cbHYb0bOiiZ4VTcrVKSPu09fVd9Gqrb7O0RswD+hu6qa7qZv6kyMHH0/JMclkJ2Wb8JMdScT6CMKnh0M0dId0023vpgPz9KexpXHE8FNXV3fRNj/Drrev720NcHilJNCIKUlhZuQ+gpn3qQN4N7APiATI/jQ07IHS/zUHHPg0xF0F01aOS3mFuBClFBkZGWRkZNDd3c2xY8fIz8+nqmpw1uzW1lZ27NjBjh07SE9PJzc3lwULFsgw/37iCHMwbe40ps2dNuJ2rTXdTd3Dqrbaq9ppP99Ox/kO2s+301nXeVkjL/e09NDT0nPBiUU9BUcGk5GUwZKkJUQkRRCRGUFkUiTh08OxxdjodnTTaTNPf5ram6ivN1VftbW1wwJQZ2cniYmJl1xOX5FAI6asaOA5YDXQDRQCnwT+iPklwapfQ/NRaD4Crl54859MI+HQkcfMEGK8hYaGsmLFClasWMH58+cpKCjgyJEjdHYOVnmUlpZSWlrKSy+9xOLFi8nNzSU5OXnSjG0TiJRShMWHERYfxoycGRfcz+V00dXQ5RVyvL7WeLyu7UA7Lz389Lb30tveS9OZpovuGxQWRGRSJPNnzGd50nIiZkcQucKqBkuK5GjpUVZdt+qS39tXJNCIKW0p8EvgI9br/wXWAw8ABIXD+r/A1hXQ1wKd5abn06atYAuMuXbE1JWUlMRNN93E9ddfT1FREfn5+RQXF7sbbPb09HDw4EEOHjxIYmIiubm5LF26NGB6tExFNruNiOkRREyPgCWj76tdmq7GC4efoetcfZdeRdTf1W/G/CltvuA+zbnNPHDogUs+py9IoBFT3ocxXbd/a73+HLACWA4QlQVr/wT/uM1srHkVjnwDcr7n/4IKcQXsdjsLFixgwYIFtLW1cfjwYfLz82lsHJwfqa6uju3bt/Pqq6+SnZ1Nbm4uWVlZ0pA4gCmbIjwhnPCEcFg0+r5aa7qbu0d+6jMkALXXtOPscV70/SOTIn10JZdOAo0QwH9iunAfBnqBf8IMuhcHMOtWWPwNOPZvZucTj8K0VTD7znEpqxBXKioqimuuuYZ169ZRUVFBfn4+x48fp6/PNEh1uVycPHmSkydPEhkZyVVXXUVubi7Tpo3c9kNMDkopwuLCCIsLI2H+6FXqWmt6WntGDT9Vp6uYvtT/E6tKoBECCAOexTyZaQVKgQ8CzwM2gMXfgob9UL3VHPDWByHmAERnj0dxhXhblFLueYNuvvlmjh8/TkFBARUVFe592tvb2bVrF7t27SI1NZWcnBwWLVokw+xPcUopQmNCCY0JZVr2yEE3Ly+PTZs2+bdgSKARwm0u8DtMbyeAF4AfAl8F02Zm7f/C1uXQUQr9bfDmu+DGveDw/6NVIXwlJCSEZcuWsWzZMurr68nPz+fw4cN0dAxOCzAwBslLL73E3LlzWbBgAdnZ2YSEhIxjyYXwJoFGCA93AV8Cfmy9fgi4GtgIEBJvZuF+ZS04u6HlBOz9OKz7M0gPETEJJCQkcMMNN3DttddSXFxMQUEBRUVF7jFF+vv7KSwspLCwELvdzpw5c1i4cCHZ2dmEhYWNc+nFVCeBRoghvg/swUxk6QLeA+QDMwDic2HlL2GP1S+q/GlIWA3zvzg+hRViDNjtdubNm8e8efNob2/nyJEjHD58mNraWvc+TqeToqIiioqKsNlsZGZmsmDBAubPny9TLohxIYFGiCEcwNNALlAP1GCmRXgF6x9M5oehfi8U/5c5IP9BiF8O0zeMR3GFGFORkZGsXbuWtWvXUl9fz4kTJygsLKSmpsa9j8vlori4mOLiYl544QXS09PdPasiI6VKVviHBBohRpACPAncBGggD/gm4O6svfyn0JQPDXtBO2HnPXDzIQhPHo/iCuEXCQkJbNiwgQ0bNtDY2EhhYSEnTpzwGpVYa01JSQklJSW89NJLpKamsnDhQhYsWEB0dPQ4ll5MdhJohLiAG4BvA9+yXj8KrAVuBbCHwPrn4OVl0FMH3edh591w3Rtgl14gYvKLj49n3bp1rFu3jubmZnfbGs+eUjDYoHjr1q2kpKSwYMECFi5cSGxs7PgUXExaEmiEGMXXMW1ptlmv78OMT5MBEJ4C1zwNr18P2gX1u2H/J2HVr8Amc+SIqSM2NpY1a9awZs0aWltbOXnyJIWFhZSVlblHJgaorKyksrKSV155hZkzZ7rDjYxzI3xBAo0Qo7ABf8K0p6kEmoF7MCMLhwAkbYarvg8FXzEHnP0dtJ4yQSd85Jl3hZjMoqOjWbVqFatWraK9vd0dbkpKSrzCTXV1NdXV1bz++uskJSW5w814TGooJgcJNEJcRALwDLAB6MeMKPwl4OcDOyz4f9B8eHBm7vrd8HIOrPkTJN/s9/IKMVFERka6J8vs7Ozk1KlTFBYWcubMGXdXcIDz589z/vx58vLySEhIcIebpKQkmTRTXDIJNEJcgjXAj4AvWK9/AawD3gdmDJo1f4DYpXD4IdNIuKcB8m6BRQ/BkofBJv/UxNQWHh5Obm4uubm5dHd3U1RUxIkTJyguLsbpHJwbqL6+njfffJM333yTuLg4d4NimRFcXIx8ygpxiT6HqWp6znp9P5ADLARQNlj4FUhYC7vuhS6r18fx70HdLjP4XthM/xdaiAkoNDSUpUuXsnTpUnp6ejh9+jSFhYWcPn3aPa8UQFNTk3v6hZiYGPeTm5SUFAk3YhgJNEJcIoWZkfswcBrowExiuQ9wj7Qx/Rq4JR923wc128262n+YKqi1T8KM6/xdbCEmtJCQEBYvXszixYvp6+ujuLiYwsJCTp06RW9vr3u/lpYW9uzZw549e4iKinKPc5OamiqzggtAAo0QlyUa+AuwGugCCoEHMA2H3X8vhk6HzS+bpzNHv2V6QHXXwus3wJJvwaKvm7mhhBBeHA6HO6j09/dz9uxZTpw4walTp+ju7nbv19bWxr59+9i3bx8RERHMnz+fhQsXkp6eLuFmCpNAI8RlWoJpQ2NNfsCTwHrgk547KRss/rqpgtr9PjNODRqOfhvqdpqJLkOn+7XcQgSSoKAgsrOzyc7Oxul0UlJS4g43nZ2d7v06Ojo4ePAgBw8eJCwsjHnz5rFw4UIyMzOx2+UPh6lEAo0QV+DDmPY0v7Vefx5YYS1eZlwLtxTArvdCbZ5ZV/OqqYJa95RMlyDEJbDb7WRlZZGVlYXL5aKsrIwTJ05w8uRJ2tvb3ft1dXVRUFBAQUEBISEh7nDj2aNKTF4SaIS4Qv+J6cJ9GOjFtKc5BMQP3TFsBlz7Khx7GI59B9DQVQ2vbYal34GFXzVPdIQQF2Wz2cjIyCAjI4NbbrmFiooK9yjFra2t7v16eno4cuQIR44cwWazUVVVRXp6OpmZmcyYMUOqpiYhCTRCXKEwTI+n5UArUAbciWlPkzp0Z5sdlj4CCevgrQ9AT71pW3P4Iah9E9b+EUJktFQhLofNZiMtLY20tDRuuukmzp075548s7m52b2fy+XizJkznDlzhtdee43Q0FDS09PdASchIUF6TU0CEmiEeBuygN8Dd1mv3wQWAz/EdOse9hGZfJPpBbXrvaYtDUD1y1YV1NOQuNYPpRZi8lFKkZKSQkpKCjfccAM1NTXucNPQ0OC1b3d3NydPnuTkyZOAGQAwIyPDHXBknqnAJIFGiLfpXcB3MfM+aaAN00D4GeA3QObQA8JT4LrX4fDXofAHZl1nJby6EXK+D/O/ZAbrE0JcEaUUM2fOZObMmVx33XVs3bqVGTNmUFJSwtmzZ73a3QC0t7dz9OhRjh49Cpi5qQaqtTIyMoiMjBzpbcQEI4FGCB94CNgIfBQosta9jukR9SjwGcy8UG42B+Q+BtPXw1sfhN4m0P2Q//+g7k24+ncQHOfPSxBi0goNDSUnJ4ecnBy01jQ0NFBSUuJePLuEAzQ3N5Ofn09+fj4AiYmJ7nCTnp5OaGjoeFyGuAgJNEL4yDqgAPg2ZpoEF9CJ6QH1DKZH1LyhB8261VRB7bwXGvaadZXPw8vL4JpnYNpKv5RdiKlCKUVCQgIJCQmsXLkSrTU1NTXucFNWVuY1WjFAXV0ddXV17Nu3z/30ZyDgpKam4nA4xulqhCcJNEL4UBjwGKbH00eA49b6XZhpEh4BvsiQf3gRaXD9Dij4Kpz6qVnXUQqvrIPcf4fsz0gVlBBjxLN6au3atTidTs6dO+cOOBUVFV7dvrXWVFVVUVVVxa5du7Db7aSkpLgDzqxZs2T8m3EigUaIMbASOAh8z1r6gW7gK8CzwH9jGg+72YNh+U9MFdSej0JfC7j64ODnoHYHrH4CgmP8fBVCTD12u53U1FRSU1PZuHEjfX19lJeXuwNOdXU1Wmv3/k6nk7KyMsrKysjLy8PhcJCWluYOODNmzJAeVH4igUaIMRICPIxpNPxRIN9avx9YBnwD+BfA62H17LsgLgfevBuaDpl1Fc9BUwGsf9ZsE0L4jcPhYM6cOcyZMwcwg/eVlZW5A05dXZ3X/gPzURUXFwMQFhZGenq6O+BMmzZNAs4Y8VugUUrFY5oR3AjUA1/TWj85wn5fBD4LJADtwNPAg1rrfn+VVQhfygH2YrpyP4wZhK8P+CZmXqj/xgQct8hMuHEXHPoynP6FWddeDNuuhuX/AVn3SxWUEOMkLCyM+fPnM3/+fMD0kPJsYOw5/g2YADQw8B9AVFSUVw+qmBh58uor/nxC83PMZ3kS5jP+RaXUYa318SH7bQF+p7VutkLQc8DngB/7saxC+JQD0xPqTszTGqv5L4eBVZgnNd/APNUBwB4KK39upkbY+3HobwdXD+z/pKmCWvUrcEhXUiHGW2RkJEuWLGHJkiUANDU1eQWcjo4Or/3b2trcIxgDxMfHu8e/SU9PJyIiwu/XMFn4JdAopSKAdwOLtdbtwE6l1BbgPsxnuZvW+oznoZjOIln+KKcQY20hpoHwfwD/imlX48SMY/NX4HeYmbzd0u6FuFzYeTc0mw9Ayp401VHXPAuxXi1xhBDjLC4ujri4OJYtW4bWmrq6One4KS0tpaenx2v/xsZGGhsbOXTIVDEnJSW5A05aWhohISEjvY0Ygb+e0GQD/VrrIo91hzFDdwyjlHof8F9AFKZ66stjXkIh/MQOfAm4Dfg4sMNaXwisxfSCegQIHzggOhtu3GMaCJ95wqxrPQnbVsGKn8OcjyCEmHiUUkyfPp3p06ezevVqXC4X1dXV7oBTXl5Of793a4rz589z/vx59u7d6+6BlZaWRnp6OqmpqTIGziiUZ2vtMXsTpdYDz2qtZ3is+wTwfq31plGOmwt8EPi51rpmhO33Y0aYJykpaflTTz3l66IDpo5URor0n6l0v13A88nJ/HrOHLo9unrO6uzkwVOnuKqlxWv/pM7tZLf8BLseHAisOuxmTsd8Hpftyj/optI9nwjkfvvXRL3fLpeL1tZWmpubaWpqoq2tjYv9To6MjCQ2NpaYmBhiY2MJCpp4fXvG8n5v3rz5oNZ6xUjb/BVocoFdWutwj3VfBjZprW+7yLHvAe7RWt812n4rVqzQBw4c8El5h8rLy2PTpk1jcm4x3FS836XAJ4BXh6z/NPB9wOujoeWEqYJqOTG4LmaxqYKKmX9F7z8V7/l4kvvtX4Fyv3t7eykvL+fs2bOUlJRQUzPs7/hhZsyYQXp6unuSzrCwMD+UdHRjeb+VUhcMNP6KdkVAkFJqrtb6tLXuKgbHHRtNEDBnzEomxASQDmzH9Hj6Emb2bjAt6V8AngCuH9g5ZiHctA/2fQpK/2jWtRyDbStg5a8g4/1+LLkQwleCg4PJysoiK8s0G+3q6qK8vJzS0lLKysqoqakZ9gSnpqaGmpoa9uzZAwy2wRkIOOHh4cPeZ7LyS6DRWncopf4KPKKU+jiml9MdmCYDXqztW7TWtUqphcDXgG3+KKcQ40kBHwNuAh4AXrLWlwE3YNrb/AiIAQiKgDX/A0kb4cBnwNkN/R3w1gegbofp3m2XunYhAllYWBjz5s1j3jwzaUp3d7dXwBk6yB94t8EBmD59ulfAmcy9qPxZ+fbPmD9Aa4EG4FNa6+NW+5qXtdYDT9XXAd9VSkUCdZiBVb/hx3IKMa5SME9l/hczXkGTtf4J4GXgV8A7wYxFM+djEL8Sdv4TtFkPP4t/DQ37TBVUlHQQFGKyCA0NJTs7m+zsbAB6enq8Ak5VVdWwgFNbW0ttbS379u0DzESbAwFnsnUT91ug0Vo3YobhGLr+TTyaCGitpcuGmPIU8AFMNdOnMV26Ac4Bt1rb/gOIB4hbCjcfhH33Q5nVML6pwExwefVvIfVu/xZeCOEXISEhzJ07l7lz5wIm4FRUVHgFHM95qGBwos39+/cDkJCQ4BVwJmLj6Us18ZpHCyHcZmBGE34WE2wGBln/E/AK8AvgLgBHFKx90gzEd/AL4OqF/jbYeY+Z3DL3R2CX8SyEmMxCQkK82uD09vZ6BZxz584NCzj19fXU19cz0Klm2rRpXgEnKirK79dxpSTQCBEA7gY2A58HBuYLOY8ZrfJu4HFgulIw91MwbbXpBdV+1uxY9DjU74FrnoHIDP8XXggxLoKDg73moert7aWystIdcCorK4cFnIaGBhoaGjh48CAwOJLxQMCJjo72+3VcKgk0QgSIBEy7mnuBTwLV1vpngdeBnwHvBVT8MlMFteejUPl/ZqfGA6YKas3vIeUOfxddCDEBBAcHk5mZSWZmJmAm0hwacJxOp9cxQ0cyjouL8wo4E2kuKgk0QgSY24ENmO7dv7PWNQDvx8zk+ksgOTgW1v8FTv0MCh4EVx/0NcOOO2H+lyDn+2BzjHB2IcRU4XA43JNkggk4586dcwecioqKYQGnqamJpqYm8vPzAYiNjfUKOLGxsf6+DDcJNEIEoFhMl8F7MQPyVVjrtwD/AH4CfFgp1PzPQ8LVpi1NZ7nZ6eSPof4tWPc0RMz2e9mFEBOTw+EgPT2d9PR0APr7+4cFnKFTNTQ3N1NQUEBBQQFgAk5ISAixsbFcddVVKKX8Vn4JNEIEsJuAY5gZXn9prWvBzOj9FPBrIC1hNdySD299CKpeMDvVvwUv58CaP8Ksd/i93EKIiS8oKMg9fg2YgFNVVeUOOCPNRdXc3AzAjh07yMnJ8W95/fpuQgifi8b0droHMzCf1RSY7cBi4AfAAyHx2DY+D4X/Doe/BtoJvY3wj3fCwn9B6etHPrkQQliCgoJITU0lNTUVAKfTOSzg9PX1AbhDkF/L5/d3FEKMiU3AEeDrmDFqNNCOGdHyGeAJZWPOwgchcS3svBe6zpkDT3yf5UHPQsm3IfUesAePR/GFEAHGbrcze/ZsZs+ezfr163E6nVRXV/P666+zZMkSv5fH5vd3FEKMmQhM+5mdwDyP9XnAEuCngDNxnamCmnmTe3tk/xl46z7Ykg7Hvgvd9X4rsxBicrDb7aSkpJCamuruSeVPEmiEmITWAgWYtjUD/8i7gC8C64GToYmw6SW46rvecz51VcORr8Pzs2HfA9BS6N+CCyHEFZJAI8QkFQo8CuzFPJ0Z8BZmdtjHlI3+RQ/BHRWURH0UQmcM7uTsNnNCvbgQ3rgFqrfDkDlihBBiIpFAI8QktwI4AHyLwUZzPZinN1cDR0MTKIu6D+4ohTV/gLhc7xNUb4U3boKXlkDxE9Df5b/CCyHEJZJAI8QUEAx8GzgILPNYfxBYDvxXZibV9hDIuM+MMnxdnjWisMcYEi3HYd8n4PlUOPJN6KrxV/GFEOKiJNAIMYUsxVRBPYoJOQB9wNOpqaQBHwGOKQVJG2HD3+C2Isj+LARFDJ6kpx6O/ZsJNm99yMzsLYQQ40wCjRBTTBCmuqkAU+U0oA/4Paa9zc3Aq4COyoIVP4M7K82M3eGpgwe4+qDkD/ByLry6GSq3gPae6E4IIfxFAo0QU9QCYBfwF2BRS4vXtm3ADZjGw38AeoNjYcGX4fYzZtbuhDXeJ6vNgx13wN/nwanHoa99zMsvhBCeJNAIMYXZgLuAx/Pz2WV97znzyhHgQ0AG8BjQbAuC1Lvhxt1w4x5IvReUffCA9mI4+Fn422zI/wp0lPvtWoQQU5sEGiEEYMau+QtQBHwaCPfYVoWpppqNGcumFCBhNVzzFNx+FhY8CI6YwQP6mqHwh7Al04xKXL/HL9cghJi6JNAIIbxkAY8D5cB3gCSPbe2Y0YbnAO8B9gNEpELuD0w7mxWPQ2TW4AHaCeXPwPY1sG0NlD0DLu/J7IQQwhck0AghRjQN+FegDPgtsNBjmwt4GlgFbAS2AC5HJGR/Gm47BRu2QNJm7xM27IFd98KWOVD4I+ht9sNVCCGmCgk0QohRhQAfBY4BLwPXDdm+A7gD08j4V0CXskHKbXDd62bOqIwPgc1jwsvOcsh/EP6WAgc+B23FfrkOIcTkJoFGCHFJFIPdufOBDzA48jCYtjefBFIxg/jVAsTlwJrfwx1lsPibEJI4eEB/BxT9J/w9G/5xB5zPk+kVhBBXTAKNEOKy5QB/BEqAB4Foj231wMNAGvAAcAogbAYsfRjuLIfVT0DMIo8jNJzbAq9thq3L4ewfwNnrl+sQQkweEmiEEFcsBfgBUAH8GPN0ZkA38GtgPnA7pmpK20NhzsfgHUdh83aYeYv3CZvyYc+H4Pk0OPYd6K73x2UIISYBCTRCiLctGtOd+wzwZ8z8UJ7+jmk8vArTmLhfKZh5A2x+Cd55ArIeAHvY4AHdNXDkG/D8bNh7P7Sc8Mt1CCEClwQaIYTPBDHYnTsPuHXI9gPW9ixM9+82gJgFsOq/4M4KuOq7EDZz8ABnN5z5Dby4CF6/Caq2SjsbIcSIJNAIIXxOYZ7I/B0oBD6B6S01oAzzRGc28FXgHEDINFj0ENxeCmv+BPFDnvPUbIe8W0y4Kf419HeN+XUIIQKHBBohxJiaj2lLUw58E0jw2NaCaYOTDnwQOAxgD4aM98NN++H6HZDyLrwmZGgthH0PmOqo/Aeh4YA8tRFCSKARQvjHdEzvpzLgl8Bcj239mF5TOZhJMbcCWimYvh42/BVuL4Z5n4egyMGDehrMAH3bVprB+vK/KuFGiClMAo0Qwq/CMePVnAT+Bqwfsv1V4BZgKfB7oAcgMhOW/9RMr7DsxxCR7n1QRwkU/mAw3BT8CzQelHAjxBQigUYIMS5smBGGdwB7gXvw/kA6BnwEUx31KNAIEBwD878It52GjS9C5oe9J8UEE25OPAZbV8Dfs6xwc0jCjRCTnAQaIcS4G+jOXQx8Hojw2FYDPIRpQPw54CyALQhmvQOu/h3cVQsbXzBTLAwNN+1nrXCz3Ao3X5NwI8QkJYFGCDFhZGC6c1cA3weSPbZ1Av+JaXtzN7BnYIM9GGa900yxcNd5K9x8EBye4xdjhZvvW+FmLhQ8BI35Em6EmCQk0AghJpw4THfuEuB/gCUe21zAc8Aaa/3nMW1xmgDsIVa4+R/ryc3fIf2+EcLNGTjxKGxdZuaSKngImgok3AgRwCTQCCEmrGAGu3NvB24csv0Y8DPgXcA0IBf4Emb8mxZ7CMy6Fdb+wYSbDVsuEG6KTbh5OdeEm8P/KuFGiAAkgUYIMeEpTHfubZhw8yFM2PGkgQLgJ5i5o+KBlcBXgJfsIbSl3GaFm/Ow4XlI/wAERXmfpL0Yjn/PhJsX5sHhr0PTYQk3QgSAoPEugBBCXI6B7tyPA7uAN6zlAKY6aoDLWncA+CFgB1YAm+2hbE65nXUptxPh7Ibq7VD+DFRugf62wRO0nYbj3zVLVDak3g2p90DsElAeA/0JISYECTRCiIAUCdxkLQCtwJuYOaTeAA5hntoMcGK6h+/FNDgOAlZZ4WZzyu2sdXYTVr0Nyp6Bc1ugv33w4LaiIeHmHhNwJNwIMWFIoBFCTArRwDutBaAZM8bNwBOcw0P27wd2W8t3gWB7KFen3MGmlDvY7Ozh6qpthJY/fYFw8x2zRM8bDDcxiyXcCDGOJNAIISalWExbmtut1w14B5xjQ/bvtbbvAB6xhxA6+3bWzL6dzc4+Nte/xaozvyG48v+gv2PwoNZTcOzfzBI9f7BaKmaRhBsh/EwCjRBiSpiG6Q31Lut1LfAPTLjJw8wK7qnb2vaG3QFJGwhL2sA6l5PNrSfZXP40K079DEdfy+ABrSeHhJt7rDY3i8b4yoQQIIFGCDFFTccM0He39bqGwfY3bwCnh+zfBbxqs/Nq7CKIfYTIJQ9zTfd5Nte8xqYzv2ZZ3S6CtNPs3HoSjj1ilugFHm1uJNwIMVYk0AghBDADeI+1AJzDO+CcHbJ/u1JsDZvB1oz3Q8b7iXb1sb75KJvLn2Vz9Tauaj6MXbugtRCOPWyWmIUw+24i+2aBaz3Y7H67PiEmOwk0QggxglnA+60FoJzBcPOG9dpTq83Bi/HLeDF+GeQ8Smx/OxvO57G55lU2n3+DJc1HsbWcgJaHWQHw3JchcS0kXgOJ62HaKggK89flCTHpSKARQohLkIoZ0O9DmO7gpXgHnHND9m8OimTLrFvZMutWAKb11LPx/D/YVJvHhtodLGo5TlD1NqjeZg6wOSBuOUy/xoSchHUQmuCXaxNiMpBAI4QQl0lhJtLMAD6KCTjFeFdR1Qw5piEkgb+mvpu/pr4bgLD+TpY1HmJV4z5WNuxnVcM+Mhv2oBr2QOGPzEHRC6wnONeYoBORIb2nhLgACTRCCPE2Kcws4HOBT2ACzikGw00eUDfkmK6gcHZNv4Zd069xr4vvaXCHm1UN+1jZuJ+kM7+BM78xO4QlDwacxGsgdqm0wxHCIoFGCCF8TAHzreVTmIBzgsFw82Z3N7WhocOOawyZxrbkm9mWfLN7XWpHmQk3VtBZXvUSUeXPmI1BUR7tcK6x2uGEj/HVCTExSaARQogxpoBF1vIZIG/PHuZt2sR+YD+wz1qaRzi2PCKN8og0nks1HcyVdrGgpdDrKc7SY48Q7OqTdjhiSvNboFFKxQO/BW4E6oGvaa2fHGG/BzHt7tKs/X6htf6hv8ophBD+MBPvkYw1cIbBcLMfMx9V95DjtLJxInYRJ2IX8fs5HwEg2NlDblP+YHVV5RbmFv47NrS0wxFThj+f0PwcM7p4EpADvKiUOqy1Pj5kPwV8EDgCzAG2K6UqtNZP+bGsQgjhVwrIspb3Wev6gON4h5xjeM8qDtBrD2FvwtXsTbjavS66t4WVjfsHQ87hrzGrq0ra4YhJyy+BRikVAbwbWKy1bgd2KqW2APcB/+K5r9b6Bx4vTymlngfWARJohBBTigPz118OcL+1rgPz5MazqqpkhGNbg2N4bcb1vDbjeve6mZ1Vg1VV57aw4sg3idNOaYcjJgV/PaHJBvq11kUe6w4DG0c7SCmlgPXAr8awbEIIETAiMB+K6z3W1TMYcAa+Du1VBVAdnszz4Xfy/Ow73euyW08NPsUp+BdyWo4TGrNY2uGIgKO01mP/JkqtB57VWs/wWPcJ4P1a602jHPcwcCewSmvdM8L2+7H+cElKSlr+1FNj8xCnvb2dyMjIMTm3GE7ut//JPfevsb7fGjgfGsrJqCizREdzKiqKbvvFq5aCXH0sbT7i1X08tbODdsci2hwL6AhKpdORSr8tZszK72vy8+1fY3m/N2/efFBrvWKkbf4KNLnALq11uMe6LwObtNa3XeCYzwBfBtZrrSsv9h4rVqzQBw4c8FWRveTl5bFp06YxObcYTu63/8k996/xuN9OzIzinlVVR7Sm/xIaCEf0tbO88SA5zQXMbTtNVlsxWT2NpAWF44iaa2YXj54PMQsgPG3CtcmRn2//Gsv7rZS6YKDxV5VTERCklJqrtR6YxPYqTHu3YZRSH8W0rdlwKWFGCCHE6OzAYmv5iLWuWykK8K6qKhrh2A5HJDuSNrIjybuVgN3VT3pH6WDIqdlOVnsZWbjICIog2DPsRM+DoIgxuz4h/BJotNYdSqm/Ao8opT6OaeN2B7B26L5KqfcD3wM2a62HTnArhBDCR0KBq61lQDNwgMGQs1drqi/wFMdpC+JMVBZnorKGbbO5nKR1lpmgU7+HrNI/Mbe3mSxsZDhiCI3OGgw7oTOkK7l42/zZbfufgf8GaoEG4FNa6+NW+5qXtdYDFW7fAaYB+9XgD/iftNaf9GNZhRBiSooFrrcWAJTiHCbcnAJOA8VaU6ydnLNd+FeIy2anJDKTkshMXpl5o9c2pV3M7qwwYadyC3M7K8hy9ZKlHGQGxxEeZYWdqDlmsEAhLoHfAo3WuhHTwHfo+jeBSI/XGf4qkxBCiIubZS1uSoEKohM4ixVyBhZXH8XaRYUtGH2Bpy5a2dwjIL8+47rh79dZaaqx6naR1ddMlu4nyxZKVkgCEQNhJzhwGiUL/5CpD4QQQlyRcAbb5bhZT1S6MOPjDAYdJ6ed3RQrKLeH4VK2C573XHgK58JTyEvaPGzbzM4qspoPk9V5jqz+DuaiyQqKZE7odKKj5kJ4CoxybjF5SaARQgjhc2HAQmsBTM8nm2kU3AOUYoLOaa0pdnZS7OymWAVR6ojEqS7cS6o6PJnq8GTeHGHb9K7zZDXsJau7gbnOLrKUnazgaJy6Gt3fiZIBAyc1CTRCCCH8KgSYZy0oZXo/WT2g+hgMO8XOHor7Wih29XHaHkKJI5b+Udrt1IYlURuWxO6hG2ZAVF8raS0nSetrJt3ZTZqykxYUQXpwAmnhM5lucyDNkgObBBohhBAThgOYay3YQ8A+3b2tHyjHVF8V9zZyur/NNFAOCudsyDR6bcEXPG+bI5pjMdEcu8D2UGc3qT31pPd3kKZdpNlDSHPEkh4cR5pSJGO6vouJSwKNEEKIgBAEZAKZNjs3hiYCie5tTqASON3bQnF3nanGUjZOO6IoCZlOV1DYqOfutodSFJ4y4jg8AEGufmb3tZLm6iVN2UgLiiQ9KJw0IA2YDVw4Tgl/kEAjhBAi4NkxwSItOIbrh/SAeiMvj8Xr11LWVUVZTz2l/R2UaSdl9hDKHLGUhs+iJTh21PP324IoCYkfcSJQAKU1yc4u0nCRZgsl3RbkDjsDi7TgGVsSaIQQQkxqCki0B5MYmc6KyPThO/S109J0hLLuGkp7mylz9VCGnTJHBKVhyZRFpFEXOn34cR60UpwLCuccDG/DY0l09ZOmbKQrm1fQSbe+Skf0t0cCjRBCiKnNEUlM3FLMf0P0NEBrEZ3V2yl3P91xUWYPpSw8mdKIdMoi0qgKS0ZfpLt4nS2IOsxIzCOJ0Zo0pdwBZxYQZy2xQ77GIL/Ah5L7IYQQQlxIyDRIXEN44hrmA/MH1msXdFVBaxGce4He9rNU9jRS6uqjzOagLCLFHXbKItKoCJ9N/0VGPW5RiiPAkUssWhQm4MQycugZ6evA9xEw6Xp1SaARQgghLpeymUH8wlNgxrUEYzVYBnD1QXsptBVBUwGUP4uztZiq/hYr7KRRFp5GaWS6+/uyiDS6L9Jweag2a6m4guIHceEwNNK6oV8n4oQUEmiEEEIIX7I5IHquWXgnYBotzwZm93dyTVuxCTttRVD/FrQWoduKqFU2r7BzPjSJpuA4mh2xNAfHmu+try2OmItWcY2mH6i3lisRzoXDTkt6OhXAfVdcuisjgUYIIYTwl6BwiFtqFg8KSOppJKntNKvaikxVVuMhaD8D7Wehxzt6uFC0OqIHg47DO/A0B8d6rIunKWwGzSHTaHLE0hwUTtfbnPSz01rOjbQxPZ0zSKARQgghpqaQeAhZDQmrh2/ra4X2EnfAsbWfJdZa0luOm2quy9BjCx4MPxFzaI6eR1NUFs3hqTSFJ9McMp2mkHia7WE0KUUz0AQ0W4vrIuePu6zS+IYEGiGEEGKic0RD3FVmGcrlhK5K8ySn/Sy0nRn8vuOs6ak1RIirl6TuWpK6a6H1FFS/NPL72sMgMgMiMiEyE6LmoCMzaYvMojkinaagUK+w0wQUlJZyfXq6r678kkmgEUIIIQKZzQ4RaWYZYYZyelsGA457sUJPRxno/guf29kFLSfMYlFAtLWkhiWboONe5nDodDPLku6GsCRfX+moJNAIIYQQk1lwDMTnmmUoVz90VnqHHM/Q09s0+rm7qsxSt9O9ahnAa4/DrSd9ehkXI4FGCCGEmKpsQRCZbhauHb69t8lquzNC4OkoA+0c+bwRmWNY6JFJoBFCCCHEyILjID4O4pcN3+bqh85y75DTdoa2msNExQ0bc3nMSaARQgghxOWzBQ22nfFwMC+PTTmb/F8cv7+jEEIIIYSPSaARQgghRMCTQCOEEEKIgCeBRgghhBABTwKNEEIIIQKeBBohhBBCBDwJNEIIIYQIeBJohBBCCBHwJNAIIYQQIuBJoBFCCCFEwJNAI4QQQoiAJ4FGCCGEEAFPAo0QQgghAp4EGiGEEEIEPAk0QgghhAh4EmiEEEIIEfAk0AghhBAi4EmgEUIIIUTAk0AjhBBCiIAngUYIIYQQAU9prce7DD6hlKoDysbo9AlA/RidWwwn99v/5J77l9xv/5L77V9jeb/TtNaJI22YNIFmLCmlDmitV4x3OaYKud/+J/fcv+R++5fcb/8ar/stVU5CCCGECHgSaIQQQggR8CTQXJpfj3cBphi53/4n99y/5H77l9xv/xqX+y1taIQQQggR8OQJjRBCCCECngQaIYQQQgS8KRtolFLxSqn/U0p1KKXKlFLvu8B+Sin1mFKqwVoeU0opj+05SqmDSqlO62uO3y4igPjifiulspVSzyul6pRSjUqpbUqpef69ksDgq59vj/0+qJTSSqmPj33pA48PP0/sSqnvKKWqlFJtSql8pVSs3y4kgPjwnl+rlDqklGpVSp1VSt3vv6sIHJdxvzcrpd5QSrUopUpH2J5ube9USp1USl3vqzJO2UAD/BzoBZKA9wO/VEotGmG/+4E7gauApcBtwAMASqlg4HngT0Ac8D/A89Z64e1t328gFtgCzLPOsw9z/8VwvrjfACil4oCHgONjWN5A56v7/TCwFlgDRAP3Ad1jVurA5ovPcAfwf8CvgBjgXuDHSqmrxrrwAehS73cH8N/Agxc4z5+BfGAa8K/Ac0qpEQfKu2xa6ym3ABHW/5hsj3V/BL4/wr67gfs9Xn8M2GN9fyNwDqtxtbWuHLh5vK9xIi2+ut8j7BsPaGDaeF/jRFp8fb+B/wL+GcgDPj7e1zfRFh9+nsQB7cCc8b6mib748J4nWZ8h4R7b9wPvHe9rnEjL5dxvj+3XA6VD1mUDPUCUx7o3gU/6opxT9QlNNtCvtS7yWHcYGCltLrK2jbTfIuCItv6vWI5c4DxTma/u91AbgBqtdYNPSjl5+Ox+K6VWASswoUaMzFf3ewnQD/yTUqpGKVWklPr0WBR4EvDJPddan8c8MfiIVd23BkgDdo5JqQPX5dzv0SwCzmqt297meUYU5IuTBKBIoHXIuhYg6gL7tgzZL9Kqgx26bbTzTGU+ud+ewVEplYJ5BPolH5d1MvDVz7cN+AXwGa21a4SmNcLw1f1OwVR7ZAMZwFzgNaVUkdb6FZ+XOrD58jPlz8ATwH9Y2z+lta7wcXkD3eXc74udZ6TfmbOusFxepuoTmnZM/bSnaKDtEvaNBtqtfwiXc56pzFf3GwCrvnU78Aut9Z99XNbJwFf3+58xTyD3jEkpJw9f3e8ua90jWusurfUR4CngHT4u72Tgk3uulJqPuccfBIIxTwq+opR6p++LHNB89btuTH9nTtVAUwQEKaXmeqy7ipEbPR63to2033Fg6ZBeIUsvcJ6pzFf3e6CB6nZgi9b6u2NQ1snAV/f7OuBdVvVHDaax6r8rpR4fgzIHMl/d7yPWV88qbBn5dGS+uueLgSKt9TattUtrfQp4EbhlDMocyC7nfo/mOJCplPJ8snMl5xnZeDc2GsdGTk9hHjVGAOswj70WjbDfJ4FCzCOxZOvGf9LaFgyUAZ8HQoDPWK+Dx/v6Jtrio/sdjenZ9Ph4X89EX3x0v2OBGR7LbkwVX8x4X99EW3xxv63tOzA9bkKABUAtcN14X99EXHz0Mz4H89TgWkBZr4vxaEQsy2XfbxsQigmFZdb3wR7b9wA/sta/C2gGEn1SxvG+SeP4Pyce+Bumi1k58D5r/XrM48iB/RTwA6DRWn6Ad6+mXOAg5nHxISB3vK9tIi6+uN/AhzB/sXZYH0IDS+p4X99EW3z18z3knHlIL6cxvd/WL92t1s/1WeCB8b62ibr48J7fAxzDVHtUAo8BtvG+vom2XMb93mR9TnsueR7b063Pki7gFHC9r8ooczkJIYQQIuBN1TY0QgghhJhEJNAIIYQQIuBJoBFCCCFEwJNAI4QQQoiAJ4FGCCGEEAFPAo0QQgghAp4EGiHEhKWUekgp9YT1fbpSSiulfDIHnVLqUaXUFy5x331KKZl0VogJTMahEUIEBKVUOlACOLTW/W/zXIlAAZClte66yO4ope4B7tVav/vtvK8QYuzIExohxFT0YeClSwkzli3AZqXUjLErkhDi7ZBAI4TwGaVUslLqL0qpOqVUiVLqc9b6byulnlNKPa2UalNKHVJKXeVx3FeVUuesbaeUUtd5HPenUd5ri1KqUSlVrJT6hMe2byulnlFK/cE653Gl1AqPw28B/jHkfDcqpfYqpZqVUueVUrcObNNad2OmOLnJF/dJCOF7EmiEED6hlLIBfwcOY+Ykug74glJqIATcATyLmRPmSeBvSimHUmoeZmLXlVrrKExoKL2Et3wKM/dOMvBPwPeUUtd6bL/d2icW84TFc5bwJZh5ZAbKvsza9+vANCAT2Dnk/QrxnrVZCDGBSKARQvjKSsysuY9orXu11meB3wDvsbYf1Fo/p7XuA36MmW33asCJmV16oVLKobUu1VqfGe2NlFKzMTP+flVr3a21LgCeAD7osdtOrfVLWmsn8Ee8w0gsZjLCAZ8Afq21fkVr7dRad2itm4e8bZt1nBBiApJAI4TwlTQg2aqyaVZKNQMPAUnW9oqBHbXWLqynK1rrYuALwLeBWqXUU0qp5Iu8VzLQqLX2DCVlmCdDA2o8vu8EQj16SDUBUUPK/jnPsiul3jfkPaOA5ouUSwgxTiTQCCF8pQIo0VrHeixRWut3WNtnD+xoVU+lAFUAWusntdbXYIKFBh67yHtVAfFKKc9Qkgqcu8SyHgGyPV6XAT8bUvYnhxyzAFOdJoSYgCTQCCF8ZR/QZjXwDVNK2ZVSi5VSK63ty5VSd1lPSb4A9AB7lFLzlFLXKqVCgG6gC3CN9kZa6wpgN/CoUipUKbUU+BgwYgPiEbwEbPR4/RvgfqXUDVa5Q5RScQMblVKhwHLglUs8vxDCzyTQCCF8wmqrciuQgxkvph7TriXG2uV54F5Mdc99wF1We5oQ4PvW/jXAdOBrl/CW7wXSMU9r/g/4ltb61Uss7h+AdyilwqyyH8K09fku0ACcxbTRGXAbkKe1rrrE8wsh/EwG1hNCjDml1Lcxg9h9YLzLMkAp9T2gVmv900vYdy/wMa31sTEvmBDiivhkCHEhhAg0WuuHLmPf1WNZFiHE2ydVTkIIIYQIeFLlJIQQQoiAJ09ohBBCCBHwJNAIIYQQIuBJoBFCCCFEwJNAI4QQQoiAJ4FGCCGEEAFPAo0QQgghAt7/Bz7XNBQt8R3vAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 576x576 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "Attack_info_df = pd.read_csv('Whitebox_Adversarial_Attack_Accuracy_final.csv')\n",
    "Attack_info_df['cutout_mixup_acc'] = cutout_mixup_FGSM_acc_list\n",
    "Attack_info_df.to_csv('Whitebox_Adversarial_Attack_Accuracy_final.csv',index=False,encoding='utf-8')\n",
    "\n",
    "\n",
    "\n",
    "plt.rcParams.update({'font.size': 12})\n",
    "plt.figure(figsize=(8,8))\n",
    "plt.plot(Attack_info_df['epsilon'], Attack_info_df['original_acc'],     color='orange',label='Original',lw=3, alpha=1)\n",
    "plt.plot(Attack_info_df['epsilon'], Attack_info_df['cutout_acc'],       color='cyan',label='Cutout',lw=3, alpha=1)\n",
    "plt.plot(Attack_info_df['epsilon'], Attack_info_df['mixup_acc'],        color='purple',label='Mixup',lw=3, alpha=1)\n",
    "plt.plot(Attack_info_df['epsilon'], Attack_info_df['aux_rot_acc'],      color='gray',label='SSRP',lw=3, alpha=1)\n",
    "plt.plot(Attack_info_df['epsilon'], Attack_info_df['cutout_mixup_acc'], color='black',label='Cutout+ Mixup',lw=3, alpha=1)\n",
    "plt.xlabel('epsilon(ϵ)')\n",
    "plt.ylabel('Accuracy')\n",
    "plt.grid()\n",
    "plt.title('Whitebox Adversarial Attack Accuracy Vs. epsilon(ϵ)')\n",
    "plt.legend()\n",
    "plt.tight_layout()\n",
    "plt.savefig('Whitebox_Adversarial_Attack_Accuracy_final.png',bbox_inches=\"tight\",dpi=170)\n",
    "plt.show()\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8f7805ed",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.12"
  },
  "varInspector": {
   "cols": {
    "lenName": 16,
    "lenType": 16,
    "lenVar": 40
   },
   "kernels_config": {
    "python": {
     "delete_cmd_postfix": "",
     "delete_cmd_prefix": "del ",
     "library": "var_list.py",
     "varRefreshCmd": "print(var_dic_list())"
    },
    "r": {
     "delete_cmd_postfix": ") ",
     "delete_cmd_prefix": "rm(",
     "library": "var_list.r",
     "varRefreshCmd": "cat(var_dic_list()) "
    }
   },
   "types_to_exclude": [
    "module",
    "function",
    "builtin_function_or_method",
    "instance",
    "_Feature"
   ],
   "window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}