Knowledge_Distillation.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "dbb89e7f",
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import json\n",
    "import time\n",
    "import random\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import pydicom\n",
    "from PIL import Image\n",
    "from sklearn.model_selection import train_test_split\n",
    "import warnings\n",
    "warnings.filterwarnings(\"ignore\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "a94298f9",
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.optim as optim\n",
    "import torchvision\n",
    "import torch.nn.functional as F\n",
    "from torchvision import transforms\n",
    "from torch.utils.data import Dataset, DataLoader\n",
    "import torchvision.models as models\n",
    "from sklearn.metrics import precision_recall_fscore_support, roc_auc_score\n",
    "import matplotlib.pyplot as plt\n",
    "import sklearn.metrics as metrics"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "760c4a8b",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "device(type='cuda')"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "os.environ[\"CUDA_DEVICE_ORDER\"]=\"PCI_BUS_ID\"  \n",
    "os.environ[\"CUDA_VISIBLE_DEVICES\"]=\"3\"\n",
    "device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n",
    "device"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "77711eff",
   "metadata": {},
   "outputs": [],
   "source": [
    "config = dict(\n",
    "    saved_path=\"saved_models/efficientb4_cifar10.pt\",\n",
    "    best_saved_path = \"saved/random_best.pt\",\n",
    "    lr=0.001, \n",
    "    EPOCHS = 3,\n",
    "    BATCH_SIZE = 32,\n",
    "    IMAGE_SIZE = 132,\n",
    "    TRAIN_VALID_SPLIT = 0.2,\n",
    "    device=device,\n",
    "    SEED = 42,\n",
    "    pin_memory=True,\n",
    "    num_workers=2,\n",
    "    USE_AMP = True,\n",
    "    channels_last=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "8776495c",
   "metadata": {},
   "outputs": [],
   "source": [
    "random.seed(config['SEED'])\n",
    "# If you or any of the libraries you are using rely on NumPy, you can seed the global NumPy RNG \n",
    "np.random.seed(config['SEED'])\n",
    "# Prevent RNG for CPU and GPU using torch\n",
    "torch.manual_seed(config['SEED'])\n",
    "torch.cuda.manual_seed(config['SEED'])\n",
    "torch.backends.cudnn.benchmarks = True\n",
    "torch.backends.cudnn.deterministic = True\n",
    "\n",
    "torch.backends.cuda.matmul.allow_tf32 = True\n",
    "\n",
    "# The flag below controls whether to allow TF32 on cuDNN. This flag defaults to True.\n",
    "torch.backends.cudnn.allow_tf32 = True"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "3e0a06e5",
   "metadata": {},
   "outputs": [],
   "source": [
    "data_transforms = {\n",
    "    'train': transforms.Compose([\n",
    "        transforms.RandomResizedCrop((config['IMAGE_SIZE'],config['IMAGE_SIZE'])),\n",
    "        transforms.RandomHorizontalFlip(),\n",
    "        transforms.RandomRotation(degrees=15),\n",
    "        transforms.ToTensor(),\n",
    "        transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225])\n",
    "    ]),\n",
    "    'val': transforms.Compose([\n",
    "        transforms.Resize((config['IMAGE_SIZE'],config['IMAGE_SIZE'])),\n",
    "        transforms.ToTensor(),\n",
    "        transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225])\n",
    "    ]),\n",
    "    'test': transforms.Compose([\n",
    "        transforms.Resize((config['IMAGE_SIZE'],config['IMAGE_SIZE'])),\n",
    "        transforms.ToTensor(),\n",
    "        transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225])\n",
    "    ]),\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "8389e20a",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Files already downloaded and verified\n",
      "Files already downloaded and verified\n"
     ]
    }
   ],
   "source": [
    "# Load data\n",
    "train_data = torchvision.datasets.CIFAR10(root='../Images', train=True, download=True, transform=data_transforms['test'])\n",
    "test_data = torchvision.datasets.CIFAR10(root='../Images', train=False, download=True, transform=data_transforms['test'])\n",
    "valid_data = test_data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "82f304d4",
   "metadata": {},
   "outputs": [],
   "source": [
    "train_dl = torch.utils.data.DataLoader(train_data, batch_size=32,shuffle=True, num_workers = config['num_workers'],\n",
    "                                          pin_memory = config['pin_memory'])\n",
    "\n",
    "\n",
    "test_dl = torch.utils.data.DataLoader(test_data, batch_size=32,shuffle=True, num_workers = config['num_workers'],\n",
    "                                          pin_memory = config['pin_memory'])\n",
    "valid_dl = torch.utils.data.DataLoader(valid_data, batch_size=32,shuffle=True, num_workers = config['num_workers'],\n",
    "                                          pin_memory = config['pin_memory'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "a2d105a0",
   "metadata": {},
   "outputs": [],
   "source": [
    "def train_model(model,criterion,optimizer,num_epochs=10):\n",
    "\n",
    "    since = time.time()                                            \n",
    "    batch_ct = 0\n",
    "    example_ct = 0\n",
    "    best_acc = 0.3\n",
    "    for epoch in range(num_epochs):\n",
    "        print('Epoch {}/{}'.format(epoch, num_epochs - 1))\n",
    "        print('-' * 10)\n",
    "        run_corrects = 0\n",
    "        #Training\n",
    "        model.train()\n",
    "        for x,y in train_dl: #BS=32 ([BS,3,224,224], [BS,4])            \n",
    "            if config['channels_last']:\n",
    "                x = x.to(config['device'], memory_format=torch.channels_last) #CHW --> #HWC\n",
    "            else:\n",
    "                x = x.to(config['device'])\n",
    "            y = y.to(config['device']) #CHW --> #HWC\n",
    "            \n",
    "            \n",
    "            \n",
    "            optimizer.zero_grad()\n",
    "            #optimizer.zero_grad(set_to_none=True)\n",
    "            ######################################################################\n",
    "            \n",
    "            train_logits = model(x) #Input = [BS,3,224,224] (Image) -- Model --> [BS,4] (Output Scores)\n",
    "            \n",
    "            _, train_preds = torch.max(train_logits, 1)\n",
    "            train_loss = criterion(train_logits,y)\n",
    "            train_loss = criterion(train_logits,y)\n",
    "            run_corrects += torch.sum(train_preds == y.data)\n",
    "            \n",
    "            train_loss.backward() # Backpropagation this is where your W_gradient\n",
    "            loss=train_loss\n",
    "\n",
    "            optimizer.step() # W_new = W_old - LR * W_gradient \n",
    "            example_ct += len(x) \n",
    "            batch_ct += 1\n",
    "            if ((batch_ct + 1) % 400) == 0:\n",
    "                train_log(loss, example_ct, epoch)\n",
    "            ########################################################################\n",
    "        \n",
    "        #validation\n",
    "        model.eval()\n",
    "        running_loss = 0.0\n",
    "        running_corrects = 0\n",
    "        total = 0\n",
    "        # Disable gradient calculation for validation or inference using torch.no_rad()\n",
    "        with torch.no_grad():\n",
    "            for x,y in valid_dl:\n",
    "                if config['channels_last']:\n",
    "                    x = x.to(config['device'], memory_format=torch.channels_last) #CHW --> #HWC\n",
    "                else:\n",
    "                    x = x.to(config['device'])\n",
    "                y = y.to(config['device'])\n",
    "                valid_logits = model(x)\n",
    "                _, valid_preds = torch.max(valid_logits, 1)\n",
    "                valid_loss = criterion(valid_logits,y)\n",
    "                running_loss += valid_loss.item() * x.size(0)\n",
    "                running_corrects += torch.sum(valid_preds == y.data)\n",
    "                total += y.size(0)\n",
    "            \n",
    "        epoch_loss = running_loss / len(valid_data)\n",
    "        epoch_acc = running_corrects.double() / len(valid_data)\n",
    "        train_acc = run_corrects.double() / len(train_data)\n",
    "        print(\"Train Accuracy\",train_acc.cpu())\n",
    "        print(\"Validation Loss is {}\".format(epoch_loss))\n",
    "        print(\"Validation Accuracy is {}\\n\".format(epoch_acc.cpu()))\n",
    "        if epoch_acc.cpu()>best_acc:\n",
    "            print('One of the best validation accuracy found.\\n')\n",
    "            #torch.save(model.state_dict(), config['best_saved_path'])\n",
    "            best_acc = epoch_acc.cpu()\n",
    "\n",
    "            \n",
    "    time_elapsed = time.time() - since\n",
    "    print('Training complete in {:.0f}m {:.0f}s'.format(\n",
    "        time_elapsed // 60, time_elapsed % 60))\n",
    "    \n",
    "    torch.save(model.state_dict(), config['saved_path'])\n",
    "\n",
    "    \n",
    "def train_log(loss, example_ct, epoch):\n",
    "    loss = float(loss)\n",
    "    print(f\"Loss after \" + str(example_ct).zfill(5) + f\" examples: {loss:.3f}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "559b82f1",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<All keys matched successfully>"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "efficientnet = models.efficientnet_b0(pretrained = True)\n",
    "efficientnet.classifier[1] = nn.Linear(in_features = 1280, out_features = 10, bias = True)\n",
    "model = efficientnet\n",
    "criterion = nn.CrossEntropyLoss()\n",
    "model = model.to(config['device'])\n",
    "optimizer = optim.Adam(model.parameters(),lr=config['lr'])\n",
    "model.load_state_dict(torch.load('saved_models/child.pt'))\n",
    "#train_model(model,criterion,optimizer,num_epochs=8)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "da57b159",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Test Loss is 0.29729179959520696\n",
      "Test Accuracy is 0.9142\n",
      "\n",
      "AUROC:\n",
      "0.9957011777777778\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.92      0.93      0.92      1000\n",
      "           1       0.97      0.95      0.96      1000\n",
      "           2       0.94      0.84      0.89      1000\n",
      "           3       0.82      0.86      0.84      1000\n",
      "           4       0.92      0.90      0.91      1000\n",
      "           5       0.85      0.89      0.87      1000\n",
      "           6       0.89      0.95      0.92      1000\n",
      "           7       0.98      0.90      0.94      1000\n",
      "           8       0.92      0.97      0.95      1000\n",
      "           9       0.95      0.94      0.95      1000\n",
      "\n",
      "    accuracy                           0.91     10000\n",
      "   macro avg       0.92      0.91      0.91     10000\n",
      "weighted avg       0.92      0.91      0.91     10000\n",
      "\n"
     ]
    }
   ],
   "source": [
    "def evaluation(model,test_dl):\n",
    "    model.eval()\n",
    "    running_loss = 0.0\n",
    "    running_corrects = 0\n",
    "    total = 0\n",
    "    preds = []\n",
    "    pred_labels = []\n",
    "    labels = []\n",
    "\n",
    "    with torch.no_grad():\n",
    "                for x,y in test_dl:\n",
    "                    x = x.to(config['device'])\n",
    "                    y = y.to(config['device']) #CHW --> #HWC\n",
    "                    valid_logits = model(x)\n",
    "                    predict_prob = F.softmax(valid_logits)\n",
    "                    _,predictions = predict_prob.max(1)\n",
    "                    predictions = predictions.to('cpu')\n",
    "\n",
    "                    _, valid_preds = torch.max(valid_logits, 1)\n",
    "                    valid_loss = criterion(valid_logits,y)\n",
    "                    running_loss += valid_loss.item() * x.size(0)\n",
    "                    running_corrects += torch.sum(valid_preds == y.data)\n",
    "                    total += y.size(0)\n",
    "                    predict_prob = predict_prob.to('cpu')\n",
    "\n",
    "                    pred_labels.extend(list(predictions.numpy()))\n",
    "                    preds.extend(list(predict_prob.numpy()))\n",
    "                    y = y.to('cpu')\n",
    "                    labels.extend(list(y.numpy()))\n",
    "\n",
    "    epoch_loss = running_loss / len(test_data)\n",
    "    epoch_acc = running_corrects.double() / len(test_data)\n",
    "    print(\"Test Loss is {}\".format(epoch_loss))\n",
    "    print(\"Test Accuracy is {}\".format(epoch_acc.cpu()))\n",
    "    return np.array(labels),np.array(pred_labels),np.array(preds)\n",
    "    \n",
    "\n",
    "labels, pred_labels,preds = evaluation(model, test_dl)\n",
    "#print(metrics.precision_recall_fscore_support(np.array(labels), np.array(pred_labels)))\n",
    "print('\\nAUROC:')\n",
    "print(metrics.roc_auc_score(np.array(labels), np.array(preds), multi_class='ovr'))\n",
    "print(metrics.classification_report(labels,pred_labels))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "f53622da",
   "metadata": {},
   "outputs": [],
   "source": [
    "def calculate_disparate_impact(model, dataloader):\n",
    "    #device = next(model.parameters()).device\n",
    "    model.eval()\n",
    "    with torch.no_grad():\n",
    "        class_counts = torch.zeros((10,))\n",
    "        correct_counts = torch.zeros((10,))\n",
    "        for images, labels in dataloader:\n",
    "            images, labels = images.to(device), labels.to(device)\n",
    "            outputs = model(images)\n",
    "            preds = torch.argmax(outputs, dim=1)\n",
    "            for i in range(10):\n",
    "                class_mask = labels == i\n",
    "                class_counts[i] += torch.sum(class_mask)\n",
    "                correct_counts[i] += torch.sum(preds[class_mask] == labels[class_mask])\n",
    "\n",
    "        proportions = correct_counts / class_counts\n",
    "        reference_proportion = torch.mean(proportions)\n",
    "        max_proportion = torch.max(proportions)\n",
    "        \n",
    "        return reference_proportion / max_proportion"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "3f083046",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Disparate Impact: 0.9376\n"
     ]
    }
   ],
   "source": [
    "device = 'cpu'\n",
    "model = model.to('cpu')\n",
    "disparate_impact = calculate_disparate_impact(model, test_dl)\n",
    "print(f\"Disparate Impact: {disparate_impact:.4f}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "9fe89b33",
   "metadata": {},
   "source": [
    "# Bias Observation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "e1091652",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Bias mean: 0.10364648806415327\n",
      "Bias std: 0.927578089418026\n"
     ]
    }
   ],
   "source": [
    "# Calculate the mean and standard deviation of the bias values of the layers\n",
    "bias_mean = []\n",
    "bias_std = []\n",
    "for name, param in model.named_parameters():\n",
    "    if 'bias' in name:\n",
    "        bias_mean.append(param.data.mean().item())\n",
    "        bias_std.append(param.data.std().item())\n",
    "print(f'Bias mean: {np.mean(bias_mean)}')\n",
    "print(f'Bias std: {np.mean(bias_std)}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "81308b16",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAlYAAAFlCAYAAAApo6aBAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjQuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8rg+JYAAAACXBIWXMAAAsTAAALEwEAmpwYAABYiUlEQVR4nO3de1zVVb7/8dcSPJGXsixIJUcdyVDYbDiIKUOBhHl3kjJvE5rOaI7SlMe0i/6so4+wY3JGTx6aOV3IbMbSSajT0UCxtBuhbYzGW5IXiPGCQaZSuvf39wewB2QrIlvx8n4+HjzY+/td37U+XzfBp7XWdy1jWRYiIiIi0njNmjoAERERkSuFEisRERERL1FiJSIiIuIlSqxEREREvESJlYiIiIiXKLESERER8RLfpg4A4KabbrI6derU1GGIiIiI1Gvz5s2HLcu62dO5SyKx6tSpE3l5eU0dhoiIiEi9jDF7z3ROQ4EiIiIiXqLESkRERMRLlFiJiIiIeMklMcdKRESuTidPnqSoqIiKioqmDkWkDj8/PwIDA2nevPk5X6PESkREmkxRURGtW7emU6dOGGOaOhwRN8uyKC0tpaioiM6dO5/zdRoKFBGRJlNRUUHbtm2VVMklxxhD27ZtG9ybqsRKRESalJIquVSdz8+mEisREbmqLV68mODgYMaMGdOg6/bs2cObb755gaLyzLIskpOT6dq1KzabjS1btngs99RTT3HrrbfSqlWrM9b12muvMXXqVI/n+vTp45V4r0aaYyUiIpeM1KydXq3v0YTb6i2zdOlSsrOzCQwMbFDd1YnV6NGjG3Sd0+nEx8enQddU+7//+z927drFrl27+Pzzz3n44Yf5/PPP65QbMmQIU6dOJSgo6Lza+eSTT87rOlGPlYiIXMUmT55MYWEhAwYMYP78+Tz00ENERUURHh5ORkYGUJlAxcTEEBERQUREhDvpmDVrFhs3bsRut5OamlqnB2jw4MFs2LABgFatWjF9+nTCwsL49NNPeeONN4iKisJutzNp0iScTidOp5Nx48YREhJCaGgoqampdeLNyMjgwQcfxBjDHXfcQVlZGSUlJXXK3XHHHbRr167e+9+/fz+xsbEEBQXxzDPPuI9X93T9+OOPxMfHExERQWhoqPvf5NixYwwaNIiwsDBCQkJYsWLFOf6LX/mUWImIyFUrLS2N9u3bk5OTw7Fjx+jbty+5ubnk5OQwY8YMjh07hr+/P1lZWWzZsoUVK1aQnJwMQEpKCjExMTgcDh599NGztnPs2DF69epFfn4+bdu2ZcWKFXz88cc4HA58fHxYvnw5DoeD4uJiCgoK+Oqrrxg/fnydeoqLi7n11lvd7wMDAykuLj7v+8/NzWXVqlVs3bqVt99+u872cn5+frzzzjts2bKFnJwcpk+fjmVZrFmzhvbt25Ofn09BQQH9+/c/7xiuNBoKFBERAT744AMyMzNZuHAhUPnE4r59+2jfvj1Tp051J0E7dzZ8uNLHx4fExEQA1q1bx+bNm+nZsycAJ06cwN/fnyFDhlBYWMi0adMYNGgQ/fr1897NnUFCQgJt27YFYPjw4WzatInIyEj3ecuyePLJJ/noo49o1qwZxcXFHDhwgNDQUKZPn87MmTMZPHgwMTExFzzWy4USKxERESqTiFWrVtGtW7dax+fOnUtAQAD5+fm4XC78/Pw8Xu/r64vL5XK/r/mYvp+fn3telWVZJCUl8dxzz9WpIz8/n7Vr15KWlsZbb73FK6+8Uut8hw4d2L9/v/t9UVERHTp0aPjNVjn9qbfT3y9fvpxDhw6xefNmmjdvTqdOnaioqOC2225jy5YtvP/++zz99NPEx8czZ86c847jSqKhQBG57JVn7T3rl8i5uOeee1iyZAmWZQHw5ZdfAlBeXk67du1o1qwZy5Ytw+l0AtC6dWuOHj3qvr5Tp044HA5cLhf79+8nNzfXYzvx8fGsXLmSgwcPAnDkyBH27t3L4cOHcblcJCYmMm/ePI9P/A0dOpTXX38dy7L47LPPuP76689pLtWZZGVlceTIEU6cOMHq1auJjo6udb68vBx/f3+aN29OTk4Oe/dW/vf03Xff0aJFC8aOHcuMGTPO+HTi1Ug9ViIiIsDs2bP5wx/+gM1mw+Vy0blzZ9577z2mTJlCYmIir7/+Ov3796dly5YA2Gw2fHx8CAsLY9y4cfzhD3+gc+fOdO/eneDgYCIiIjy20717d+bNm0e/fv1wuVw0b96cF198kWuvvZbx48e7e72qe7TS0tKAyon2AwcO5P3336dr1660aNGCV1991V2v3W7H4XAA8Pjjj/Pmm29y/PhxAgMDmThxInPnziUzM5O8vDyeffZZAKKiokhMTKSoqIixY8fWGgYEGDNmDEOGDCE0NJTIyEhuv/12AL766itmzJhBs2bNaN68Of/93//tpU/h8meqM/OmFBkZaZ0+YU5E5FzV1yt1fcIvLlIk0lDbtm0jODi4qcMQOSNPP6PGmM2WZUV6Kq+hQBEREREvUWIlIiIi4iVKrERERES8RImViIiIiJcosRIRERHxEiVWIiIiIl6ixEpERK5qixcvJjg4mDFjxjTouj179vDmm29eoKg82759O7179+aaa65xb73jybfffkuvXr3o2rUrDzzwAD///HOdMqdvGl1T9SbMDRUbG1tnv8HTHTlyhISEBIKCgkhISOD777/3WK5///60adOGwYMHn7GucePGsXLlyjrH8/Ly3Hs6XmxaIFRERC4dOXW3eWmUuCfqLbJ06VKys7MJDAxsUNXVidXo0aMbdJ3T6XRvb9NQN954I4sXL2b16tVnLTdz5kweffRRRo4cyeTJk3n55Zd5+OGHz6tNb0tJSSE+Pp5Zs2aRkpJCSkoKCxYsqFNuxowZHD9+nJdeeqnBbURGRtZZ7PRiUY+ViIhctSZPnkxhYSEDBgxg/vz5PPTQQ0RFRREeHk5GRgZQmUDFxMQQERFBREQEn3zyCQCzZs1i48aN2O12UlNT6/QADR48mA0bNgCVPUDTp08nLCyMTz/9lDfeeIOoqCjsdjuTJk3C6XTidDoZN24cISEhhIaGkpqaWidef39/evbsSfPmzc94T5ZlsX79eu677z4AkpKSzpiI7d+/n9jYWIKCgnjmmWc81jVjxgx3TCtWrHCfW7BgAaGhoYSFhTFr1qxa17lcLsaNG8fTTz9dp86MjAySkpLqjS0+Pp7WrVuf8T6rZWdnExkZyW233cZ7770HwIYNG9w9Xbm5ufTu3Zvw8HD69OnDjh07APj666/dn4HNZmPXrl31tnUu1GMlIiJXrbS0NNasWUNOTg6LFi2ib9++vPLKK5SVlREVFcXdd9+Nv78/WVlZ+Pn5sWvXLkaNGkVeXh4pKSksXLjQ/cf8tddeO2M7x44do1evXrzwwgts27aNBQsW8PHHH9O8eXOmTJnC8uXL6dGjB8XFxRQUFABQVlZ2XvdUWlpKmzZt8PWt/BMfGBhIcXGxx7K5ubkUFBTQokULevbsyaBBg2r19Pztb3/D4XCQn5/P4cOH6dmzJ3feeScOh4OMjAw+//xzWrRowZEjR9zXnDp1ijFjxhASEsJTTz1Vp80DBw649ze85ZZbOHDgwHndZ7U9e/aQm5vL7t27iYuL45tvvql1/vbbb2fjxo34+vqSnZ3Nk08+yapVq0hLS+ORRx5hzJgx/Pzzz+49IBtLiZWIiAjwwQcfkJmZ6Z67VFFRwb59+2jfvj1Tp07F4XDg4+PDzp07G1y3j48PiYmJAKxbt47NmzfTs2dPAE6cOIG/vz9DhgyhsLCQadOmMWjQIPr16+e9mzuDhIQE2rZtC8Dw4cPZtGlTrcRq06ZNjBo1Ch8fHwICArjrrrv44osv+PDDDxk/fjwtWrQAKocoq02aNIkRI0Z4TKpOZ4zBGNOoexgxYgTNmjUjKCiILl26sH379lrny8vLSUpKYteuXRhjOHnyJAC9e/dm/vz5FBUVMXz4cIKCghoVRzUNBYqIiFA57LVq1SocDgcOh4N9+/YRHBxMamoqAQEB5Ofnk5eX53EiOICvr697A2WoTMyq+fn5uedVWZZFUlKSu50dO3Ywd+5cbrjhBvLz84mNjSUtLY2JEyee1320bduWsrIyTp06BUBRUREdOnTwWPb0pKaxSQ5Anz59yMnJqXX/NQUEBFBSUgJASUkJ/v7+jWqvvnuYPXs2cXFxFBQU8O6777rjGj16NJmZmVx77bUMHDiQ9evXNyqOakqsREREgHvuuYclS5ZgWRYAX375JVDZ49GuXTuaNWvGsmXL3ENGrVu35ujRo+7rO3XqhMPhwOVysX//fnJzcz22Ex8fz8qVKzl48CBQ+ZTc3r17OXz4MC6Xi8TERObNm8eWLVvO6z6MMcTFxbmflktPT2fYsGEey2ZlZXHkyBFOnDjB6tWriY6OrnU+JiaGFStW4HQ6OXToEB999BFRUVEkJCTw6quvcvz4cfc9VJswYQIDBw5kxIgR7uSupqFDh5Kenl5vbOfq7bffxuVysXv3bgoLC+nWrVut8+Xl5e7EsuZwbWFhIV26dCE5OZlhw4axdevWRsVRTYmViIgIlT0bJ0+exGaz0aNHD2bPng3AlClTSE9PJywsjO3bt9OyZUsAbDYbPj4+hIWFkZqaSnR0NJ07d6Z79+4kJycTERHhsZ3u3bszb948+vXrh81mIyEhgZKSEoqLi4mNjcVutzN27Fiee67yCcm0tDTS0tIA+Mc//kFgYCCLFi1i3rx5BAYG8sMPPwAwcOBAvvvuO6ByYvmiRYvo2rUrpaWlTJgwAYDMzEzmzJnjjiUqKorExERsNhuJiYl1nqS79957sdlshIWF0bdvX55//nluueUW+vfvz9ChQ4mMjMRut9dZ+uGxxx4jPDyc3/zmN7hcLiZOnOhehmHWrFlkZWURFBREdna2e+J7Xl5erV66mJgY7r//ftatW0dgYCBr164FYM6cOWRmZrrLdezYkaioKAYMGEBaWhp+fn61Ynn88cd54oknCA8Pr5XovfXWW4SEhGC32ykoKODBBx/0+Hk1lKnOzJtSZGSkVd+6FyIiZ1Ketfes569P+MVFikQaatu2bQQHBzd1GCJn5Oln1Biz2bIsj+s5qMdKRERExEuUWImIiIh4iRIrERERES9RYiUiIiLiJUqsRERERLxEiZWIiIiIlyixEhGRq9rixYsJDg5mzJgxDbpuz549vPnmmxcoKs+WL1+OzWYjNDSUPn36kJ+f77Hct99+S69evejatSsPPPCAx9XiT980uqZWrVqdV3yxsbHUt3zSkSNHSEhIICgoiISEBL7//nuP5fr370+bNm3cmyl7Mm7cOPdCqDXl5eWRnJzcsOC9pN69Ao0xtwKvAwGABfzJsqw/GmPmAr8FDlUVfdKyrPerrnkCmAA4gWTLstZegNhFROQKs9Sx1Kv1TbFPqb/NpUvJzs4mMDCwQXVXJ1ajR49u0HVOp9O9vU1Dde7cmQ8//JAbbriB//u//+N3v/sdn3/+eZ1yM2fO5NFHH2XkyJFMnjyZl19+mYcffvi82vS2lJQU4uPjmTVrFikpKaSkpLBgwYI65WbMmMHx48d56aWXGtxGZGRkncVOL5Zz6bE6BUy3LKs7cAfwe2NM96pzqZZl2au+qpOq7sBIoAfQH1hqjDm/nyAREZELaPLkyRQWFjJgwADmz5/PQw89RFRUFOHh4WRkZACVCVRMTAwRERFERETwySefAJUriG/cuBG73U5qamqdHqDBgwezYcMGoLIHaPr06YSFhfHpp5/yxhtvEBUVhd1uZ9KkSTidTpxOJ+PGjSMkJITQ0FBSU1PrxNunTx9uuOEGAO644w6KiorqlLEsi/Xr13PfffcBkJSUxOrVqz3e//79+4mNjSUoKIhnnnnGY10zZsxwx7RixQr3uQULFhAaGkpYWJh79fRqLpeLcePG8fTTT9epMyMjg6SkpHpji4+Pp3Xr1h7P1ZSdnU1kZCS33XYb7733HgAbNmxw93Tl5ubSu3dvwsPD6dOnDzt27ADg66+/dn8GNpuNXbt21dvWuai3x8qyrBKgpOr1UWPMNsDzbo6VhgF/tSzrJ+BbY8w3QBTwqRfiFRER8Zq0tDTWrFlDTk4OixYtom/fvrzyyiuUlZURFRXF3Xffjb+/P1lZWfj5+bFr1y5GjRpFXl4eKSkpLFy40P3HvOY+dKc7duwYvXr14oUXXmDbtm0sWLCAjz/+mObNmzNlyhSWL19Ojx49KC4upqCgAICysrKzxv7yyy8zYMCAOsdLS0tp06YNvr6Vf+IDAwMpLi72WEdubi4FBQW0aNGCnj17MmjQoFo9PX/7299wOBzk5+dz+PBhevbsyZ133onD4SAjI4PPP/+cFi1a1Nor8NSpU4wZM4aQkBCeeuqpOm0eOHCAdu3aAXDLLbdw4MCBs95nffbs2UNubi67d+8mLi6Ob775ptb522+/nY0bN+Lr60t2djZPPvkkq1atIi0tjUceeYQxY8bw888/u/eAbKx6E6uajDGdgHDgcyAamGqMeRDIo7JX63sqk67PalxWhIdEzBjzO+B3ULnPj4iISFP64IMPyMzMdO97V1FRwb59+2jfvj1Tp07F4XDg4+PDzp07G1y3j48PiYmJAKxbt47NmzfTs2dPAE6cOIG/vz9DhgyhsLCQadOmMWjQIPr163fG+nJycnj55ZfZtGnTedzpPyUkJNC2bVsAhg8fzqZNm2olVps2bWLUqFH4+PgQEBDAXXfdxRdffMGHH37I+PHjadGiBQA33nij+5pJkyYxYsQIj0nV6YwxGGMadQ8jRoygWbNmBAUF0aVLF7Zv317rfHl5OUlJSezatQtjDCdPngSgd+/ezJ8/n6KiIoYPH05QUFCj4qh2zpPXjTGtgFXAHyzL+gH4b+CXgJ3KHq0XGtKwZVl/siwr0rKsyJtvvrkhl4qIiHidZVmsWrUKh8OBw+Fg3759BAcHk5qaSkBAAPn5+eTl5XmcCA7g6+uLy+Vyv6+oqHC/9vPzc8+rsiyLpKQkdzs7duxg7ty53HDDDeTn5xMbG0taWlqtDYlr2rp1KxMnTiQjI8OdFNXUtm1bysrK3BsOFxUV0aGD54Gm05OaxiY5UDlcmZOTU+v+awoICKCkpASAkpIS/P39G9Veffcwe/Zs4uLiKCgo4N1333XHNXr0aDIzM7n22msZOHAg69evb1Qc1c4psTLGNKcyqVpuWdbfACzLOmBZltOyLBfwZyqH+wCKgVtrXB5YdUxEROSSdc8997BkyRIsywLgyy+/BCp7PNq1a0ezZs1YtmyZe8iodevWHD161H19p06dcDgcuFwu9u/fT25ursd24uPjWblyJQcPHgQqn5Lbu3cvhw8fxuVykZiYyLx589iyZUuda/ft28fw4cNZtmwZt912m8f6jTHExcW5n5ZLT09n2LBhHstmZWVx5MgRTpw4werVq4mOjq51PiYmhhUrVuB0Ojl06BAfffQRUVFRJCQk8Oqrr3L8+HH3PVSbMGECAwcOZMSIEe7krqahQ4eSnp5eb2zn6u2338blcrF7924KCwvp1q1brfPl5eXuxLLmcG1hYSFdunQhOTmZYcOGsXXr1kbFUa3exMpUpn4vA9ssy1pU43i7GsXuBQqqXmcCI40x1xhjOgNBgOefLhERkUvE7NmzOXnyJDabjR49ejB79mwApkyZQnp6OmFhYWzfvp2WLVsCYLPZ8PHxISwsjNTUVKKjo+ncuTPdu3cnOTmZiIgIj+10796defPm0a9fP2w2GwkJCZSUlFBcXExsbCx2u52xY8fy3HPPAZXzwNLS0gB49tlnKS0tZcqUKdjt9lrDdgMHDuS7774DKieWL1q0iK5du1JaWsqECRMAyMzMZM6cOe5roqKiSExMxGazkZiYWOdJunvvvRebzUZYWBh9+/bl+eef55ZbbqF///4MHTqUyMhI7Ha7e/i02mOPPUZ4eDi/+c1vcLlcTJw40b0Mw6xZs8jKyiIoKIjs7Gz3xPe8vLxavXQxMTHcf//9rFu3jsDAQNaurVxgYM6cOWRmZrrLdezYkaioKAYMGEBaWhp+fn61Ynn88cd54oknCA8Pr5XovfXWW4SEhGC32ykoKODBBx/0+Hk1lKnOzM9YwJhfARuBr4DqPs4ngVFUDgNawB5gUtVEd4wxTwEPUflE4R8sy/q/s7URGRlp1bfuhYjImZRn7T3r+esTfnGRIpGG2rZtG8HBwU0dhsgZefoZNcZstizL43oO5/JU4CbA06Dr+2e5Zj4wv766RURERK4kWnldRERExEuUWImIiIh4iRIrERERES9RYiUiIiLiJUqsRERERLxEiZWIiFzVFi9eTHBwMGPGjGnQdXv27OHNN9+8QFF5lpGRgc1mc69hdaYtbTZv3kxoaChdu3YlOTkZT0srzZ07t876U1B5XyEhIecVX6dOnTh8+PBZy3z77bf06tWLrl278sADD3hcyb60tJS4uDhatWpVa2Pr08XGxuJpuabMzExSUlIafgNe0KC9AkVERC6kQ0v+y6v13TztzH+Uqy1dupTs7GwCAwMbVHd1YjV69OgGXed0Ot3b2zRUfHw8Q4cOxRjD1q1bGTFiRJ298QAefvhh/vznP9OrVy8GDhzImjVrPG7Y3BRmzpzJo48+ysiRI5k8eTIvv/wyDz/8cK0yfn5+/Pu//zsFBQXuTakbYujQoQwdOtRbITeIeqxEROSqNXnyZAoLCxkwYADz58/noYceIioqivDwcDIyMoDKBComJoaIiAgiIiL45JNPgMoVxDdu3Ijdbic1NZXXXnutVu/K4MGD2bBhAwCtWrVi+vTphIWF8emnn/LGG28QFRWF3W5n0qRJOJ1OnE4n48aNIyQkhNDQUFJTU+vE26pVK/deeMeOHfO4t19JSQk//PADd9xxB8YYHnzwQVavXu3x/vPz8+nduzdBQUH8+c9/rnO+oqKC8ePHExoaSnh4ODk5OUBlcvhv//ZvhISEYLPZWLJkSa3rTpw4wYABA+rUaVkW69ev57777gMgKSnJY2wtW7bkV7/6VZ1V1D1ZtmwZdrudkJAQ9zZCNT+Ld999l169ehEeHs7dd9/NgQMHAPjwww+x2+3Y7XbCw8NrbU/UGOqxEhE5R9V/VM4mLi7uIkQi3pKWlsaaNWvIyclh0aJF9O3bl1deeYWysjKioqK4++678ff3JysrCz8/P3bt2sWoUaPIy8sjJSWFhQsX8t577wG196E73bFjx+jVqxcvvPAC27ZtY8GCBXz88cc0b96cKVOmsHz5cnr06EFxcbG7h6asrMxjXe+88w5PPPEEBw8e5H//93/rnC8uLq7V+xYYGEhxsecte7du3cpnn33GsWPHCA8PZ9CgQbXOv/jiixhj+Oqrr9i+fTv9+vVj586dvPrqq+zZsweHw4Gvr2+tvQJ//PFHRo4cyYMPPlhnm5jS0lLatGmDr69vvbGdq+PHj+NwOPjoo4946KGH6vRw/epXv+Kzzz7DGMP//M//8Pzzz/PCCy+wcOFCXnzxRaKjo/nxxx/PKYk7F+qxEhERAT744ANSUlKw2+3ExsZSUVHBvn37OHnyJL/97W8JDQ3l/vvv5+9//3uD6/bx8SExMRGAdevWsXnzZnr27IndbmfdunXuDYELCwuZNm0aa9as4brrrvNY17333sv27dtZvXq1ez/D8zVs2DCuvfZabrrpJuLi4upsHL1p0ybGjh0LwO23384vfvELdu7cSXZ2NpMmTXInSDfeeGOtOsePH++1vffqM2rUKADuvPNOfvjhhzoJaVFREffccw+hoaH8x3/8B19//TUA0dHRPPbYYyxevJiysjL3vTSWEisREREqh6lWrVqFw+HA4XCwb98+goODSU1NJSAggPz8fPLy8jxOtgbw9fXF5XK531dUVLhf+/n5uedVWZZFUlKSu50dO3Ywd+5cbrjhBvLz84mNjSUtLa3WhsSe3HnnnRQWFtaZLN6hQweKiorc74uKiujQoYPHOk4fSvQ0tNhQ0dHRrFmzxuOE+bZt21JWVubeDPlssZ2r+u5h2rRpTJ06la+++oqXXnrJ/bnMmjWL//mf/+HEiRNER0d7nKt2PpRYiYiIAPfccw9LlixxJwRffvklAOXl5bRr145mzZqxbNkynE4nAK1bt641L6dTp044HA5cLhf79++v0/tTLT4+npUrV3Lw4EEAjhw5wt69ezl8+DAul4vExETmzZvHli1b6lz7zTffuOPbsmULP/30E23btq1Vpl27dlx33XV89tlnWJbF66+/zrBhwzzGkpGRQUVFBaWlpWzYsIGePXvWOh8TE8Py5csB2LlzJ/v27aNbt24kJCTw0ksvuROkmkOBzz77LDfccAO///3v67RnjCEuLo6VK1cCkJ6efsbYztWKFSuAyt6166+/nuuvv77W+fLycnfylp6e7j6+e/duQkNDmTlzJj179lRiJSIi4k2zZ8/m5MmT2Gw2evTo4R5mmzJlCunp6YSFhbF9+3ZatmwJgM1mw8fHh7CwMFJTU4mOjqZz5850796d5ORkIiIiPLbTvXt35s2bR79+/bDZbCQkJFBSUkJxcTGxsbHY7XbGjh3Lc889B1TOA0tLSwNg1apVhISEYLfb+f3vf8+KFSvcPTR2u93dxtKlS5k4cSJdu3bll7/8pfuJwJp1Vd9DXFwcd9xxB7Nnz6Z9+/a1Yp0yZQoul4vQ0FAeeOABXnvtNa655homTpxIx44dsdlshIWF1Vl24o9//CMnTpzg8ccfB2DgwIF89913ACxYsIBFixbRtWtXSktLmTBhAlC5RMKcOXPcdXTq1InHHnuM1157jcDAQPcQ7MSJE2stseDn50d4eLj7CcPTzZ07l/vvv59//dd/5aabbnIf/8///E/35PvmzZt77alJ46mr7mKLjIy0PK1DISJyLsqz9p71/PUJv/BKO5q87n3btm0jODi4qcMQOSNPP6PGmM2WZUV6Kq8eKxEREREvUWIlIiIi4iVKrERERES8RImViIiIiJcosRIRERHxEiVWIiIiIl6ixEpERK5qixcvJjg4mDFjxjTouj179tRZv+li+eKLL/D19XUvtHm6zZs3ExoaSteuXUlOTva4CvrcuXNZuHBhneN79uwhJCTkvOLq1KlTnZXgT/ftt9/Sq1cvunbtygMPPOBxJfvS0lLi4uJo1apVrY2tTxcbG4un5ZoyMzNJSUlp+A14gTZhFhGRS0buu4VerS9qSJd6yyxdupTs7OxaGxefi+rEavTo0Q26zul0ure3OR9Op5OZM2fSr1+/M5Z5+OGH+fOf/0yvXr0YOHAga9as8doCmI01c+ZMHn30UUaOHOle1PPhhx+uVcbPz49///d/p6CgoM6myudi6NChDB061FshN4h6rERE5Ko1efJkCgsLGTBgAPPnz+ehhx4iKiqK8PBwMjIygMoEKiYmhoiICCIiIvjkk0+Ayr3mNm7ciN1uJzU1lddee61W78rgwYPZsGEDAK1atWL69OmEhYXx6aef8sYbbxAVFYXdbmfSpEk4nU6cTifjxo0jJCSE0NBQUlNTPca8ZMkSEhMT8ff393i+pKSEH374gTvuuANjDA8++CCrV6/2WDY/P5/evXsTFBTEn//85zrnKyoqGD9+PKGhoYSHh7sXyXU6nfzbv/2be+XyJUuW1LruxIkTDBgwoE6dlmWxfv167rvvPgCSkpI8xtayZUt+9atf4efn5zHumpYtW4bdbickJMS9jVDNz+Ldd9+lV69ehIeHc/fdd3PgwAEAPvzwQ+x2O3a7nfDw8FrbEzWGEisREblqpaWl0b59e3Jycjh27Bh9+/YlNzeXnJwcZsyYwbFjx/D39ycrK4stW7awYsUKkpOTAUhJSSEmJgaHw8Gjjz561naOHTtGr169yM/Pp23btqxYsYKPP/4Yh8OBj48Py5cvx+FwUFxcTEFBAV999RXjx4+vU09xcTHvvPNOnR6e08vU7H0LDAykuLjYY9mtW7eyfv16Pv30U5599ln3tjPVXnzxRYwxfPXVV/zlL38hKSmJiooK/vSnP7Fnzx4cDgdbt26tNYz6448/MmTIEEaNGsVvf/vbWvWVlpbSpk0bfH19643tXB0/fhyHw8HSpUt56KGH6pz/1a9+xWeffcaXX37JyJEjef755wFYuHAhL774Ig6Hg40bN3Lttdc2Ko5qGgoUEeHctquRK9sHH3xAZmame95RRUUF+/bto3379kydOtWdBO3cubPBdfv4+JCYmAjAunXr2Lx5s3vD4xMnTuDv78+QIUMoLCxk2rRpDBo0yONQ3x/+8AcWLFhAs2be6RcZNmwY1157Lddeey1xcXHk5ubW2nNw06ZNTJs2DYDbb7+dX/ziF+zcuZPs7GwmT57sTpBuvPHGWnU+/vjjDZ6zdr5GjRoFwJ133skPP/xAWVlZrfNFRUU88MADlJSU8PPPP9O5c2cAoqOjeeyxxxgzZgzDhw9v8FDwmSixEhERoXKYatWqVXTr1q3W8blz5xIQEEB+fj4ul+uMw1O+vr64XC73+4qKCvdrPz8/97wqy7JISkpyb7JcU35+PmvXriUtLY233nqLV155pdb5vLw8Ro4cCcDhw4d5//338fX15de//rW7TIcOHSgqKnK/LyoqokOHDh5jrt7A+Uzvz0d0dDRr1qxh9OjRdepr27YtZWVlnDp1Cl9f37PGdq7qu4dp06bx2GOPMXToUDZs2MDcuXOByqHcQYMG8f777xMdHc3atWu5/fbbGxULaChQREQEgHvuuYclS5a4n6D78ssvASgvL6ddu3Y0a9aMZcuW4XQ6AWjdunWteTmdOnXC4XDgcrnYv3+/e77P6eLj41m5ciUHDx4E4MiRI+zdu5fDhw/jcrlITExk3rx5bNmypc613377LXv27GHPnj3cd999LF26tFZSBdCuXTuuu+46PvvsMyzL4vXXX2fYsGEeY8nIyKCiooLS0lI2bNjg7kWrFhMTw/LlywHYuXMn+/bto1u3biQkJPDSSy9x6tQp9z1Ue/bZZ7nhhhv4/e9/X6c9YwxxcXHupxnT09PPGNu5WrFiBVDZu3b99ddz/fXX1zpfXl7uTt7S09Pdx3fv3k1oaCgzZ86kZ8+ebN++vVFxVFNiJSJXvJycnHq/RGbPns3Jkyex2Wz06NGD2bNnAzBlyhTS09MJCwtj+/bttGzZEgCbzYaPjw9hYWGkpqYSHR1N586d6d69O8nJyURERHhsp3v37sybN49+/fphs9lISEigpKSE4uJiYmNjsdvtjB071t2jlZaWRlpaWr3x1xzCW7p0KRMnTqRr16788pe/dD8ReHpdNpuNuLg47rjjDmbPnk379u1r1TllyhRcLhehoaE88MADvPbaa1xzzTVMnDiRjh07YrPZCAsLq7PsxB//+EdOnDjB448/DsDAgQPd87cWLFjAokWL6Nq1K6WlpUyYMAGoXCJhzpw57jo6derEY489xmuvvUZgYCB///vfAZg4cWKtJRb8/PwIDw93P2F4urlz53L//ffzr//6r9x0003u4//5n//pnnzfvHlzrz01aTytbXGxRUZGWp7WoRARORflWXvPen6Lr3cf4T+buLi4i9bWlWDbtm0EBwc3dRgiZ+TpZ9QYs9myrEhP5dVjJSIiIuIlSqxEREREvESJlYiIiIiXKLESERER8RIlViIiIiJeosRKRERExEuUWImIyFVt8eLFBAcHN3gLlj179tRZv+lC27BhA9dff7178+Bnn33WY7lvv/2WXr160bVrVx544AF+/vnnOmVO3zS6platWp1XfLGxsdS3fNKRI0dISEggKCiIhIQEvv/+e4/l+vfvT5s2bRg8ePAZ6xo3bpx7sdGa8vLy3Hs6Xmza0kZERC4Zn7y93Kv19bm//mRp6dKlZGdnN3ivuOrEavTo0Q26zul0ure3OR8xMTG89957Zy0zc+ZMHn30UUaOHOleOPNsGzdfTCkpKcTHxzNr1ixSUlJISUlhwYIFdcrNmDGD48eP89JLLzW4jcjISCIjPS4zdcGpx0pERK5akydPprCwkAEDBjB//nweeughoqKiCA8PJyMjA6hMoGJiYoiIiCAiIoJPPvkEqNxrbuPGjdjtdlJTU+v0AA0ePJgNGzYAlT1A06dPJywsjE8//ZQ33niDqKgo7HY7kyZNwul04nQ6GTduHCEhIYSGhpKamnpe92RZFuvXr+e+++4DICkpidWrV3ssu3//fmJjYwkKCuKZZ57xWNeMGTPcMVVvHwOVK6iHhoYSFhbGrFmzal3ncrkYN24cTz/9dJ06MzIySEpKqje2+Ph4WrduXe/9ZmdnExkZyW233eZOODds2ODu6crNzaV3796Eh4fTp08fduzYAcDXX3/t/gxsNhu7du2qt61zoR4rERG5aqWlpbFmzRpycnJYtGgRffv25ZVXXqGsrIyoqCjuvvtu/P39ycrKws/Pj127djFq1Cjy8vJISUlh4cKF7j/mr7322hnbOXbsGL169eKFF15g27ZtLFiwgI8//pjmzZszZcoUli9fTo8ePSguLqagoACAsrIyj3V9+umnhIWF0b59exYuXEiPHj1qnS8tLaVNmzb4+lb+iQ8MDKS4uNhjXbm5uRQUFNCiRQt69uzJoEGDavX0/O1vf8PhcJCfn8/hw4fp2bMnd955Jw6Hg4yMDD7//HNatGhRa6/AU6dOMWbMGEJCQnjqqafqtHngwAHatWsHwC233MKBAwfO+O92Lvbs2UNubi67d+8mLi6Ob775ptb522+/nY0bN+Lr60t2djZPPvkkq1atIi0tjUceeYQxY8bw888/u/eAbCwlViIiIsAHH3xAZmYmCxcuBKCiooJ9+/bRvn17pk6disPhwMfHh507dza4bh8fHxITEwFYt24dmzdvdm94fOLECfz9/RkyZAiFhYVMmzaNQYMG0a9fvzr1REREsHfvXlq1asX777/Pr3/960b1tCQkJNC2bVsAhg8fzqZNm2olVps2bWLUqFH4+PgQEBDAXXfdxRdffMGHH37I+PHjadGiBQA33nij+5pJkyYxYsQIj0nV6YwxGGPOO36AESNG0KxZM4KCgujSpUudzZTLy8tJSkpi165dGGM4efIkAL1792b+/PkUFRUxfPhwgoKCGhVHNQ0FioiIUDnstWrVKhwOBw6Hg3379hEcHExqaioBAQHk5+eTl5fncSI4gK+vLy6Xy/2+oqLC/drPz889r8qyLJKSktzt7Nixg7lz53LDDTeQn59PbGwsaWlpTJw4sU4b1113nXti+cCBAzl58iSHDx+uVaZt27aUlZVx6tQpAIqKiujQoYPHmE9Pahqb5AD06dOHnJycWvdfU0BAACUlJQCUlJTg7+/fqPbqu4fZs2cTFxdHQUEB7777rjuu0aNHk5mZybXXXsvAgQNZv359o+KopsRKREQEuOeee1iyZAmWZQHw5ZdfApU9Hu3ataNZs2YsW7bMPWTUunVrjh496r6+U6dOOBwOXC4X+/fvJzc312M78fHxrFy5koMHDwKVT8nt3buXw4cP43K5SExMZN68eWzZsqXOtf/4xz/c8eXm5uJyudw9TtWMMcTFxbmflktPT2fYsGEeY8nKyuLIkSOcOHGC1atXEx0dXet8TEwMK1aswOl0cujQIT766COioqJISEjg1Vdf5fjx4+57qDZhwgQGDhzIiBEj3MldTUOHDiU9Pb3e2M7V22+/jcvlYvfu3RQWFtKtW7da58vLy92JZc3h2sLCQrp06UJycjLDhg1j69atjYqjmhIrERERKns2Tp48ic1mo0ePHsyePRuAKVOmkJ6eTlhYGNu3b6dly5YA2Gw2fHx8CAsLIzU1lejoaDp37kz37t1JTk4mIiLCYzvdu3dn3rx59OvXD5vNRkJCAiUlJRQXFxMbG4vdbmfs2LE899xzQOU8sLS0NABWrlxJSEgIYWFhJCcn89e//tXdQzNw4EC+++47oHJi+aJFi+jatSulpaVMmDABgMzMTObMmeOOJSoqisTERGw2G4mJiXWepLv33nux2WyEhYXRt29fnn/+eW655Rb69+/P0KFDiYyMxG63u4dPqz322GOEh4fzm9/8BpfLxcSJE93LMMyaNYusrCyCgoLIzs52T3zPy8ur1UsXExPD/fffz7p16wgMDGTt2rUAzJkzh8zMTHe5jh07EhUVxYABA0hLS8PPz69WLI8//jhPPPEE4eHhtRK9t956i5CQEOx2OwUFBTz44IMeP6+GMtWZb1OKjIy06lv3QkTkTMqz9p71/BbfwosUCcTFxV20tq4E27ZtIzg4uKnDEDkjTz+jxpjNlmV5XM9BPVYiIiIiXqLESkRERMRLlFiJiIiIeEm9iZUx5lZjTI4x5u/GmK+NMY9UHb/RGJNljNlV9f2GquPGGLPYGPONMWarMcbz7D0RERGRK8y59FidAqZbltUduAP4vTGmOzALWGdZVhCwruo9wAAgqOrrd8B/ez1qERERkUtQvYmVZVkllmVtqXp9FNgGdACGAelVxdKBX1e9Hga8blX6DGhjjGnn7cBFRERELjUNmmNljOkEhAOfAwGWZZVUnfoHEFD1ugOwv8ZlRVXHRERELjmLFy8mODiYMWPGNOi6PXv28Oabb16gqM5sw4YN2O12evTowV133eWxzLfffkuvXr3o2rUrDzzwgMfV4k/fNLqm6tXdGyo2Npb6lk86cuQICQkJBAUFkZCQwPfff++xXP/+/WnTpo17M2VPxo0b514Itaa8vDySk5MbFryXnPNegcaYVsAq4A+WZf1Qc8l4y7IsY0yDFsQyxvyOyqFCOnbs2JBLRUTkClXfmmQNdX3CL+ots3TpUrKzswkMDGxQ3dWJ1ejRoxt0ndPpdG9v01BlZWVMmTKFNWvW0LFjR/fq7aebOXMmjz76KCNHjmTy5Mm8/PLLPPzww+fVprelpKQQHx/PrFmzSElJISUlhQULFtQpN2PGDI4fP85LL73U4DYiIyPrLHZ6sZxTj5UxpjmVSdVyy7L+VnX4QPUQX9X36k+3GLi1xuWBVcdqsSzrT5ZlRVqWFXnzzTefb/wiIiLnbfLkyRQWFjJgwADmz5/PQw89RFRUFOHh4WRkZACVCVRMTAwRERFERETwySefAJUriG/cuBG73U5qamqdHqDBgwezYcMGoLIHaPr06YSFhfHpp5/yxhtvEBUVhd1uZ9KkSTidTpxOJ+PGjSMkJITQ0FBSU1PrxPvmm28yfPhwd4eEp332LMti/fr13HfffQAkJSWxevVqj/e/f/9+YmNjCQoK4plnnvFY14wZM9wxrVixwn1uwYIFhIaGEhYW5l49vZrL5WLcuHE8/fTTderMyMggKSmp3tji4+Np3bq1x3M1ZWdnExkZyW233cZ7770HVPbqVfd05ebm0rt3b8LDw+nTpw87duwA4Ouvv3Z/BjabrVGbWddUb4+VqeyaehnYZlnWohqnMoEkIKXqe0aN41ONMX8FegHlNYYMRURELhlpaWmsWbOGnJwcFi1aRN++fXnllVcoKysjKiqKu+++G39/f7KysvDz82PXrl2MGjWKvLw8UlJSWLhwofuPec196E537NgxevXqxQsvvMC2bdtYsGABH3/8Mc2bN2fKlCksX76cHj16UFxcTEFBAVDZO3W6nTt3cvLkSWJjYzl69CiPPPJIna1YSktLadOmDb6+lX/iAwMDKS6u078BVCYdBQUFtGjRgp49ezJo0KBaPT1/+9vfcDgc5Ofnc/jwYXr27Mmdd96Jw+EgIyODzz//nBYtWtTaK/DUqVOMGTOGkJAQnnrqqTptHjhwgHbtKqde33LLLRw4cOCM/27nYs+ePeTm5rJ7927i4uL45ptvap2//fbb2bhxI76+vmRnZ/Pkk0+yatUq0tLSeOSRRxgzZgw///yzew/IxjqXocBo4DfAV8YYR9WxJ6lMqN4yxkwA9gIjqs69DwwEvgGOA+O9EqmIiMgF9MEHH5CZmene966iooJ9+/bRvn17pk6disPhwMfHh507dza4bh8fHxITEwFYt24dmzdvpmfPngCcOHECf39/hgwZQmFhIdOmTWPQoEH069evTj2nTp1i8+bNrFu3jhMnTtC7d2/uuOMObrvttvO654SEBPcmzsOHD2fTpk21EqtNmzYxatQofHx8CAgI4K677uKLL77gww8/ZPz48bRo0QKAG2+80X3NpEmTGDFihMek6nTGGGpOLTofI0aMoFmzZgQFBdGlSxe2b99e63x5eTlJSUns2rULYwwnT54EoHfv3syfP5+ioiKGDx9OUFBQo+KoVm9iZVnWJuBMdx3vobwF/L6RcYmIiFxUlmWxatUqunXrVuv43LlzCQgIID8/H5fLVWeT32q+vr64XC73+4qKCvdrPz8/97wqy7JISkpyb7JcU35+PmvXriUtLY233nqLV155pdb5wMBA2rZtS8uWLWnZsiV33nkn+fn5tRKrtm3bUlZWxqlTp/D19aWoqIgOHTw/Q3Z6UtPYJAegT58+5OTkMH36dI//VgEBAZSUlNCuXTtKSko8Dmc2RH33MHv2bOLi4njnnXfYs2cPsbGxAIwePZpevXrxv//7vwwcOJCXXnqJvn37NioW0MrrIiIiANxzzz0sWbKEyv4B+PLLL4HKHo927drRrFkzli1b5h4yat26NUePHnVf36lTJxwOBy6Xi/3795Obm+uxnfj4eFauXOmeeH7kyBH27t3L4cOHcblcJCYmMm/ePLZs2VLn2mHDhrFp0yZOnTrF8ePH+fzzzz1tEExcXJz7abn09HSGDRvmMZasrCyOHDnCiRMnWL16NdHR0bXOx8TEsGLFCpxOJ4cOHeKjjz4iKiqKhIQEXn31VY4fP+6+h2oTJkxg4MCBjBgxglOnTtVpc+jQoaSnp9cb27l6++23cblc7N69m8LCwjqJcXl5uTuxrDlcW1hYSJcuXUhOTmbYsGFs3bq1UXFUU2IlIiJCZc/GyZMnsdls9OjRg9mzZwMwZcoU0tPTCQsLY/v27bRs2RIAm82Gj48PYWFhpKamEh0dTefOnenevTvJyclERHjeeKR79+7MmzePfv36YbPZSEhIoKSkhOLiYmJjY7Hb7YwdO9bdo5WWlkZaWhoAwcHB9O/fH5vNRlRUFBMnTiQkJASAgQMH8t133wGVE8sXLVpE165dKS0tZcKECQBkZmYyZ84cdyxRUVEkJiZis9lITEys8yTdvffei81mIywsjL59+/L8889zyy230L9/f4YOHUpkZCR2u909fFrtscceIzw8nN/85je4XC4mTpzoXoZh1qxZZGVlERQURHZ2tnvie15eHhMnTnTXERMTw/3338+6desIDAxk7dq1AMyZM4fMzEx3uY4dOxIVFcWAAQNIS0ur00v2+OOP88QTTxAeHl4r0XvrrbcICQnBbrdTUFBQZ67a+TLVmXlTioyMtOpb90JE5Ezqe0R/i2/hRYoE4uLiLlpbV4Jt27bV6XERuZR4+hk1xmy2LMvjeg7nvI6ViMiFVlj4x1rvu3R5pIkiERE5PxoKFBEREfESJVYiIiIiXqLESkRERMRLlFiJiIiIeIkmr4tIkzl9srqIyOVOPVYiInJVW7x4McHBwYwZM6ZB1+3Zs4c333zzAkXl2X/8x39gt9ux2+2EhITg4+NTa3HOat9++y29evWia9euPPDAA/z88891ypy+aXRNrVq1Oq/4YmNjqW/5pCNHjpCQkEBQUBAJCQl8//33Hsv179+fNm3auDdT9mTcuHHuhVBrysvLIzk5uWHBe4l6rERE5JKRk5Pj1frOZV2xpUuXkp2dTWBgYIPqrk6sRo8e3aDrnE6ne3ubhpoxYwYzZswA4N133yU1NbXWPn3VZs6cyaOPPsrIkSOZPHkyL7/8Mg8//PB5teltKSkpxMfHM2vWLFJSUkhJSWHBggV1ys2YMYPjx4/z0ksvNbiNyMjIOoudXizqsRIRkavW5MmTKSwsZMCAAcyfP5+HHnqIqKgowsPDycjIACoTqJiYGCIiIoiIiOCTTz4BKlcQ37hxI3a7ndTU1Do9QIMHD2bDhg1AZQ/Q9OnTCQsL49NPP+WNN94gKioKu93OpEmTcDqdOJ1Oxo0bR0hICKGhoaSmpp419r/85S+MGjWqznHLsli/fj333XcfAElJSaxevdpjHfv37yc2NpagoCCeeeYZj3XNmDHDHdOKFSvc5xYsWEBoaChhYWHu1dOruVwuxo0bx9NPP12nzoyMDJKSkuqNLT4+ntatW3s8V1N2djaRkZHcdtttvPfeewBs2LDB3dOVm5tL7969CQ8Pp0+fPuzYsQOAr7/+2v0Z2Gw2du3aVW9b50I9ViIictVKS0tjzZo15OTksGjRIvr27csrr7xCWVkZUVFR3H333fj7+5OVlYWfnx+7du1i1KhR5OXlkZKSwsKFC91/zGvuQ3e6Y8eO0atXL1544QW2bdvGggUL+Pjjj2nevDlTpkxh+fLl9OjRg+LiYgoKCgAoKys7Y33Hjx9nzZo1/Nd//Vedc6WlpbRp0wZf38o/8YGBgRQXF3usJzc3l4KCAlq0aEHPnj0ZNGhQrZ6ev/3tbzgcDvLz8zl8+DA9e/bkzjvvxOFwkJGRweeff06LFi1qDUeeOnWKMWPGEBISwlNPPVWnzQMHDtCuXTsAbrnlFg4cOHDG+zwXe/bsITc3l927dxMXF8c333xT6/ztt9/Oxo0b8fX1JTs7myeffJJVq1aRlpbGI488wpgxY/j555/de0A2lhIrERER4IMPPiAzM9O9711FRQX79u2jffv2TJ06FYfDgY+PDzt37mxw3T4+PiQmJgKwbt06Nm/eTM+ePQE4ceIE/v7+DBkyhMLCQqZNm8agQYPo16/fGet79913iY6O9jgM2BAJCQm0bdsWgOHDh7Np06ZaidWmTZsYNWoUPj4+BAQEcNddd/HFF1/w4YcfMn78eFq0aAFQK45JkyYxYsQIj0nV6YwxGGMadQ8jRoygWbNmBAUF0aVLF7Zv317rfHl5OUlJSezatQtjDCdPngSgd+/ezJ8/n6KiIoYPH05QUFCj4qimoUAREREqh71WrVqFw+HA4XCwb98+goODSU1NJSAggPz8fPLy8jxOBAfw9fXF5XK531dUVLhf+/n5uedVWZZFUlKSu50dO3Ywd+5cbrjhBvLz84mNjSUtLa3WhsSn++tf/+pxGBCgbdu2lJWVuTccLioqokOHDh7Lnp7UNDbJAejTpw85OTm17r+mgIAASkpKACgpKcHf379R7dV3D7NnzyYuLo6CggLeffddd1yjR48mMzOTa6+9loEDB7J+/fpGxVFNiZWIiAhwzz33sGTJEizLAuDLL78EKns82rVrR7NmzVi2bJl7yKh169YcPXrUfX2nTp1wOBy4XC72799Pbm6ux3bi4+NZuXIlBw8eBCqfktu7dy+HDx/G5XKRmJjIvHnz2LJli8fry8vL+fDDDxk2bJjH88YY4uLi3E/Lpaenn7FsVlYWR44c4cSJE6xevZro6Oha52NiYlixYgVOp5NDhw7x0UcfERUVRUJCAq+++irHjx9330O1CRMmMHDgQEaMGOFO7moaOnQo6enp9cZ2rt5++21cLhe7d++msLCQbt261TpfXl7uTixrDtcWFhbSpUsXkpOTGTZsGFu3bm1UHNWUWImIiFDZs3Hy5ElsNhs9evRg9uzZAEyZMoX09HTCwsLYvn07LVu2BMBms+Hj40NYWBipqalER0fTuXNnunfvTnJyMhERER7b6d69O/PmzaNfv37YbDYSEhIoKSmhuLiY2NhY7HY7Y8eO5bnnngMq54GlpaW5r3/nnXfo16+fO45qAwcO5LvvvgMqJ5YvWrSIrl27UlpayoQJEwDIzMxkzpw57muioqJITEzEZrORmJhY50m6e++9F5vNRlhYGH379uX555/nlltuoX///gwdOpTIyEjsdrt7+LTaY489Rnh4OL/5zW9wuVxMnDjRvQzDrFmzyMrKIigoiOzsbPfE97y8vFq9dDExMdx///2sW7eOwMBA1q5dC8CcOXPIzMx0l+vYsSNRUVEMGDCAtLQ0/Pz8asXy+OOP88QTTxAeHl4r0XvrrbcICQnBbrdTUFDAgw8+6PHzaihTnZk3pcjISKu+dS9E5MpT3wKhXbo8ck71lGftPev5Lb6F5xxTY53L4/3yT9u2bSM4OLipwxA5I08/o8aYzZZleVzPQT1WIiIiIl6ipwJF5KLRFjYicqVTj5WIiIiIlyixEhEREfESJVYiIiIiXqLESkRERMRLlFiJiNRj/9+/OucvufwsXryY4OBgxowZ06Dr9uzZw5tvvnmBovKsvLycIUOGEBYWRo8ePXj11Vc9lnvqqae49dZbadWq1RnrOn3T6Jr69OnjlXivRnoqUERELhnefnL0XNZCW7p0KdnZ2QQGBjao7urEavTo0Q26zul0ure3aagXX3yR7t278+6773Lo0CG6devGmDFj+Jd/+Zda5YYMGcLUqVPPe/+7Tz755LyuE/VYiYjIVWzy5MkUFhYyYMAA5s+fz0MPPURUVBTh4eFkZGQAlQlUTEwMERERREREuJOOWbNmsXHjRux2O6mpqXV6gAYPHsyGDRsAaNWqFdOnTycsLIxPP/2UN954g6ioKOx2O5MmTcLpdOJ0Ohk3bhwhISGEhoaSmppaJ15jDEePHsWyLH788UduvPFGfH3r9pHccccdtGvXrt77379/P7GxsQQFBfHMM8+4j1f3dP3444/Ex8cTERFBaGio+9/k2LFjDBo0iLCwMEJCQlixYsU5/otf+ZRYiYjIVSstLY327duTk5PDsWPH6Nu3L7m5ueTk5DBjxgyOHTuGv78/WVlZbNmyhRUrVpCcnAxASkoKMTExOBwOHn300bO2c+zYMXr16kV+fj5t27ZlxYoVfPzxxzgcDnx8fFi+fDkOh4Pi4mIKCgr46quvGD9+fJ16pk6dyrZt22jfvj2hoaH88Y9/pFmz8/9Tnpuby6pVq9i6dStvv/02p++C4ufnxzvvvMOWLVvIyclh+vTpWJbFmjVraN++Pfn5+RQUFNC/f//zjuFKo8RKREQE+OCDD0hJScFutxMbG0tFRQX79u3j5MmT/Pa3vyU0NJT777+fv//97w2u28fHh8TERADWrVvH5s2b6dmzJ3a7nXXr1rk3BC4sLGTatGmsWbOG6667rk49a9euxW6389133+FwOJg6dSo//PDDed9zQkICbdu25dprr2X48OFs2rSp1nnLsnjyySex2WzcfffdFBcXc+DAAUJDQ8nKymLmzJls3LiR66+//rxjuNJojpWIiAiVScSqVavo1q1breNz584lICCA/Px8XC5XnU1+q/n6+uJyudzvKyoq3K/9/Pzc86osyyIpKcm9yXJN+fn5rF27lrS0NN566y1eeeWVWudfffVVZs2ahTGGrl270rlzZ7Zv305UVNR53bMx5qzvly9fzqFDh9i8eTPNmzenU6dOVFRUcNttt7Flyxbef/99nn76aeLj42tt7nw1U4+ViIgIcM8997BkyRIsywLgyy+/BCqfxGvXrh3NmjVj2bJlOJ1OAFq3bs3Ro0fd13fq1AmHw4HL5WL//v3k5uZ6bCc+Pp6VK1dy8OBBAI4cOcLevXs5fPgwLpeLxMRE5s2bx5YtW+pc27FjR9atWwfAgQMH2LFjB126dDnve87KyuLIkSOcOHGC1atXEx0dXet8eXk5/v7+NG/enJycHPburdzw/LvvvqNFixaMHTuWGTNmeIz1aqXESkREBJg9ezYnT57EZrPRo0cPZs+eDcCUKVNIT08nLCyM7du307JlSwBsNhs+Pj6EhYWRmppKdHQ0nTt3pnv37iQnJxMREeGxne7duzNv3jz69euHzWYjISGBkpISiouLiY2NxW63M3bsWHePVlpaGmlpae4YP/nkE0JDQ4mPj2fBggXcdNNNANjtdncbjz/+OIGBgRw/fpzAwEDmzp0LQGZmZq2epaioKBITE7HZbCQmJhIZGVkr1jFjxpCXl0doaCivv/46t99+OwBfffWVe/L9M888w9NPP93If/0rh6nOzJtSZGSkdfqEORG58jT0UfpzeVQeoDxr71nPb/EtbFC7p2vI+lQP/j65UW1dbbZt20ZwcHBThyFyRp5+Ro0xmy3LivRUXj1WIiIiIl6ixEpERETES5RYiYiIiHiJEisREWlSl8JcXxFPzudnU4mViIg0GT8/P0pLS5VcySXHsixKS0vPuG7ZmWiBUBERaTKBgYEUFRVx6NChpg5FpA4/P78Gb86txEpERJpM8+bN6dy5c1OHIeI1GgoUERER8RIlViIiIiJeosRKRERExEuUWImIiIh4iRIrERERES9RYiUiIiLiJUqsRERERLxEiZWIiIiIl9SbWBljXjHGHDTGFNQ4NtcYU2yMcVR9Daxx7gljzDfGmB3GmHsuVOAiIiIil5pz6bF6Dejv4XiqZVn2qq/3AYwx3YGRQI+qa5YaY3y8FayIiIjIpazexMqyrI+AI+dY3zDgr5Zl/WRZ1rfAN0BUI+ITERERuWw0Zq/AqcaYB4E8YLplWd8DHYDPapQpqjpWhzHmd8DvADp27NiIMETkSlVY+Ef36y5dHmnCSEREzs35Tl7/b+CXgB0oAV5oaAWWZf3JsqxIy7Iib7755vMMQ0REROTScV6JlWVZByzLclqW5QL+zD+H+4qBW2sUDaw6JiIiInLFO6/EyhjTrsbbe4HqJwYzgZHGmGuMMZ2BICC3cSGKiIiIXB7qnWNljPkLEAvcZIwpAv4fEGuMsQMWsAeYBGBZ1tfGmLeAvwOngN9bluW8IJGLiIiIXGLqTawsyxrl4fDLZyk/H5jfmKBERERELkdaeV1ERETES5RYiYiIiHiJEisRERERL1FiJSIiIuIlSqxEREREvESJlYiIiIiXKLESERER8ZLGbMIsItLkcnJy+Glf+VnLXNPl+osUjYhc7dRjJSIiIuIlSqxEREREvESJlYiIiIiXaI6ViIgX5eTk1FsmLi7uIkQiIk1BPVYiIiIiXqLESkRERMRLlFiJiIiIeInmWInIJa+isIzy3Xs9nqtvDSsRkYtJPVYiIiIiXqIeKxG5LBSb12q972CNa5I4RETORj1WIiIiIl6ixEpERETES5RYiYiIiHiJEisRERERL1FiJSIiIuIlSqxEREREvESJlYiIiIiXKLESERER8RIlViIiIiJeosRKRERExEuUWImIiIh4iRIrERERES9RYiUiIiLiJUqsRERERLxEiZWIiIiIlyixEhEREfESJVYiIiIiXqLESkRERMRLfJs6ABG5shUW/rGpQxARuWjUYyUiIiLiJUqsRERERLxEiZWIiIiIlyixEhEREfESJVYiIiIiXqLESkRERMRLlFiJiIiIeInWsRKRy1KxeQ2AZm3+AYCr7J4mjEZEpJJ6rERERES8RImViIiIiJfUm1gZY14xxhw0xhTUOHajMSbLGLOr6vsNVceNMWaxMeYbY8xWY0zEhQxeRERE5FJyLj1WrwH9Tzs2C1hnWVYQsK7qPcAAIKjq63fAf3snTBEREZFLX72JlWVZHwFHTjs8DEivep0O/LrG8detSp8BbYwx7bwUq4iIiMgl7XznWAVYllVS9fofQEDV6w7A/hrliqqOiYiIiFzxGj153bIsC7Aaep0x5nfGmDxjTN6hQ4caG4aIiIhIkzvfxOpA9RBf1feDVceLgVtrlAusOlaHZVl/siwr0rKsyJtvvvk8wxARERG5dJxvYpUJJFW9TgIyahx/sOrpwDuA8hpDhiIiIiJXtHpXXjfG/AWIBW4yxhQB/w9IAd4yxkwA9gIjqoq/DwwEvgGOA+MvQMwiIiIil6R6EyvLskad4VS8h7IW8PvGBiUicrna//ev6i3zyeHvAOhz/5gLHY6IXGRaeV1ERETES5RYiYiIiHiJEisRERERL1FiJSIiIuIlSqxEREREvESJlYiIiIiXKLESERER8RIlViIiIiJeosRKRERExEuUWImIiIh4iRIrERERES9RYiUiIiLiJUqsRERERLxEiZWIiIiIlyixEhEREfESJVYiIiIiXqLESkRERMRLlFiJiIiIeIkSKxEREREvUWIlIiIi4iVKrERERES8RImViIiIiJcosRIRERHxEt+mDkBE5EL7qbD8rOev6XL9RYpERK506rESERER8RIlViIiIiJeosRKRERExEuUWImIiIh4iRIrERERES9RYiUiIiLiJUqsRERERLxEiZWIiIiIlyixEhEREfESrbwuIl5VWPjHpg5BRKTJqMdKRERExEuUWImIiIh4iRIrERERES9RYiUiIiLiJUqsRERERLxEiZWIiIiIlyixEhEREfESJVYiIiIiXqIFQkXkinS09MQZz7Vue61X2/I7es1Zz1e0/smr7YnIpUs9ViIiIiJeosRKRERExEuUWImIiIh4iRIrERERES/R5HURuSI0a7O29oHSO5smEBG5qjUqsTLG7AGOAk7glGVZkcaYG4EVQCdgDzDCsqzvGxemiIiIyKXPG0OBcZZl2S3Liqx6PwtYZ1lWELCu6r2IiIjIFe9CzLEaBqRXvU4Hfn0B2hARERG55DR2jpUFfGCMsYCXLMv6ExBgWVZJ1fl/AAGeLjTG/A74HUDHjh0bGYaISKWfjv8IQMWPR85YxnL5ct3NHn81iYg0SmMTq19ZllVsjPEHsowx22uetCzLqkq66qhKwv4EEBkZ6bGMiIiIyOWkUUOBlmUVV30/CLwDRAEHjDHtAKq+H2xskCIiIiKXg/NOrIwxLY0xratfA/2AAiATSKoqlgRkNDZIERERkctBY4YCA4B3jDHV9bxpWdYaY8wXwFvGmAnAXmBE48MUERERufSdd2JlWVYhEObheCkQ35igRERERC5H2tJGRERExEuUWImIiIh4iRIrERERES/RJswiclX64dAB9+uKin1NGImIXEmUWInIVc/v6DVnPV/R+ievtvfNwVIAfsrJaVQ9cXFx3ghHRLxIQ4EiIiIiXqIeKxG56vx0/NQZz13TQr8WReT86TeIiFyyvt3zLQDOcu8OxYmIXCgaChQRERHxEiVWIiIiIl6ixEpERETESzTHSkSkiez/+1fnXPbW7qEXMBIR8RYlViJyRWrd6RP366N7+jRhJCJyNdFQoIiIiIiXKLESERER8RIlViIiIiJeojlWIiKXqZxz2GtQ+wmKXFzqsRIRERHxEiVWIiIiIl6ixEpERETESzTHSkSkAY4ePlHnmHH6AHBNC/1KFbna6beAiDRKYeEfm6ztn0+crHPMedJ1wdt1Hv2x1nuXb/PK4866SZdP69YXPB4RuXRoKFBERETES9RjJSJSD7+j17hfG6ePu4dKROR0SqxERC4D2rBZ5PKgxEpEBHD9/BPgeZ5UrXLOK2cGxVLH0nMuO8U+5QJGInLlUGIlIlc0y+l0J03n4ucrKHESkYtPv0FEREREvEQ9ViLS5CoKyzwed5afe0+TiMilQD1WIiIiIl6iHisRuey4jv5Q+b1qxfOaLJfzYocjIuKmHisRERERL1GPlYhc8n46XnsLGevUKQCc1uXRO1VzgVFPKlprLpnIlUKJlYjIBeQ8erTeMid/OARA8w4dLnQ4InKBKbESEbkMfPfjd+dc9la08rpIU1FiJSJXvOt+mVvr/Q+7o5ookouj5vY3nxw+c0J28h87aR5z28UISeSqocRKROQKU7N36+w9XX78tO1HWgW3uvBBiVwllFiJiFzCjh6u3LvQOtnc43nT+uTFDEdE6qHESkSaxP6/b3W/vuaon8cyP5360ePxS9XZ9hn8Fx/XRYykhhPlZz9f9iN8W1r7WOeYCxePyBVO61iJiIiIeIl6rESkQQoL/+iVeo4cO+F+7XvqZ6/Uebk69vO/VL749tAZy/g6/7nW1amWZ18Xq6F+/Efb2gcqPPQU2r3apMgVS4mViMhFcLZhwnPRtvnNNSr758tm31e+KTtV7PG6Cs1LF7molFiJyCXNebLu3CTLMhc3hgbsP+jTrO7+hZcy14F/3ttPzsrXPx04UKfcCymLafav/b3S5qMJWuJBrlxKrESkXt4a/ruUNCRZutCu972h3jKXUrwicmZKrERELmMuV+X8rJPOumN+zX2891TlTyePY/ZtPWsZv442r7UncrlSYiUiXlNzCYVL2XW/zMVl/XOIsfybnl6r+0ING7os66znb/C9qc4xY67j+mOV8ZT+tLfeNn66/sx/En50HuLHn86+WfT3P2wDIOK6B+ptS+RKpcRKRC5trlN1j50lyXBZFpiLOwdLatvyw4qznl/q+OdTiFPsUy50OCIX1QVLrIwx/YE/Aj7A/1iWlXKh2hIR76pvTtWx3Np777l+agmA7091l03waX2dxzpOX2LB8nAtgOU6e09Nk7NcNDRCi8rEz+X0kDR6kWX9C9XRuWhT61wzys543TXOox6PN/vp2Bmv+eGadtxwoO6DBh2//edEeMtqT6sfm3Pdpgr3sZxVlUlYiP8hbp429Yz1X81Ss3Ze8Da8/kBBznPnd13cE96NowlckAVCjTE+wIvAAKA7MMoY0/1CtCUiIiJyqbhQPVZRwDeWZRUCGGP+CgwD/n6B2hO5Knjr/1zrG6qx+33pft3umpA652/8/nit9y2bVS5YefJU3R6LUyfOsJed563vvKqeaUlu1fOXTIP7nq4sLcsre9BO792qdm35P3u6jp02H6uVzw+13v/o77mnEuCHGj8TP5zYA8Cu0jw2rvr+rPE1ZO5Wky3pcD49NVdAL825+LSwtN4yn51q/O+4pl7O40IlVh2A/TXeFwG9LlBb5+xidKdeDE39QyMXxqEl/+Xx+DsHd7lfOysq/yDt6xxQq8z3AZWdzzX/8ASYN2qVOWCNdb/+1Q8f1jr3zb5f1hqyuTnsn699mtdd0+j0gaLm/AvQGk+/UkyNdah8rH8O9/1QYyVxq9kpfJyekxrrfH5NuROlc3M1JFRtrwk47Ujt96U/1f2cz6Y6Cavm16z2MOGNNVbWP5NrnP98arHFyRICfzjzhtI9D/emm/UhRdf/a2X9uwpqnf+H89ta79/fUAjAnvvP/qenoXO8Pnl7+dkLfLvP/bJPz44NqvuSdb7DelcpY53r/9I1pFJj7gP6W5Y1ser9b4BelmVNrVHmd8Dvqt52A3Z4PRA5FzcBh5s6CKmXPqdLnz6jy4M+p8vDpf45/cKyrJs9nbhQPVbFwK013gdWHXOzLOtPwJ8uUPtyjowxeZZlRTZ1HHJ2+pwuffqMLg/6nC4Pl/PndEEmrwNfAEHGmM7GmH8BRgKZF6gtERERkUvCBemxsizrlDFmKrCWyuUWXrEs6+sL0ZaIiIjIpeKCrWNlWdb7wPsXqn7xGg3HXh70OV369BldHvQ5XR4u28/pgkxeFxEREbkaXag5ViIiIiJXHSVWVyljzP3GmK+NMS5jTORp554wxnxjjNlhjLmnqWKUfzLGzDXGFBtjHFVfA5s6JvknY0z/qv9evjHGzGrqeMQzY8weY8xXVf8N5TV1PFLJGPOKMeagMaagxrEbjTFZxphdVd9vaMoYG0KJ1dWrABgOfFTzYNXWQyOBHkB/YGnVFkXS9FIty7JXfWn+4iVCW3hdduKq/hu6LB/lv0K9RuXfm5pmAessywoC1lW9vywosbpKWZa1zbIsT4uyDgP+alnWT5ZlfQt8Q+UWRSLimXsLL8uyfgaqt/ASkXNgWdZHwJHTDg8D0qtepwO/vpgxNYYSKzmdp+2IOjRRLFLbVGPM1qpu88umW/wqoP9mLh8W8IExZnPV7h9y6QqwLKuk6vU/OH0PpkvYBVtuQZqeMSYbuMXDqacsy8q42PHI2Z3t8wL+G/h3Kv8w/DvwAvDQxYtO5IrwK8uyio0x/kCWMWZ7VW+JXMIsy7KMMZfNEgZKrK5glmXdfR6X1bsdkVwY5/p5GWP+DLx3gcORc6f/Zi4TlmUVV30/aIx5h8phXCVWl6YDxph2lmWVGGPaAQebOqBzpaFAOV0mMNIYc40xpjMQBOQ2cUxXvapfLNXupfLhA7k0aAuvy4AxpqUxpnX1a6Af+u/oUpYJJFW9TgIum1EW9VhdpYwx9wJLgJuB/zXGOCzLuseyrK+NMW8BfwdOAb+3LMvZlLEKAM8bY+xUDgXuASY1aTTipi28LhsBwDvGGKj82/emZVlrmjYkATDG/AWIBW4yxhQB/w9IAd4yxkwA9gIjmi7ChtHK6yIiIiJeoqFAERERES9RYiUiIiLiJUqsRERERLxEiZWIiIiIlyixEhEREfESJVYiIiIiXqLESkRERMRLlFiJiIiIeMn/B59h2FSgvDrgAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 720x432 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Plotting a histogram of the bias values for some layers\n",
    "\n",
    "model = model.to('cpu')\n",
    "num = 0\n",
    "plt.figure(figsize=(10,6))\n",
    "for name, param in model.named_parameters():\n",
    "    if 'bias' in name and f'features.{num}' in name:\n",
    "        plt.hist(param.data.numpy(), bins=20, alpha=0.5, label=name)\n",
    "        num+=1\n",
    "        \n",
    "plt.legend(loc='upper right')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "c6ef137c",
   "metadata": {},
   "source": [
    "# Cross Inductive Bias Distillation\n",
    "\n",
    "Parent model: EfficientNet-B4\n",
    "\n",
    "Child model: EfficientNet-B0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "95ad076c",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<All keys matched successfully>"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "efficientnet = models.efficientnet_b4(pretrained = True)\n",
    "efficientnet.classifier[1] = nn.Linear(in_features = 1792, out_features = 10, bias = True)\n",
    "model = efficientnet\n",
    "criterion = nn.CrossEntropyLoss()\n",
    "model = model.to(config['device'])\n",
    "optimizer = optim.Adam(model.parameters(),lr=config['lr'])\n",
    "\n",
    "model.load_state_dict(torch.load('saved_models/parent_best.pt'))\n",
    "#train_model(model,criterion,optimizer,num_epochs=8)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "d7c178b4",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Test Loss is 0.16448353337105365\n",
      "Test Accuracy is 0.9524\n",
      "\n",
      "AUROC:\n",
      "0.998272772222222\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.98      0.94      0.96      1000\n",
      "           1       0.97      0.98      0.98      1000\n",
      "           2       0.94      0.95      0.95      1000\n",
      "           3       0.90      0.90      0.90      1000\n",
      "           4       0.94      0.96      0.95      1000\n",
      "           5       0.90      0.93      0.92      1000\n",
      "           6       0.97      0.97      0.97      1000\n",
      "           7       1.00      0.94      0.97      1000\n",
      "           8       0.97      0.97      0.97      1000\n",
      "           9       0.96      0.97      0.96      1000\n",
      "\n",
      "    accuracy                           0.95     10000\n",
      "   macro avg       0.95      0.95      0.95     10000\n",
      "weighted avg       0.95      0.95      0.95     10000\n",
      "\n"
     ]
    }
   ],
   "source": [
    "labels, pred_labels,preds = evaluation(model, test_dl)\n",
    "#print(metrics.precision_recall_fscore_support(np.array(labels), np.array(pred_labels)))\n",
    "print('\\nAUROC:')\n",
    "print(metrics.roc_auc_score(np.array(labels), np.array(preds), multi_class='ovr'))\n",
    "print(metrics.classification_report(labels,pred_labels))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a83e4e02",
   "metadata": {},
   "source": [
    "# Knowledge Distillation: Co-advice"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "1115a054",
   "metadata": {},
   "outputs": [],
   "source": [
    "teacher_model = model\n",
    "\n",
    "\n",
    "efficientnet = models.efficientnet_b0(pretrained = True)\n",
    "efficientnet.classifier[1] = nn.Linear(in_features = 1280, out_features = 10, bias = True)\n",
    "model1 = efficientnet\n",
    "model1 = model.to(config['device'])\n",
    "student_model = model1\n",
    "\n",
    "# Define the criterion (loss function) for knowledge distillation\n",
    "class DistillationLoss(nn.Module):\n",
    "    def __init__(self, T):\n",
    "        super(DistillationLoss, self).__init__()\n",
    "        self.T = T\n",
    "        self.kl_div = nn.KLDivLoss(reduction='batchmean')\n",
    "        \n",
    "    def forward(self, outputs, labels, teacher_outputs):\n",
    "        soft_labels = nn.functional.softmax(teacher_outputs/self.T, dim=1)\n",
    "        log_soft_labels = nn.functional.log_softmax(outputs/self.T, dim=1)\n",
    "        loss = self.kl_div(log_soft_labels, soft_labels) * (self.T * self.T)\n",
    "        hard_loss = nn.functional.cross_entropy(outputs, labels)\n",
    "        return loss + hard_loss\n",
    "    \n",
    "# Define the optimizer\n",
    "optimizer = optim.Adam(student_model.parameters(), lr=0.001)\n",
    "\n",
    "# Train the student model with knowledge distillation\n",
    "def train_student_model(train_loader):\n",
    "    distillation_loss = DistillationLoss(T=5) # T is the temperature parameter\n",
    "    student_model.train()\n",
    "    running_loss = 0.0\n",
    "    for i, (inputs, labels) in enumerate(train_loader):\n",
    "        inputs = inputs.cuda()\n",
    "        labels = labels.cuda()\n",
    "        optimizer.zero_grad()\n",
    "        outputs = student_model(inputs)\n",
    "        teacher_outputs = teacher_model(inputs).detach()\n",
    "        loss = distillation_loss(outputs, labels, teacher_outputs)\n",
    "        loss.backward()\n",
    "        optimizer.step()\n",
    "        running_loss += loss.item()\n",
    "    train_loss = running_loss / len(train_loader)\n",
    "    print('Train Loss: {:.4f}'.format(train_loss))\n",
    "\n",
    "# Evaluate the student model on the test set\n",
    "def test_student_model(test_loader):\n",
    "    student_model.eval()\n",
    "    correct = 0\n",
    "    total = 0\n",
    "    with torch.no_grad():\n",
    "        for inputs, labels in test_loader:\n",
    "            inputs = inputs.cuda()\n",
    "            labels = labels.cuda()\n",
    "            outputs = student_model(inputs)\n",
    "            _, predicted = torch.max(outputs.data, 1)\n",
    "            total += labels.size(0)\n",
    "            correct += (predicted == labels).sum().item()\n",
    "    test_acc = 100 * correct / total\n",
    "    print('Test Accuracy: {:.2f}%'.format(test_acc))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "a1934948",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 1\n",
      "Train Loss: 0.1217\n",
      "Test Accuracy: 94.69%\n",
      "Epoch: 2\n",
      "Train Loss: 0.1186\n",
      "Test Accuracy: 94.64%\n",
      "Epoch: 3\n",
      "Train Loss: 0.1063\n",
      "Test Accuracy: 94.19%\n",
      "Epoch: 4\n",
      "Train Loss: 0.0997\n",
      "Test Accuracy: 95.25%\n"
     ]
    }
   ],
   "source": [
    "# Train the student model for 10 epochs using knowledge distillation\n",
    "for epoch in range(3):\n",
    "    print('Epoch: {}'.format(epoch+1))\n",
    "    train_student_model(train_dl)\n",
    "    test_student_model(test_dl)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "dec42977",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Test Loss is 0.17310410901606083\n",
      "Test Accuracy is 0.9525\n",
      "\n",
      "AUROC:\n",
      "0.9967054833333332\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.97      0.96      0.96      1000\n",
      "           1       0.95      0.99      0.97      1000\n",
      "           2       0.91      0.97      0.94      1000\n",
      "           3       0.93      0.87      0.90      1000\n",
      "           4       0.96      0.94      0.95      1000\n",
      "           5       0.90      0.93      0.91      1000\n",
      "           6       0.97      0.98      0.98      1000\n",
      "           7       0.98      0.96      0.97      1000\n",
      "           8       0.97      0.97      0.97      1000\n",
      "           9       0.98      0.95      0.97      1000\n",
      "\n",
      "    accuracy                           0.95     10000\n",
      "   macro avg       0.95      0.95      0.95     10000\n",
      "weighted avg       0.95      0.95      0.95     10000\n",
      "\n"
     ]
    }
   ],
   "source": [
    "labels, pred_labels,preds = evaluation(student_model, test_dl)\n",
    "#print(metrics.precision_recall_fscore_support(np.array(labels), np.array(pred_labels)))\n",
    "print('\\nAUROC:')\n",
    "print(metrics.roc_auc_score(np.array(labels), np.array(preds), multi_class='ovr'))\n",
    "print(metrics.classification_report(labels,pred_labels))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "id": "b55e44bb",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Disparate Impact: 0.9669\n"
     ]
    }
   ],
   "source": [
    "device = 'cpu'\n",
    "model = model.to('cpu')\n",
    "disparate_impact = calculate_disparate_impact(model, test_dl)\n",
    "print(f\"Disparate Impact: {disparate_impact:.4f}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d050053f",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}