The Sample Program - Using Meta Pattern Concern Score to Customizably Evaluate a CNN Classifier Built for MNIST Dataset.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "63b1b25f",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\thinkpad\\AppData\\Roaming\\Python\\Python38\\site-packages\\torchvision\\datasets\\mnist.py:62: UserWarning: train_data has been renamed data\n",
      "  warnings.warn(\"train_data has been renamed data\")\n",
      "C:\\Users\\thinkpad\\AppData\\Roaming\\Python\\Python38\\site-packages\\torchvision\\datasets\\mnist.py:52: UserWarning: train_labels has been renamed targets\n",
      "  warnings.warn(\"train_labels has been renamed targets\")\n",
      "C:\\Users\\thinkpad\\AppData\\Roaming\\Python\\Python38\\site-packages\\torchvision\\datasets\\mnist.py:67: UserWarning: test_data has been renamed data\n",
      "  warnings.warn(\"test_data has been renamed data\")\n",
      "C:\\Users\\thinkpad\\AppData\\Roaming\\Python\\Python38\\site-packages\\torchvision\\datasets\\mnist.py:57: UserWarning: test_labels has been renamed targets\n",
      "  warnings.warn(\"test_labels has been renamed targets\")\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CNN(\n",
      "  (conv1): Conv2d(1, 16, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
      "  (pool): MaxPool2d(kernel_size=2, stride=2, padding=0, dilation=1, ceil_mode=False)\n",
      "  (conv2): Conv2d(16, 32, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
      "  (fc1): Linear(in_features=1568, out_features=16, bias=True)\n",
      "  (fc2): Linear(in_features=16, out_features=10, bias=True)\n",
      ")\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "D:\\Download\\Anaconda3\\lib\\site-packages\\sklearn\\metrics\\_classification.py:870: RuntimeWarning: invalid value encountered in double_scalars\n",
      "  mcc = cov_ytyp / np.sqrt(cov_ytyt * cov_ypyp)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "TURNS ID:  1 | Acc: 0.0992 | F1: 0.0180 | MCC: 0.0000 | MS Loss: 0.0904 | CE Loss: 2.3298 | Score: 0.4401\n",
      "TURNS ID:  2 | Acc: 0.3359 | F1: 0.2610 | MCC: 0.2876 | MS Loss: 0.0860 | CE Loss: 2.1336 | Score: 0.7972\n",
      "TURNS ID:  3 | Acc: 0.4759 | F1: 0.4280 | MCC: 0.4353 | MS Loss: 0.0725 | CE Loss: 1.6409 | Score: 0.7453\n",
      "TURNS ID:  4 | Acc: 0.6176 | F1: 0.5946 | MCC: 0.5847 | MS Loss: 0.0547 | CE Loss: 1.2171 | Score: 0.5720\n",
      "TURNS ID:  5 | Acc: 0.6830 | F1: 0.6688 | MCC: 0.6523 | MS Loss: 0.0450 | CE Loss: 0.9825 | Score: 0.5154\n",
      "TURNS ID:  6 | Acc: 0.7367 | F1: 0.7255 | MCC: 0.7099 | MS Loss: 0.0377 | CE Loss: 0.8128 | Score: 0.4397\n",
      "TURNS ID:  7 | Acc: 0.7782 | F1: 0.7748 | MCC: 0.7551 | MS Loss: 0.0325 | CE Loss: 0.6992 | Score: 0.4400\n",
      "TURNS ID:  8 | Acc: 0.8244 | F1: 0.8218 | MCC: 0.8062 | MS Loss: 0.0257 | CE Loss: 0.5671 | Score: 0.3543\n",
      "TURNS ID:  9 | Acc: 0.8281 | F1: 0.8256 | MCC: 0.8103 | MS Loss: 0.0256 | CE Loss: 0.5589 | Score: 0.4085\n",
      "TURNS ID:  10 | Acc: 0.8526 | F1: 0.8517 | MCC: 0.8371 | MS Loss: 0.0222 | CE Loss: 0.5031 | Score: 0.2964\n",
      "TURNS ID:  11 | Acc: 0.8544 | F1: 0.8544 | MCC: 0.8398 | MS Loss: 0.0212 | CE Loss: 0.4579 | Score: 0.3324\n",
      "TURNS ID:  12 | Acc: 0.8792 | F1: 0.8799 | MCC: 0.8663 | MS Loss: 0.0186 | CE Loss: 0.4218 | Score: 0.2632\n",
      "TURNS ID:  13 | Acc: 0.8724 | F1: 0.8694 | MCC: 0.8590 | MS Loss: 0.0194 | CE Loss: 0.4264 | Score: 0.2960\n",
      "TURNS ID:  14 | Acc: 0.8929 | F1: 0.8926 | MCC: 0.8813 | MS Loss: 0.0162 | CE Loss: 0.3593 | Score: 0.2136\n",
      "TURNS ID:  15 | Acc: 0.8899 | F1: 0.8902 | MCC: 0.8779 | MS Loss: 0.0163 | CE Loss: 0.3558 | Score: 0.2711\n",
      "TURNS ID:  16 | Acc: 0.9097 | F1: 0.9094 | MCC: 0.8999 | MS Loss: 0.0135 | CE Loss: 0.3217 | Score: 0.2128\n",
      "TURNS ID:  17 | Acc: 0.9050 | F1: 0.9054 | MCC: 0.8949 | MS Loss: 0.0149 | CE Loss: 0.3452 | Score: 0.3178\n",
      "TURNS ID:  18 | Acc: 0.9002 | F1: 0.9010 | MCC: 0.8901 | MS Loss: 0.0154 | CE Loss: 0.3569 | Score: 0.2713\n",
      "TURNS ID:  19 | Acc: 0.8841 | F1: 0.8832 | MCC: 0.8718 | MS Loss: 0.0169 | CE Loss: 0.3625 | Score: 0.2958\n",
      "TURNS ID:  20 | Acc: 0.9205 | F1: 0.9208 | MCC: 0.9119 | MS Loss: 0.0120 | CE Loss: 0.2744 | Score: 0.1852\n",
      "TURNS ID:  21 | Acc: 0.9216 | F1: 0.9223 | MCC: 0.9131 | MS Loss: 0.0120 | CE Loss: 0.2820 | Score: 0.1794\n",
      "TURNS ID:  22 | Acc: 0.9162 | F1: 0.9163 | MCC: 0.9070 | MS Loss: 0.0124 | CE Loss: 0.2749 | Score: 0.1867\n",
      "TURNS ID:  23 | Acc: 0.9259 | F1: 0.9260 | MCC: 0.9178 | MS Loss: 0.0112 | CE Loss: 0.2571 | Score: 0.1845\n",
      "TURNS ID:  24 | Acc: 0.9290 | F1: 0.9291 | MCC: 0.9213 | MS Loss: 0.0108 | CE Loss: 0.2572 | Score: 0.2001\n",
      "TURNS ID:  25 | Acc: 0.9146 | F1: 0.9151 | MCC: 0.9058 | MS Loss: 0.0130 | CE Loss: 0.2988 | Score: 0.2870\n",
      "TURNS ID:  26 | Acc: 0.9098 | F1: 0.9100 | MCC: 0.9002 | MS Loss: 0.0136 | CE Loss: 0.3022 | Score: 0.2535\n",
      "TURNS ID:  27 | Acc: 0.9329 | F1: 0.9326 | MCC: 0.9255 | MS Loss: 0.0103 | CE Loss: 0.2392 | Score: 0.1730\n",
      "TURNS ID:  28 | Acc: 0.9107 | F1: 0.9119 | MCC: 0.9019 | MS Loss: 0.0130 | CE Loss: 0.2862 | Score: 0.2276\n",
      "TURNS ID:  29 | Acc: 0.9240 | F1: 0.9242 | MCC: 0.9161 | MS Loss: 0.0113 | CE Loss: 0.2628 | Score: 0.2274\n",
      "TURNS ID:  30 | Acc: 0.9219 | F1: 0.9209 | MCC: 0.9136 | MS Loss: 0.0116 | CE Loss: 0.2560 | Score: 0.2089\n",
      "TURNS ID:  31 | Acc: 0.9302 | F1: 0.9305 | MCC: 0.9226 | MS Loss: 0.0104 | CE Loss: 0.2336 | Score: 0.1804\n",
      "TURNS ID:  32 | Acc: 0.9346 | F1: 0.9348 | MCC: 0.9275 | MS Loss: 0.0097 | CE Loss: 0.2191 | Score: 0.1691\n",
      "TURNS ID:  33 | Acc: 0.9363 | F1: 0.9362 | MCC: 0.9293 | MS Loss: 0.0096 | CE Loss: 0.2176 | Score: 0.1692\n",
      "TURNS ID:  34 | Acc: 0.9373 | F1: 0.9372 | MCC: 0.9305 | MS Loss: 0.0095 | CE Loss: 0.2213 | Score: 0.1781\n",
      "TURNS ID:  35 | Acc: 0.9367 | F1: 0.9360 | MCC: 0.9299 | MS Loss: 0.0095 | CE Loss: 0.2222 | Score: 0.1941\n",
      "TURNS ID:  36 | Acc: 0.9319 | F1: 0.9318 | MCC: 0.9248 | MS Loss: 0.0100 | CE Loss: 0.2280 | Score: 0.1912\n",
      "TURNS ID:  37 | Acc: 0.9325 | F1: 0.9333 | MCC: 0.9254 | MS Loss: 0.0104 | CE Loss: 0.2342 | Score: 0.1945\n",
      "TURNS ID:  38 | Acc: 0.9230 | F1: 0.9219 | MCC: 0.9150 | MS Loss: 0.0117 | CE Loss: 0.2652 | Score: 0.2569\n",
      "TURNS ID:  39 | Acc: 0.9432 | F1: 0.9430 | MCC: 0.9370 | MS Loss: 0.0084 | CE Loss: 0.1959 | Score: 0.1539\n",
      "TURNS ID:  40 | Acc: 0.9418 | F1: 0.9424 | MCC: 0.9356 | MS Loss: 0.0090 | CE Loss: 0.2042 | Score: 0.1716\n",
      "TURNS ID:  41 | Acc: 0.9457 | F1: 0.9458 | MCC: 0.9398 | MS Loss: 0.0083 | CE Loss: 0.1920 | Score: 0.1557\n",
      "TURNS ID:  42 | Acc: 0.9452 | F1: 0.9455 | MCC: 0.9393 | MS Loss: 0.0083 | CE Loss: 0.1989 | Score: 0.1721\n",
      "TURNS ID:  43 | Acc: 0.9398 | F1: 0.9394 | MCC: 0.9334 | MS Loss: 0.0091 | CE Loss: 0.2047 | Score: 0.1743\n",
      "TURNS ID:  44 | Acc: 0.9318 | F1: 0.9316 | MCC: 0.9246 | MS Loss: 0.0099 | CE Loss: 0.2216 | Score: 0.2035\n",
      "TURNS ID:  45 | Acc: 0.9494 | F1: 0.9494 | MCC: 0.9439 | MS Loss: 0.0078 | CE Loss: 0.1866 | Score: 0.1491\n",
      "TURNS ID:  46 | Acc: 0.9412 | F1: 0.9413 | MCC: 0.9349 | MS Loss: 0.0088 | CE Loss: 0.2096 | Score: 0.1962\n",
      "TURNS ID:  47 | Acc: 0.9343 | F1: 0.9335 | MCC: 0.9273 | MS Loss: 0.0099 | CE Loss: 0.2239 | Score: 0.1678\n",
      "TURNS ID:  48 | Acc: 0.9372 | F1: 0.9372 | MCC: 0.9304 | MS Loss: 0.0095 | CE Loss: 0.2124 | Score: 0.1482\n",
      "TURNS ID:  49 | Acc: 0.9407 | F1: 0.9408 | MCC: 0.9343 | MS Loss: 0.0089 | CE Loss: 0.2004 | Score: 0.1552\n",
      "TURNS ID:  50 | Acc: 0.9451 | F1: 0.9447 | MCC: 0.9391 | MS Loss: 0.0081 | CE Loss: 0.1849 | Score: 0.1533\n",
      "TURNS ID:  51 | Acc: 0.9378 | F1: 0.9374 | MCC: 0.9312 | MS Loss: 0.0095 | CE Loss: 0.2149 | Score: 0.1902\n",
      "TURNS ID:  52 | Acc: 0.9428 | F1: 0.9426 | MCC: 0.9365 | MS Loss: 0.0081 | CE Loss: 0.1838 | Score: 0.1525\n",
      "TURNS ID:  53 | Acc: 0.9483 | F1: 0.9481 | MCC: 0.9426 | MS Loss: 0.0077 | CE Loss: 0.1704 | Score: 0.1379\n",
      "TURNS ID:  54 | Acc: 0.9497 | F1: 0.9498 | MCC: 0.9442 | MS Loss: 0.0078 | CE Loss: 0.1696 | Score: 0.1443\n",
      "TURNS ID:  55 | Acc: 0.9510 | F1: 0.9510 | MCC: 0.9456 | MS Loss: 0.0075 | CE Loss: 0.1652 | Score: 0.1495\n",
      "TURNS ID:  56 | Acc: 0.9488 | F1: 0.9484 | MCC: 0.9432 | MS Loss: 0.0077 | CE Loss: 0.1735 | Score: 0.1483\n",
      "TURNS ID:  57 | Acc: 0.9537 | F1: 0.9536 | MCC: 0.9487 | MS Loss: 0.0067 | CE Loss: 0.1500 | Score: 0.1198\n",
      "TURNS ID:  58 | Acc: 0.9451 | F1: 0.9453 | MCC: 0.9392 | MS Loss: 0.0081 | CE Loss: 0.1748 | Score: 0.1670\n",
      "TURNS ID:  59 | Acc: 0.9460 | F1: 0.9463 | MCC: 0.9402 | MS Loss: 0.0079 | CE Loss: 0.1701 | Score: 0.1575\n",
      "TURNS ID:  60 | Acc: 0.9595 | F1: 0.9594 | MCC: 0.9550 | MS Loss: 0.0062 | CE Loss: 0.1401 | Score: 0.1085\n",
      "TURNS ID:  61 | Acc: 0.9579 | F1: 0.9576 | MCC: 0.9533 | MS Loss: 0.0063 | CE Loss: 0.1439 | Score: 0.1113\n",
      "TURNS ID:  62 | Acc: 0.9619 | F1: 0.9619 | MCC: 0.9577 | MS Loss: 0.0060 | CE Loss: 0.1379 | Score: 0.1271\n",
      "TURNS ID:  63 | Acc: 0.9528 | F1: 0.9530 | MCC: 0.9477 | MS Loss: 0.0071 | CE Loss: 0.1559 | Score: 0.1628\n",
      "TURNS ID:  64 | Acc: 0.9602 | F1: 0.9601 | MCC: 0.9558 | MS Loss: 0.0062 | CE Loss: 0.1415 | Score: 0.1269\n",
      "TURNS ID:  65 | Acc: 0.9496 | F1: 0.9497 | MCC: 0.9441 | MS Loss: 0.0072 | CE Loss: 0.1583 | Score: 0.1392\n",
      "TURNS ID:  66 | Acc: 0.9497 | F1: 0.9497 | MCC: 0.9442 | MS Loss: 0.0073 | CE Loss: 0.1595 | Score: 0.1370\n",
      "TURNS ID:  67 | Acc: 0.9584 | F1: 0.9586 | MCC: 0.9538 | MS Loss: 0.0061 | CE Loss: 0.1342 | Score: 0.1138\n",
      "TURNS ID:  68 | Acc: 0.9575 | F1: 0.9573 | MCC: 0.9528 | MS Loss: 0.0064 | CE Loss: 0.1423 | Score: 0.1201\n",
      "TURNS ID:  69 | Acc: 0.9479 | F1: 0.9478 | MCC: 0.9422 | MS Loss: 0.0078 | CE Loss: 0.1643 | Score: 0.1549\n",
      "TURNS ID:  70 | Acc: 0.9575 | F1: 0.9574 | MCC: 0.9527 | MS Loss: 0.0064 | CE Loss: 0.1359 | Score: 0.1363\n",
      "TURNS ID:  71 | Acc: 0.9429 | F1: 0.9425 | MCC: 0.9370 | MS Loss: 0.0089 | CE Loss: 0.1966 | Score: 0.2375\n",
      "TURNS ID:  72 | Acc: 0.9533 | F1: 0.9530 | MCC: 0.9482 | MS Loss: 0.0075 | CE Loss: 0.1650 | Score: 0.2056\n",
      "TURNS ID:  73 | Acc: 0.9525 | F1: 0.9525 | MCC: 0.9474 | MS Loss: 0.0075 | CE Loss: 0.1685 | Score: 0.1714\n",
      "TURNS ID:  74 | Acc: 0.9601 | F1: 0.9599 | MCC: 0.9557 | MS Loss: 0.0063 | CE Loss: 0.1464 | Score: 0.1110\n",
      "TURNS ID:  75 | Acc: 0.9513 | F1: 0.9511 | MCC: 0.9459 | MS Loss: 0.0074 | CE Loss: 0.1640 | Score: 0.1292\n",
      "TURNS ID:  76 | Acc: 0.9636 | F1: 0.9635 | MCC: 0.9596 | MS Loss: 0.0057 | CE Loss: 0.1283 | Score: 0.1136\n",
      "TURNS ID:  77 | Acc: 0.9615 | F1: 0.9615 | MCC: 0.9573 | MS Loss: 0.0062 | CE Loss: 0.1399 | Score: 0.1293\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "TURNS ID:  78 | Acc: 0.9615 | F1: 0.9615 | MCC: 0.9572 | MS Loss: 0.0055 | CE Loss: 0.1196 | Score: 0.1018\n",
      "TURNS ID:  79 | Acc: 0.9635 | F1: 0.9634 | MCC: 0.9595 | MS Loss: 0.0057 | CE Loss: 0.1270 | Score: 0.1068\n",
      "TURNS ID:  80 | Acc: 0.9616 | F1: 0.9617 | MCC: 0.9575 | MS Loss: 0.0060 | CE Loss: 0.1357 | Score: 0.1136\n",
      "TURNS ID:  81 | Acc: 0.9669 | F1: 0.9668 | MCC: 0.9632 | MS Loss: 0.0050 | CE Loss: 0.1161 | Score: 0.0959\n",
      "TURNS ID:  82 | Acc: 0.9606 | F1: 0.9602 | MCC: 0.9562 | MS Loss: 0.0061 | CE Loss: 0.1324 | Score: 0.1243\n",
      "TURNS ID:  83 | Acc: 0.9688 | F1: 0.9685 | MCC: 0.9653 | MS Loss: 0.0048 | CE Loss: 0.1076 | Score: 0.0943\n",
      "TURNS ID:  84 | Acc: 0.9674 | F1: 0.9673 | MCC: 0.9638 | MS Loss: 0.0050 | CE Loss: 0.1131 | Score: 0.0906\n",
      "TURNS ID:  85 | Acc: 0.9570 | F1: 0.9571 | MCC: 0.9524 | MS Loss: 0.0066 | CE Loss: 0.1483 | Score: 0.1544\n",
      "TURNS ID:  86 | Acc: 0.9624 | F1: 0.9623 | MCC: 0.9584 | MS Loss: 0.0058 | CE Loss: 0.1278 | Score: 0.1347\n",
      "TURNS ID:  87 | Acc: 0.9610 | F1: 0.9606 | MCC: 0.9568 | MS Loss: 0.0059 | CE Loss: 0.1276 | Score: 0.1074\n",
      "TURNS ID:  88 | Acc: 0.9663 | F1: 0.9661 | MCC: 0.9626 | MS Loss: 0.0050 | CE Loss: 0.1127 | Score: 0.0936\n",
      "TURNS ID:  89 | Acc: 0.9548 | F1: 0.9546 | MCC: 0.9500 | MS Loss: 0.0067 | CE Loss: 0.1400 | Score: 0.1466\n",
      "TURNS ID:  90 | Acc: 0.9650 | F1: 0.9648 | MCC: 0.9612 | MS Loss: 0.0051 | CE Loss: 0.1087 | Score: 0.0938\n",
      "TURNS ID:  91 | Acc: 0.9649 | F1: 0.9648 | MCC: 0.9610 | MS Loss: 0.0053 | CE Loss: 0.1133 | Score: 0.0982\n",
      "TURNS ID:  92 | Acc: 0.9640 | F1: 0.9638 | MCC: 0.9600 | MS Loss: 0.0053 | CE Loss: 0.1153 | Score: 0.1019\n",
      "TURNS ID:  93 | Acc: 0.9703 | F1: 0.9703 | MCC: 0.9670 | MS Loss: 0.0046 | CE Loss: 0.1014 | Score: 0.0868\n",
      "TURNS ID:  94 | Acc: 0.9638 | F1: 0.9637 | MCC: 0.9598 | MS Loss: 0.0052 | CE Loss: 0.1125 | Score: 0.0986\n",
      "TURNS ID:  95 | Acc: 0.9672 | F1: 0.9672 | MCC: 0.9636 | MS Loss: 0.0050 | CE Loss: 0.1112 | Score: 0.1053\n",
      "TURNS ID:  96 | Acc: 0.9678 | F1: 0.9678 | MCC: 0.9643 | MS Loss: 0.0049 | CE Loss: 0.1105 | Score: 0.1008\n",
      "TURNS ID:  97 | Acc: 0.9688 | F1: 0.9687 | MCC: 0.9653 | MS Loss: 0.0045 | CE Loss: 0.0970 | Score: 0.0968\n",
      "TURNS ID:  98 | Acc: 0.9643 | F1: 0.9641 | MCC: 0.9603 | MS Loss: 0.0052 | CE Loss: 0.1096 | Score: 0.1031\n",
      "TURNS ID:  99 | Acc: 0.9658 | F1: 0.9657 | MCC: 0.9620 | MS Loss: 0.0051 | CE Loss: 0.1082 | Score: 0.1049\n",
      "TURNS ID:  100 | Acc: 0.9618 | F1: 0.9622 | MCC: 0.9577 | MS Loss: 0.0058 | CE Loss: 0.1259 | Score: 0.1317\n",
      "TURNS ID:  101 | Acc: 0.9567 | F1: 0.9566 | MCC: 0.9520 | MS Loss: 0.0064 | CE Loss: 0.1358 | Score: 0.1361\n",
      "TURNS ID:  102 | Acc: 0.9649 | F1: 0.9648 | MCC: 0.9610 | MS Loss: 0.0053 | CE Loss: 0.1140 | Score: 0.0936\n",
      "TURNS ID:  103 | Acc: 0.9686 | F1: 0.9686 | MCC: 0.9651 | MS Loss: 0.0047 | CE Loss: 0.1017 | Score: 0.0869\n",
      "TURNS ID:  104 | Acc: 0.9609 | F1: 0.9609 | MCC: 0.9566 | MS Loss: 0.0058 | CE Loss: 0.1241 | Score: 0.1167\n",
      "TURNS ID:  105 | Acc: 0.9686 | F1: 0.9686 | MCC: 0.9651 | MS Loss: 0.0048 | CE Loss: 0.0996 | Score: 0.1067\n",
      "TURNS ID:  106 | Acc: 0.9571 | F1: 0.9576 | MCC: 0.9526 | MS Loss: 0.0064 | CE Loss: 0.1324 | Score: 0.1499\n",
      "TURNS ID:  107 | Acc: 0.9653 | F1: 0.9651 | MCC: 0.9615 | MS Loss: 0.0052 | CE Loss: 0.1124 | Score: 0.1062\n",
      "TURNS ID:  108 | Acc: 0.9568 | F1: 0.9568 | MCC: 0.9523 | MS Loss: 0.0064 | CE Loss: 0.1331 | Score: 0.1497\n",
      "TURNS ID:  109 | Acc: 0.9683 | F1: 0.9683 | MCC: 0.9648 | MS Loss: 0.0047 | CE Loss: 0.1022 | Score: 0.0917\n",
      "TURNS ID:  110 | Acc: 0.9677 | F1: 0.9676 | MCC: 0.9641 | MS Loss: 0.0051 | CE Loss: 0.1125 | Score: 0.1060\n",
      "TURNS ID:  111 | Acc: 0.9606 | F1: 0.9603 | MCC: 0.9563 | MS Loss: 0.0061 | CE Loss: 0.1349 | Score: 0.1670\n",
      "TURNS ID:  112 | Acc: 0.9630 | F1: 0.9628 | MCC: 0.9590 | MS Loss: 0.0055 | CE Loss: 0.1176 | Score: 0.1339\n",
      "TURNS ID:  113 | Acc: 0.9640 | F1: 0.9639 | MCC: 0.9600 | MS Loss: 0.0055 | CE Loss: 0.1164 | Score: 0.1235\n",
      "TURNS ID:  114 | Acc: 0.9615 | F1: 0.9614 | MCC: 0.9573 | MS Loss: 0.0059 | CE Loss: 0.1249 | Score: 0.1333\n",
      "TURNS ID:  115 | Acc: 0.9681 | F1: 0.9680 | MCC: 0.9646 | MS Loss: 0.0047 | CE Loss: 0.1049 | Score: 0.1037\n",
      "TURNS ID:  116 | Acc: 0.9694 | F1: 0.9694 | MCC: 0.9660 | MS Loss: 0.0050 | CE Loss: 0.1124 | Score: 0.1184\n",
      "TURNS ID:  117 | Acc: 0.9701 | F1: 0.9700 | MCC: 0.9669 | MS Loss: 0.0045 | CE Loss: 0.0994 | Score: 0.1166\n",
      "TURNS ID:  118 | Acc: 0.9720 | F1: 0.9719 | MCC: 0.9689 | MS Loss: 0.0043 | CE Loss: 0.0924 | Score: 0.1029\n",
      "TURNS ID:  119 | Acc: 0.9703 | F1: 0.9704 | MCC: 0.9670 | MS Loss: 0.0045 | CE Loss: 0.0978 | Score: 0.1213\n",
      "TURNS ID:  120 | Acc: 0.9757 | F1: 0.9758 | MCC: 0.9730 | MS Loss: 0.0039 | CE Loss: 0.0860 | Score: 0.0963\n",
      "TURNS ID:  121 | Acc: 0.9722 | F1: 0.9720 | MCC: 0.9691 | MS Loss: 0.0041 | CE Loss: 0.0887 | Score: 0.1010\n",
      "TURNS ID:  122 | Acc: 0.9708 | F1: 0.9707 | MCC: 0.9676 | MS Loss: 0.0045 | CE Loss: 0.0942 | Score: 0.1095\n",
      "TURNS ID:  123 | Acc: 0.9664 | F1: 0.9663 | MCC: 0.9628 | MS Loss: 0.0051 | CE Loss: 0.1068 | Score: 0.1171\n",
      "TURNS ID:  124 | Acc: 0.9663 | F1: 0.9660 | MCC: 0.9626 | MS Loss: 0.0052 | CE Loss: 0.1106 | Score: 0.1135\n",
      "TURNS ID:  125 | Acc: 0.9607 | F1: 0.9606 | MCC: 0.9564 | MS Loss: 0.0057 | CE Loss: 0.1216 | Score: 0.1309\n",
      "TURNS ID:  126 | Acc: 0.9691 | F1: 0.9689 | MCC: 0.9657 | MS Loss: 0.0047 | CE Loss: 0.1035 | Score: 0.1113\n",
      "TURNS ID:  127 | Acc: 0.9587 | F1: 0.9591 | MCC: 0.9543 | MS Loss: 0.0063 | CE Loss: 0.1344 | Score: 0.1724\n",
      "TURNS ID:  128 | Acc: 0.9576 | F1: 0.9574 | MCC: 0.9531 | MS Loss: 0.0063 | CE Loss: 0.1369 | Score: 0.1473\n",
      "TURNS ID:  129 | Acc: 0.9703 | F1: 0.9703 | MCC: 0.9670 | MS Loss: 0.0043 | CE Loss: 0.0955 | Score: 0.0955\n",
      "TURNS ID:  130 | Acc: 0.9666 | F1: 0.9666 | MCC: 0.9629 | MS Loss: 0.0050 | CE Loss: 0.1091 | Score: 0.1022\n",
      "TURNS ID:  131 | Acc: 0.9681 | F1: 0.9681 | MCC: 0.9646 | MS Loss: 0.0048 | CE Loss: 0.1053 | Score: 0.0997\n",
      "TURNS ID:  132 | Acc: 0.9680 | F1: 0.9676 | MCC: 0.9645 | MS Loss: 0.0048 | CE Loss: 0.1050 | Score: 0.0929\n",
      "TURNS ID:  133 | Acc: 0.9658 | F1: 0.9656 | MCC: 0.9621 | MS Loss: 0.0051 | CE Loss: 0.1093 | Score: 0.0955\n",
      "TURNS ID:  134 | Acc: 0.9734 | F1: 0.9733 | MCC: 0.9705 | MS Loss: 0.0041 | CE Loss: 0.0908 | Score: 0.0861\n",
      "TURNS ID:  135 | Acc: 0.9742 | F1: 0.9740 | MCC: 0.9713 | MS Loss: 0.0039 | CE Loss: 0.0895 | Score: 0.0981\n",
      "TURNS ID:  136 | Acc: 0.9731 | F1: 0.9729 | MCC: 0.9701 | MS Loss: 0.0040 | CE Loss: 0.0900 | Score: 0.0979\n",
      "TURNS ID:  137 | Acc: 0.9653 | F1: 0.9652 | MCC: 0.9616 | MS Loss: 0.0052 | CE Loss: 0.1092 | Score: 0.1412\n",
      "TURNS ID:  138 | Acc: 0.9666 | F1: 0.9664 | MCC: 0.9629 | MS Loss: 0.0051 | CE Loss: 0.1075 | Score: 0.1211\n",
      "TURNS ID:  139 | Acc: 0.9486 | F1: 0.9484 | MCC: 0.9432 | MS Loss: 0.0074 | CE Loss: 0.1554 | Score: 0.1676\n",
      "TURNS ID:  140 | Acc: 0.9740 | F1: 0.9737 | MCC: 0.9711 | MS Loss: 0.0041 | CE Loss: 0.0895 | Score: 0.0993\n",
      "TURNS ID:  141 | Acc: 0.9647 | F1: 0.9645 | MCC: 0.9610 | MS Loss: 0.0054 | CE Loss: 0.1137 | Score: 0.1386\n",
      "TURNS ID:  142 | Acc: 0.9714 | F1: 0.9711 | MCC: 0.9682 | MS Loss: 0.0044 | CE Loss: 0.0966 | Score: 0.0991\n",
      "TURNS ID:  143 | Acc: 0.9703 | F1: 0.9701 | MCC: 0.9670 | MS Loss: 0.0046 | CE Loss: 0.1002 | Score: 0.1277\n",
      "TURNS ID:  144 | Acc: 0.9715 | F1: 0.9716 | MCC: 0.9684 | MS Loss: 0.0043 | CE Loss: 0.0939 | Score: 0.0972\n",
      "TURNS ID:  145 | Acc: 0.9700 | F1: 0.9701 | MCC: 0.9667 | MS Loss: 0.0046 | CE Loss: 0.0996 | Score: 0.1109\n",
      "TURNS ID:  146 | Acc: 0.9757 | F1: 0.9756 | MCC: 0.9730 | MS Loss: 0.0038 | CE Loss: 0.0833 | Score: 0.0962\n",
      "TURNS ID:  147 | Acc: 0.9700 | F1: 0.9696 | MCC: 0.9667 | MS Loss: 0.0045 | CE Loss: 0.0990 | Score: 0.1229\n",
      "TURNS ID:  148 | Acc: 0.9749 | F1: 0.9747 | MCC: 0.9722 | MS Loss: 0.0039 | CE Loss: 0.0846 | Score: 0.0975\n",
      "TURNS ID:  149 | Acc: 0.9722 | F1: 0.9721 | MCC: 0.9691 | MS Loss: 0.0044 | CE Loss: 0.0948 | Score: 0.0915\n",
      "\n",
      "Average Time Cost:  \n",
      "Acc: 0.65463539 \n",
      "F1: 0.65898836 \n",
      "MCC: 0.66416161 \n",
      "MS Loss: 0.65575845 \n",
      "CE Loss: 0.75356098 \n",
      "Score: 0.72884554\n"
     ]
    }
   ],
   "source": [
    "import torch\n",
    "from torch import nn\n",
    "import torch.nn.functional as F\n",
    "import torchvision.datasets as dsets\n",
    "import torchvision.transforms as transforms\n",
    "from sklearn.metrics import precision_score, recall_score, f1_score, matthews_corrcoef, roc_auc_score\n",
    "import numpy as np\n",
    "import itertools\n",
    "from itertools import permutations, combinations\n",
    "import time\n",
    "from pylab import *\n",
    "\n",
    "torch.manual_seed(1)                      # reproducible\n",
    "torch.set_printoptions(threshold=np.inf)  # print all\n",
    "\n",
    "def findByRow(mat, row):\n",
    "    return np.where((mat == row).all(1))[0]\n",
    "\n",
    "\n",
    "# Set Hyper Parameters\n",
    "\n",
    "NAME = 'CNN-MNIST'\n",
    "OUTPUT_CLASS = 10\n",
    "LR = 0.01\n",
    "BATCH_SIZE = 64\n",
    "TURNS = 150\n",
    "\n",
    "K = 3\n",
    "T = 20\n",
    "K_release = K-1\n",
    "T_max = T-1\n",
    "RELEASE_LISTS = [\n",
    "    [2,3],\n",
    "    [3,2]\n",
    "]\n",
    "RELEASE_FACTOR = 0.1\n",
    "\n",
    "\n",
    "# Load MNIST Dataset and Create Loader\n",
    "\n",
    "train_data = dsets.MNIST(root='../Datasets', train=True, transform=transforms.ToTensor(), download=True)\n",
    "test_data = dsets.MNIST(root='../Datasets', train=False, transform=transforms.ToTensor())\n",
    "train_loader = torch.utils.data.DataLoader(dataset=train_data, batch_size=BATCH_SIZE, shuffle=True)\n",
    "test_loader = torch.utils.data.DataLoader(dataset=test_data, batch_size=BATCH_SIZE, shuffle=False)\n",
    "train_X = train_data.train_data.type(torch.FloatTensor)/255.\n",
    "train_Y = train_data.train_labels.numpy()\n",
    "test_X = test_data.test_data.type(torch.FloatTensor)/255.\n",
    "test_Y = test_data.test_labels.numpy()\n",
    "train_Y1 = train_data.train_labels.type(torch.LongTensor)\n",
    "\n",
    "\n",
    "# Model Class\n",
    "\n",
    "class CNN(nn.Module):\n",
    "    def __init__(self):\n",
    "        super(CNN,self).__init__()\n",
    "        self.conv1 = nn.Conv2d(1,16,kernel_size=3,stride=1,padding=1)\n",
    "        self.pool = nn.MaxPool2d(2)\n",
    "        self.conv2 = nn.Conv2d(16,32,kernel_size=3,stride=1,padding=1)\n",
    "        self.pool = nn.MaxPool2d(2)\n",
    "        self.fc1 = nn.Linear(32*7*7,16)\n",
    "        self.fc2 = nn.Linear(16,10)\n",
    "\n",
    "    def forward(self,x):\n",
    "        x = self.pool(F.relu(self.conv1(x)))\n",
    "        x = self.pool(F.relu(self.conv2(x)))\n",
    "        x = x.view(-1, 32 * 7* 7)\n",
    "        x = F.relu(self.fc1(x))\n",
    "        x = self.fc2(x)  \n",
    "        return x\n",
    "\n",
    "\n",
    "# Instantiate\n",
    "\n",
    "model = CNN()\n",
    "print(model)\n",
    "loss_fn = torch.nn.CrossEntropyLoss()\n",
    "optimizer = torch.optim.Adam(model.parameters(), lr=LR)\n",
    "MS_loss_fn = torch.nn.MSELoss()\n",
    "\n",
    "\n",
    "# Model Training\n",
    "\n",
    "turns_record = []\n",
    "loss_record = []\n",
    "MS_loss_record = []\n",
    "train_acc_record = []\n",
    "F1_record = []\n",
    "MCC_record = []\n",
    "Score_record = []\n",
    "\n",
    "total_loss_time = 0\n",
    "total_MS_loss_time = 0\n",
    "total_acc_time = 0\n",
    "total_F1_time = 0\n",
    "total_MCC_time = 0\n",
    "total_Score_time = 0          \n",
    "\n",
    "device = torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\")\n",
    "model = model.to(device)\n",
    "\n",
    "train_x_tensor = None\n",
    "train_y_tensor = None\n",
    "for batch_idx, data in enumerate(train_loader, 0):\n",
    "    if batch_idx > 100:\n",
    "        break\n",
    "    inputs, target = data[0].to(device), data[1].to(device)\n",
    "    if train_x_tensor == None:\n",
    "        train_x_tensor = inputs\n",
    "        train_y_tensor = target\n",
    "    else:\n",
    "        train_x_tensor = torch.cat([train_x_tensor, inputs], dim=0)\n",
    "        train_y_tensor = torch.cat([train_y_tensor, target], dim=0) \n",
    "    \n",
    "train_y_npy = train_y_tensor.numpy()\n",
    "y_train_onehot = torch.nn.functional.one_hot(train_y_tensor)\n",
    "\n",
    "turns_index = 0\n",
    "for turns in range(1, 6):\n",
    "    if turns_index >= 150:\n",
    "            break\n",
    "            \n",
    "    for batch_idx, data in enumerate(train_loader, 0):\n",
    "        inputs, target = data[0].to(device), data[1].to(device)\n",
    "        optimizer.zero_grad()\n",
    "        outputs = model(inputs)\n",
    "        loss = loss_fn(outputs, target)\n",
    "        loss.backward()\n",
    "        optimizer.step()\n",
    "    \n",
    "        if batch_idx % 5 == 0:\n",
    "            turns_index += 1\n",
    "            if turns_index >= 150:\n",
    "                break\n",
    "            \n",
    "            # Shared Part - train_output\n",
    "            SP_trainout_time_part_start = time.time()\n",
    "            train_output = model(train_x_tensor)\n",
    "            SP_trainout_time_part_stop = time.time()\n",
    "\n",
    "            # Shared Part - softmax\n",
    "            SP_softmax_time_part_start = time.time()\n",
    "            train_output_result = F.softmax(train_output, dim=1)\n",
    "            SP_softmax_time_part_stop = time.time()\n",
    "\n",
    "            # Shared Part - pred_train_y\n",
    "            SP_predy_time_part_start = time.time()\n",
    "            pred_train_y = torch.max(train_output, 1)[1].data.numpy()\n",
    "            SP_predy_time_part_stop = time.time()\n",
    "\n",
    "            # Acc\n",
    "            acc_time_part_start = time.time()\n",
    "            train_accuracy = float((pred_train_y == train_y_npy).astype(int).sum()) / float(train_y_npy.size)\n",
    "            acc_time_part_stop = time.time()\n",
    "\n",
    "            # MS Loss\n",
    "            MS_loss_time_part_start = time.time()\n",
    "            MS_train_loss = MS_loss_fn(train_output_result, y_train_onehot)\n",
    "            MS_loss_time_part_stop = time.time()\n",
    "\n",
    "            # CE Loss\n",
    "            loss_time_part_start = time.time()\n",
    "            train_loss = loss_fn(train_output, train_y_tensor)\n",
    "            loss_time_part_stop = time.time()\n",
    "\n",
    "            # F1\n",
    "            F1_time_part_start = time.time()\n",
    "            F1 = f1_score(train_y_npy, pred_train_y, average='macro')\n",
    "            F1_time_part_stop = time.time()\n",
    "\n",
    "            # MCC\n",
    "            MCC_time_part_start = time.time()\n",
    "            MCC = matthews_corrcoef(train_y_npy, pred_train_y)\n",
    "            MCC_time_part_stop = time.time()\n",
    "\n",
    "            # Score\n",
    "            Score_time_part_start = time.time()\n",
    "            output_results = train_output_result.detach().numpy()\n",
    "            prediction_pattern = np.argsort(-output_results)[:, :K]\n",
    "            confidence_pattern = T_max - np.floor((-np.sort(-output_results)[:, :K]) * T)\n",
    "            confidence_pattern[confidence_pattern == -1] = 0\n",
    "\n",
    "            total_score = 0\n",
    "            pattern_num = len(confidence_pattern)\n",
    "            release_factors = np.ones((pattern_num, K))\n",
    "\n",
    "            for i in range(pattern_num):\n",
    "                current_prediction_pattern = list(prediction_pattern[i])\n",
    "                if train_y_npy[i] in current_prediction_pattern:\n",
    "                    correct_index = current_prediction_pattern.index(train_y_npy[i])\n",
    "                    confidence_pattern[i][correct_index] = T_max - confidence_pattern[i][correct_index]\n",
    "                    release_factors[i][correct_index] = K_release\n",
    "\n",
    "                    for release_list in RELEASE_LISTS:\n",
    "                        if current_prediction_pattern[correct_index] == release_list[0]:\n",
    "                            for j in range(K):\n",
    "                                if j != correct_index and current_prediction_pattern[j] in release_list[1:]:\n",
    "                                    release_factors[i][j] = RELEASE_FACTOR\n",
    "\n",
    "            release_factors /= release_factors.sum(axis=1).reshape(-1,1)\n",
    "            confidence_pattern[confidence_pattern == 0] = 1e-7\n",
    "            confidence_pattern = -np.log(confidence_pattern / T_max)\n",
    "            total_score = (np.multiply(release_factors, confidence_pattern).sum()) / pattern_num\n",
    "            Score_time_part_stop = time.time()\n",
    "\n",
    "            print('TURNS ID: ', turns_index, '| Acc: %.4f' % train_accuracy, '| F1: %.4f' % F1, '| MCC: %.4f' % MCC, \n",
    "                  '| MS Loss: %.4f' % MS_train_loss.data.numpy(), '| CE Loss: %.4f' % train_loss.data.numpy(), \n",
    "                  '| Score: %.4f' % total_score)\n",
    "\n",
    "            turns_record.append(turns_index)\n",
    "            loss_record.append(train_loss.data.numpy())\n",
    "            MS_loss_record.append(MS_train_loss.data.numpy())\n",
    "            train_acc_record.append(train_accuracy)\n",
    "            F1_record.append(F1)\n",
    "            MCC_record.append(MCC)\n",
    "            Score_record.append(total_score)\n",
    "\n",
    "            SP_softmax_time = (SP_softmax_time_part_stop - SP_softmax_time_part_start)\n",
    "            SP_predy_time = (SP_predy_time_part_stop - SP_predy_time_part_start)\n",
    "            SP_trainout_time = (SP_trainout_time_part_stop - SP_trainout_time_part_start)\n",
    "\n",
    "            total_loss_time += ((loss_time_part_stop - loss_time_part_start) + SP_trainout_time)\n",
    "            total_MS_loss_time += ((MS_loss_time_part_stop - MS_loss_time_part_start) + SP_trainout_time + SP_softmax_time)\n",
    "            total_acc_time += ((acc_time_part_stop - acc_time_part_start) + SP_predy_time + SP_trainout_time)\n",
    "            total_F1_time += ((F1_time_part_stop - F1_time_part_start) + SP_predy_time + SP_trainout_time)\n",
    "            total_MCC_time += ((MCC_time_part_stop - MCC_time_part_start) + SP_predy_time + SP_trainout_time)\n",
    "            total_Score_time += ((Score_time_part_stop - Score_time_part_start) + SP_softmax_time + SP_trainout_time)\n",
    "\n",
    "print('\\nAverage Time Cost: ', '\\nAcc: %.8f' % (total_acc_time/TURNS), '\\nF1: %.8f' % (total_F1_time/TURNS), \n",
    "      '\\nMCC: %.8f' % (total_MCC_time/TURNS), '\\nMS Loss: %.8f' % (total_MS_loss_time/TURNS), \n",
    "      '\\nCE Loss: %.8f' % (total_loss_time/TURNS), '\\nScore: %.8f' % (total_Score_time/TURNS))      \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "2f1ca702",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXQAAAEGCAYAAAB1iW6ZAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuNCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8QVMy6AAAACXBIWXMAAAsTAAALEwEAmpwYAABEzklEQVR4nO29e5wcVZn//36qpnsCCcwkIReSAAkkQBKSmcAATlQcCWC4uFFBRVzBG8h+ZV10Udf15y67KOiSXdEVJYAgKGuQiwjIJRgcQTNcJmRCSAIh3APkQpIZkgDpS53fH1XVU11TfZmZnpnunuf9euXV6erqqqd6uj/nqc95zjlijEFRFEWpfKyhDkBRFEUpDSroiqIoVYIKuqIoSpWggq4oilIlqKAriqJUCTVDdeIDDjjATJ06dahOryiKUpGsXLnyLWPMuKjXhkzQp06dSnt7+1CdXlEUpSIRkVdyvaaWi6IoSpWggq4oilIlqKAriqJUCSroiqIoVYIKuqIoSpVQUNBF5AYR2Soiz+R4XUTkpyKyUUSeFpGjSx+moiiKUohiyhZ/BfwMuDnH66cCM7x/xwO/8B4HhPUPPczOh5YzqWUiUw/ZDeNbYFzzQJ1OURSlYigo6MaYR0Rkap5dFgE3G3ce3sdEpF5EDjTGvFmqIH3WP/QwUz96GjMOSWDPNJhOC7Fr4cTlKuqKogx7SuGhTwZeCzzf5G3rgYhcICLtItK+bdu2Xp9o50PLiSWT1BxpoAYEB5wEbG3tU+CKoijVRCkEXSK2Ra6aYYy51hjTZIxpGjcucuRqXkafvIBkLEbqWQtSYLDAiru2i6IoyjCnFEP/NwEHBZ5PAd4owXF7MPPkE1l/z32uhz5BPXRFUZQgpRD0u4GLRGQpbmdo10D45z4zTz4RTj5xoA6vKMogc9MfN/CHZV0sOqWO804/vCzP3ZcYw+8ZjOuUQmuKishvgRbgAGAL8O9ADMAYc42ICG4VzELgHeALxpiCs241NTUZnZxLqUYG4odb6Ji9Pedgio1/7HFjYdt2OPKdNKPWptk92+aht2p4956pNKR30WXZzDpyEzXzLZ7d1+7TtRZ7Hb9d/Ayrbnyb9c9OYX8nnTn3vC/sz2cuOarHMa/82GgaUrtZXTOKb961EyDymqZ9oo7EzDg/u/7tzHV12Pvx3vteZ5/HJmddZ9S5ikFEVhpjmiJfG6pFolXQlXInlzjkE0Mg8+Mv9MMtdBxfMJ5aJVnisO9HX+bkA1KMWpvGPkDY/gIZYfJfP3qeYdt2ImMPis3qmlG8/3sv87fLpkYKVvD9v138DC/d2YV9gJB+yzDtE3VZ1xW+nh6iSZwZ7OJUtmBjSCE8zhjexw5st0cMB0ghPMAEltkH9LiW4Ofrx+rHFxTeqM8p/ZZh92ybDRuELz+ylxhO5pz+YxKL60+oZewCK3POx2/awd/dliCGQxKL/zhoDLvemOiKc+iaklhcbU1luvNOwetMYrHzyjG9FnUVdKXiKJRBFiu2fT0n0EMcjp5neohrMPPqsPdj/0mb+ffXdmSJhf/DBXjpzq4emWlYVDvs/RCIFIxc4pAthhN5nlHUkaTDHsW+H32Zi768f+aaTk5tzxKga/cbywW7tmcE6/oTanl2xaFZohlfn2D0N3te1/Un1HL44YbUCierUZk86xW+sWZPD9E0uFUU4v0//Dy4PRlxLcHP1z+/02j1EN58jcULjOIIdmGHzilACniKeh5hHHUkWV0zkvM/tIGjliexvdc3MIrp7MkcN3gNKQAEC1PwOlPAE80x/nXF+3v1PVVBV8qWKAEOZlpdls20g97g5dcmsb+TzplR+iIfFuGLvrx/ZAPQsbWD9i3tNE1oonF8Y4/b6iPnv5jJ4oIimU9cc4lFCvhT3b60dL0XKTZhUU0hCEQKRj4RDL8eFPhl9tiMGMZxsgToTUYwifcyghWMNYnFD4/Yn+Psd/jIukSP6wqLWzGiGXUN4X1yXUvwuCngjzKBrWYEHfZ+nDhlIx975b3MOfM1Fv4xww1iVMb+txNr+PBfE5hEdKMUPG7aE/NcMQTPrRl6gPauPazo3M38+lE01Y0sYWSKT3991bB36j/6GXD41t8X5v/3qccymVbUDyycUf7wiP054pwkz/1fjH957u0sEX6oZkyPBuAzv36avy17kIOfnMeGQ7ZgHTSH+nWjMgKexKK1bgQnd73T44dZSFyjxCJK4MJZXVBU0952i+JEMFf2G852c8XQM9aRHMHujGg+RT2PWGO4yHmZGhzsIs8ZFZsBHBtePsRm2itpxAFsmPzlA1ljv8Mbf9nN7PVpJF34uI6AMdKjca3JkR2HP/MXxtuM/eQojp08hidf38H223YzfWu6R6O1bFaceV/Yn/r/e5v4qkTkd8KxYd3MGiZ9aCRjruuCpHtd484Yy7Y/bncPFLjODU+908OyKpaqE/T2rj2c1bGRpGOIWcLtjdNV1IsgKisNCm6uLDnsVYbJlWV/4RMHYxI1zGI3jXTSRSxz6yzAyem3sm79fWGuuzXZIxsM/sDCGaV/i3wC2ziGzqwfXCKiAbhjjsOZa6yA8E8AMZxhtvS4rQ6LQyFx9WNMzouzZ/4+dNy6lxPe2lsw2w6KqtR4G5Pd+1GTLQ6O5QrIu9Ms0m8ZDj96X+ak9+X1X76Z9b5c55IaSBwaI7ah20p4fbTFgTtNpJWTxOLexhom7WOwDxDqH09niV8xohkbGyO5PUl9Sz11zXV0tXXR2dqZee7T1dbF5ps357wW//NNTqxh5P3ZjW4S4bVDLQ59Ld3jc9rnpbTbWDhADI5pndfjvCtbVkGi+5xJ7/vxUM1YvnvFFiZ8962MWE/+8oHsN2+/rGvyjxO8rlzX2VeqTtB/+soWfvTim6QBG/j2oQfytUMmlDS+aqNjawefufqndK5rILbf29Rt+zDm/iMzPm3Ybw13BP3wiP35wn/vW7CDze9Q++6/J6lbOYlT2MqpbO5xSy6QJZZBYQ7aGoVuiaMe82W/KWDtyBEctee9HkJgWQbLyc5WM+KQR1x7ZGIBsfj9eSvZ/+ZdWaI55W0nk5mGRTU5L878q2cD0NnaWbQI+vivx8bG2LVqV5YoSsQ5VrascgUqBrsX7JsRyBTwzhhh5A6Teb7r3P34+E3HZM7jv9cXzUkfGtndqIQ+h74QeS2B42biT2Rf365z9+PEC6fnbCyK/fxu/98XOPIZJ5Nw3P3JOFd8fWZJxbkvVJ2ghzP0+w/Zzsw9jw+LQUa5bIxwlt3WBq2t0NICzc3wnd/czT1feB8NqT09OtrCHWoP1Yzp0RH0FPXcZE/p0TkYzrL9CgB/e1SGG7YUct36P3eUzaQPjWTDU+9w+NH7MnLFu5lb3rDgBDP49OE1xF5KYZI9Bd+9NR/N+9iZuXb/fe+cui+xzamscwTFoZC4RolFUPiCQuTvB/R4vZRCkZXtRohsMOZwLEf+7+E8+48bcsaWL8MeCNHL9fnmu77+EG6Mgw3aUFJ1gg7dHvrJZi0zHz8D0gmw42U/UVdfqzP8bHjlvXMwKbuHjRG0M5z3b+OFxw4nlRLicfjzwxYPXPMCzTe/nvGl83mfCSx+c/C+fP6N3Ugq+7b7n5kLwCls7iHY+ayK4DmoCbwhlKUGhTn8AwqLY0ZwArfIxLtFc8VX12aJc9BSSCGsG70Pc3e9g6TpmfWVUGCLzQoHMusr9hwDbRcMFAMRZ1RjXA6fQVUKeoa1V8Dq7wFpEBvmXgazv9P/45aYLEFO20gsxY13vtqjOiOqDhjw/OgYs9jFKWzpYWNEl62l6JD9aLj4dVo2vc5ht43o4Uvn8j6fop5H7dGcM/UVxr/gBLzl/ZjO7qI6ngzuf6yYMPGLE7P8RojIUiOEOfwDyiU44ezZ3zefpZDr1rxSREwZeMrxu1Ddgr6tDR5e4M66aA1thh41MCS7pnmPl0m7QnvEP77AnT85Nquqw6+tvfOxBkyyBuw0Yw59lYkbDsj40bk66nKVRv3o9JWk3xnFdx85lFjazsqU/Y4d36MM2hTFlGz5xwln2cHjTzx3YsEfQz5h7g/5LIVyybgUpTdUp6AHTeLpuFPoDqGHnqnq8ER4ttlVcLDJ9SfU8tFvjWb/3+3K8uqWzYrzx3VH0sguuqjJ+N2FsuIoGyUtDvfKBLY6+/K2LXz0E89yxklzI0Wzq62rh03hZ7EvX/oyOx7aiZhowYaAWNrFC/lgU44Zl6L0hnyCXorJuQaftjZYsAASCYjHYflyaB54myWfz/2HZV3MTLzjinDK5iJedDPutMWDr00g5tXvGvAeHWofGc/nV0zizPkdXFgDjle2804c/ptncvrdUTZGsBLAJEEwGAuMWJyafsvttExbtO0zg0kXTIq8vrrmOuZfPZuVLatwkmDF4MQLp1PXXMfUS6fS9WgXTsLBst1zhwX7mNZ5ZS+Wdc11ZRubovSXyhT01lZXzNNp97G11S3lGECCGfjvf57mZ2e0Z0YhAnzyoH0Y7YmwQ3C0mMMMdpP25NjP0NMI40kwK/UudzxyLM/WbOf8Ezdw3V8Op6FjNzFeyTQAmWqQgJDnyn4nnjsxy7p45Yn3SN71ZiaWk/bN/yeva66LFOa65joaljfkFWwVS0UZWipT0Fta3Mzcz9BbWgb8lMEMvCNVR/tdR/P5e5MZYW9J7seL1luIA5YYsNz2xgaOYBcphD2n7kvTx6bQftcmRtz/LmfwJgvZ7A7NTo/jys566lI240mQRhDPYnGgaD86LKr1bV2sun8LJuFQE7eYe259wWvNJcwq2IpS3lSmoDc3uzZLsNB6AAhaLMEMPDM/RmoC7XcdzRfuS3DLDzYyudbCSTjYcYvpV01n2x3b2PHQTmwDlmU48oMTmHTBJBq2J3nxwZdc8cfwUd7kI2YLqyfV0LCyez6P3V4D0J9OwrrmOub9OX9mrShKdVCZgg6uiA+gzRK2WL55+EoWioNlukU4k10nx3Hba+9yY8iSGDlnZMZ3tuNWptKivqUeu9bCec8BN5mn1nJYuHckO6ydrtAHGoD+opm1ogwPSrGmaFXyh2VdbsWKqYFUnD+um0XCWBhxX7eAuCfs/22ewepwuGvHFg75ziE9fOdpl02jYXlDj+0HfuVArFoBG+xai3FnjsOutTLP/QZAURSlGCq3bHEAiJ4Pu7t2vAuLU456mYbnHUzCYEz3QJwbOYTfxg/MDBYqlkodmacoytBQfWWLA0DQYrnrmhTfvvgOfmJNAiR7oYLzxtD4/oPYfPNmNt+4mXTCkDIWHYzBpGr4w7Iuzju9+POG7RC1RxRF6Ssq6B5Bi8UkYfNNB0EyhY1kascty2F+cr+M6E48dyIPX7ORG245gEZnB2IlMtm9oijKYKOC7rHolDo2/Hynu9qNGc0TW+bwadYgksYyAlZPX7uuuY4TmU7d0lWZCVWOGTNvyK5BUZThjQq6x8fGTOAgaxUGSPEqDzCRnzGNhR/q4hOfGZOzdLCztRMrjb/+FJ2tnWqZKIoyJAxrQfdX8Nmydhry0xE0e1PF+mWJCSz2+0wDky7ILdD1LfVYcbf+3IprZYqiKEPHsBV0fwWfLY9+hM5HP8DM1B6O5hlqxUG86pURlsOU7Z1A/pGZhYbEK4qiDAbDVtBvXfYqz/3wakwiDliso5Z/lqP4yuy1blliyhSdcWtliqIo5cCwFfTd65uYmXyXRrbSRQ11JFltj6TuhxNoHDNBM25FUSqOYSvoZx8yktPN6qwpav0qFc24FUWpRIbl0P+OrR1sXvsnai13jnJ/wWLLq1JRFEWpRIZVhn7THzfwm3s38dzevzH15Yn8f1YdcWoyNeRapaIoSiUzbAQ9M7Q/cSjwYV4Th2/Y2/namU/lXJJNURSlkhg2gp4Z2k8NYMDYrHXGsGbGXC4owRS1iqIoQ82w8dAXnVKHxFIgKWbRxTnyMo01uznnDBVzRVGqg6IydBFZCPwEd46q640xPwy9Xgf8BjjYO+ZiY8yNJY41J+1de1jRuZv59aNoqhsZuc95px9O/AfPsO2mncx+No2VBhGLWTSQb+CQoihKpVBQ0EXEBq4GTgY2AU+KyN3GmHWB3b4KrDPGfFRExgHPicgtxpjEgEQdoL1rD2d1bCTpGGKWcHvj9EhR72rrYvK/7eBAb5UgAFKOzr2iKErVUIzlchyw0RjzoifQS4FFoX0MsJ+ICDAK2IG77sOAs6JzN0nHkAaSjmFF5+7I/TpbO3ESATEXrWpRFKW6KEbQJwOvBZ5v8rYF+RkwE3gDWAP8kzHGCR9IRC4QkXYRad+2bVsfQ85mfv0oYpZgAzFLmF8/KnI/fxItbJC4cOBXDsxaFk5RFKXSKcZDl4ht4XXrPgJ0ACcChwEPicijxpi3s95kzLXAteAuQdfraCNoqhvJ7Y3TC3roOomWoijVTjGCvgk4KPB8Cm4mHuQLwA+Nu0DpRhF5CTgSeKIkURagqW5kTiGH7mlymw5rorG5cTBCUhRFGXSKsVyeBGaIyDQRiQNnA3eH9nkVWAAgIhOAI4AXSxloX/Gnyf3ef+7hM1f/lI6tHUMdkqIoyoBQMEM3xqRE5CLgQdyyxRuMMWtF5ELv9WuAy4BficgaXIvm28aYtwYw7qLJTJObjLHl7iS3zniIxr9vHOqwFEVRSk5RdejGmPuA+0Lbrgn8/w3glNKGVhp2r2/CpGLu4s8p97miKEo1UvUjRc85YxK1cQvLdqiNWzoyVFGUqqXq53JpboblP9nFxjs6mX5mPc1a3aIoSpVS9YLe1dZF+uLVHJJwSD9q0TVHa88VRalOqt5yyYwQTYOTcHQBC0VRqpaqFvSOrR08etCjEANsHeqvKEp1U7WWS8fWDs5fdj6JdIIZ35rBJalLmHnGTLVbFEWpWqo2Q2/f0k4incDBYeNhG3nm7GdUzBVFqWqqNkNvmtBE4sVj6FzXSP2sDpoWav25oijVTXUJ+rY22NoK41t494VmXvqv60kkYOe98O5CC8YPdYCKoigDR/UI+rY2eHgBpBNgx3l+03JSyWaMA6kktLa6NemKoijVSvUI+tZWV8zd+kQ+NKuVufYsZjudrLXraWlR/1xRlOqmOgS9rQ1WvAqTvMux4owd0cT/yGoMjq4dqijKsKDyBb2tDRYsgEQCjrThyvOh6Vy2Xz8JUi8hBl07VFGUYUHlly22trpink7Ds2noOBjGNWctOacDihRFGQ5Ufobe0gLxuCvq8bj7HF1yTlGU4UflC3pzMyxf7mbqLS3Q3KxLzimKMiypfEEHV9S9msTgkP+4Hee6U66jcXzj0ManKIoyCFS+hx4iOOQ/6SRp39I+1CEpiqIMCtWRoQfQIf+KogxXqk7Q332hUYf8K4oyLKk6y6W1FVJJC+NYpJIWra1DHZGiKMrgUHUZeksLzLW7dMi/oijDjqoT9Fl0cf30W6ibvoqujfM4jM+iQ/4VRRkOVJ2gv7NiOQ2XfAOrJomTirF1xUTqmj8x1GEpiqIMOFXnoY+etRqrJonYDlKTZPSs1UMdkqIoyqBQdYI+omkh1NRijI3U1LrPFUVRhgFVZ7kwrhk5aXlm5SLG6aoWiqIMD6pP0MEVcRVyRVGGGVVluXRs7eD6NdfTsbVjqENRFEUZdKomQ9dJuRRFGe4UlaGLyEIReU5ENorIv+TYp0VEOkRkrYj8pbRhFqZ9Szu77YPYvf/p7Kk5WCflUhRl2FEwQxcRG7gaOBnYBDwpIncbY9YF9qkHfg4sNMa8KiKDPnvKvvs10Tl+LkZs3jFp9t0vPtghKIqiDCnFZOjHARuNMS8aYxLAUmBRaJ9zgDuNMa8CGGO2ljbMwrwlB4LEQWyQuPtcURRlGFGMoE8GXgs83+RtC3I4MFpEWkVkpYicG3UgEblARNpFpH3btm19izgH+20ahbNXMClw9gr7bRpV0uMriqKUO8V0ikrENhNxnGOABcA+QJuIPGaM2ZD1JmOuBa4FaGpqCh+jX2x+ZCSjb5nI2Lk72f70aDZ/diS8v5RnUBRFKW+KEfRNwEGB51OANyL2ecsYswfYIyKPAA3ABgaJE8Z2MW/tc8TWOiTZyn5jR6CTcimKMpwoxnJ5EpghItNEJA6cDdwd2ucPwAdFpEZE9gWOB9aXNtT8TNneyQjLwQZGWA5TtncO5ukVRVGGnIIZujEmJSIXAQ8CNnCDMWatiFzovX6NMWa9iDwAPA04wPXGmGcGMvAw9S31WLUWTsLBilvUt9QP5ukVRVGGHDGmpFZ20TQ1NZn29tLWine1ddHZ2kl9Sz11zWq3KIpSfYjISmNM5GLJVTNSFKCuuU6FXFGUYUtVzeWiKIoynKm8DH1bW/FT4/r7xsdCYrtOp6soSlVTWYK+rQ0eXgDpBNhxOHF5boHO7LsXt5/WArs2/3sURVEqmMqyXLa2umJOGpyE+7zgvo63wSn8HkVRlAqmsgR9fIubmYsNVtx9XmjfzCVahd+jKIpSwVSW5TKu2bVMivHQg/uqh64oyjCgsgQdere8nC5FpyjKMKKyLJdCtLXBFVe4j4qiKMOMysvQc9HWBgsWQCIB8TgsXw7Nmp0rijJ8qJ4MvbXVFfN02n1sbc2977Y2WHuF+6goilIlVE+G3tLiZuZ+ht7SEr1fb2rZFUVRKojqEfTmZtdmaW11xTyX3RJVy66CrihKFVA9gg6uiDc30961hxWvbGF+/Sia6kZm7+PXpzsJrUtXFKWqqC5BB9q79nBWx0aSjiFmCbc3Ts8W9d7UsiuKolQQVSfoKzp3k3QMaQDHsKJzd88sXevTFUWpQqqnysVjfv0oYpZgAzFLmF8/aqhDUhRFGRSqIkPv2NpB+5Z2miY00TS+kdsbp7Oic3e0h64oilKlVLygd2zt4Pxl55NIJ4jbca475TqaxjeqkCuKMuyoeMulfUs7iXQCB4ekk6R9S2nXKVUURakUKl7QmyY0Ebfj2GITs2I0TYhcO1VRFKXqqXjLpXF8I9edcl3GQ28c3zjUISmKogwJFS/o4Iq6CrmiKMOdirdceoVOyqUoShVTFRl6UeikXIqiVDlVkaF3tXXxyhWv0NXWlXun3iwwrSiKUoFUfIbe1dbFqg+vxiQcJG4x788N1DXX9dxRJ+VSFKXKqfgM/embO0nvdRADqb0OT9/cGb2jPynX3MvUblEUpSqp+Ay9g3pmYGFwSGGxnno+2NYGra2sbzqehw6f3T0FQK5Juba16eyLiqJUPBUv6E3n1vHVGxqYnexkbayem+atgwULMIkEU2tq+NPiJfzPnMae0+j6aGepoihVQsVbLs3NcHVrHbN+cAhXt9YxZ3srJBJIOk1NMsXxq9tJetPoRqKdpYqiVAlFZegishD4CWAD1xtjfphjv2OBx4BPG2NuL1mUBfAWKvJogXgck0iQqqnh8Yam/NPoamepoihVQkFBFxEbuBo4GdgEPCkidxtj1kXs9yPgwYEItGi8tUWltZWXm47npMNn82/5ptHtywpG6rkrilKGFJOhHwdsNMa8CCAiS4FFwLrQfv8I3AEcW9II+4KXss8EZhazf29WMFLPXVGUMqUYD30y8Frg+SZvWwYRmQx8HLgm34FE5AIRaReR9m3btvU21vJAPXdFUcqUYgRdIraZ0POrgG8bY9L5DmSMudYY02SMaRo3blyRIZYZvucutnruiqKUFcVYLpuAgwLPpwBvhPZpApaKCMABwGkikjLG3FWKIMuKvnjuiqIog0Axgv4kMENEpgGvA2cD5wR3MMZM8/8vIr8C7q0IMe9r52ZvPHdFUZRBoqCgG2NSInIRbvWKDdxgjFkrIhd6r+f1zcuWfJ2bvRV6rXpRFKUMKKoO3RhzH3BfaFukkBtjPt//sAaBqM7Ncc29r2LRqhdFUcqEih8pWgztXXv46StbaO/a070xV+dmb6tY+lr1oottKIpSYip+LpdCtHft4ayOjSQdQ8yS7jldcnVu9nbkaF9GmmpWryjKAFDdgt7WRuL39zL34MN5YnYDeHO6ZEaNRnVu9raKpS9VL7nsHkVRlH5QvYLe1gYLFtCcSHBrTQ2fXryEp+c05p7TJUhvq1h6u7/OH6MoygBQvYLe2pqZdXEE8C+vbiD+ubNyz+mSj1xVLP0pe9RadkVRSkzFCnrH1g7at7TTNKGJxvGNPXdoaYF43BX1eJz5Hz8D+irmUX53f31wrWVXFKXEVKSgd2zt4Pxl55NIJ4jbca475bqeou7NukhrqyvuzUWIZ1TGncvvVh9cUZQyoyIFvX1LO4l0AgeHpJOkfUt7dJaePVF6fnJl3Ln87krywXXgk6IMCypS0JsmNBG34ySdJDErRtOEpv4fNFfGncvvLpUPPtBiqyWSijJsqEhBbxzfyHWnXJffQ+8t+TLuXH53f33wwRBbtYYUZdhQkYIOrqiXRMh9hqLyZDDEtpKsIUVR+kXFCvqAMNiVJ4MhtloiqSjDhuEh6G1tvat2GSwGS2y1RLIb7SBWqpjqF3RvxCiJhFuXvnx5+Ym6CsvgoB3ESpVTsbMtdrV18coVr9DV1pV/R2/EKOm0+9jaOhjh9R2dhXHg0PVglSqnIjP0rrYuVi9YjZNwsOIWDcsbqGuui945MGI0HY9zxxFz2fv6W+xMpZlfP6pvUwEMFMEM0rLh0C/CtHM1iywV2kGsVDkVKeidrZ04CcdLtBw6WztzC7o3YnTTA8v42oRpPDZ6Ms6GTVhAPDidbjmQlUGmYeMSeOkmtQZKhXYQK1VORQp6fUs9VtzKZOj1LfX539DczJ2TDuXxF9/E8TY5QDI8ne5Q42eQ6fcA4/7T2vHSon0WShVTkYJe11xHw/IGOls7qW+pz52dB5hfP4qYJRjH4OB2HsQsyT+dbrgiYqArJPwM8qWb4cUbwaTUGlAUpWgqUtAB1lFHK3W0AMVIa1PdSG5vnM6Kzt2MrrELe+jhioijr4KnLh74Cgk/g5x2rtt4xMd2d95pZqkoSh4qUtD7WonYVDeyeHslXBHx2h2DO4TeP/ZQlNkNh1rt4XCNyrCjIgU9qhKx5KXl4YqIg86EbY/2v0KiN0IyFPOwDIda7eFwjcqwpCIFPVCJSDzuPu8L7V17WNG5O9p6iaqIqJ/Tv6yut0IyFGV2uWq1qymb1QnLlCqlIgW9L2tXAN1TAIwdy6bX3+TyCdN4fOZcYrnKF8MVEf2tkOitkPSmzK5UHbjhRiQ+tvqyWa1HV6qUihR06N3aFUC38b53LzgOkyyLW2Ixzlq8hI7ZDYNTvtgXISmmEQkPSDrwNHjzfnBSvRfhcCPSm0aoUnxprUdXqpSKFfRe4xvvjluJbjkOsWSKD6xuZ81RDWx6L0F7157+iXqUoPnb4mMhsd2tlklsL62QhAckvX5X92t9sRTCjUgxjVCl+dJaj65UIcNH0H3j3cvQsSykNs5+CxYAwi1vbOd3m3f0feRoVJnjzlVuPbmTBL/63a7tm9jly357DEjykf5bCsVms+pLK8qQM3wEPWi8jx0L27djt7TgTDqU9Itvkgboz8jRoKCl90L7Re7AoCyBdfomdvmyX1/ogw2ISYGUcC6YYrJZ9aUVZcgZPoIOPY33tjY+ccM1tAY6R+fXj8pf/ZKLoKAhYNJkizmAlVvs8mXgubLfKKH3ByTly6ZzdaD6tlBf7KBK9aUrxfdXotG/XxbDS9CDeJ2kUxIJbovHueM3t3HoghYAzurYSNIxuatfoggKWnysO6rUSXRnyqPn5RbLQv5zsLEQG/a8Cs9f6w122ktW5j/7O4WrYSJHwHrHyWcLFfrxVJovXWm+v5LNYPz9KqzBKErQRWQh8BPABq43xvww9PpngW97T3cD/2CMWV3KQEtOYHSSnUjwqeeepn1BC4tf2kzCm++l1xZMUNB6U7NeyH8Oz/Gy8Vpc8RXcu4A8mX+hc2VGwAamLYuKYaB/PEPxw1Hfv38MtdgN9N+vAhv8goIuIjZwNXAysAl4UkTuNsasC+z2EvAhY8xOETkVuBY4fiACLhmh0Unrm47nrI6NGTEvavKufPQmWy3Gfx7X7H5hnRTd4uuJ+cSTYM6lxZ0v1wjYYIYeFcNA/nh688MppYio7993ykHsBvrvV4ENfjEZ+nHARmPMiwAishRYBGQE3RizIrD/Y8CUUgY5IIQ6SXc+tJy5Bx/OE7MbsIATRu/HJdMmDs7UusX6z5lqlpA9UqyYh88VLqXM56FH/XhKJa7hDuU1l0ZfU6lFpFJ9/3KgHMRuoP9+FdjgFyPok4HXAs83kT/7/hJwf9QLInIBcAHAwQcfXGSIA4jfQbpgAc2JBLfW1PDpxUt4ek7j4Im5TzEZfZQY9+aLHBTg8S29E8fwjwdyr67UW6EPN1Sb/+TeNYRj6ouIlIPvP9TWxEBQLmI3kH+/CmzwixF0idgWLt9wdxT5MK6gfyDqdWPMtbh2DE1NTZHHGHQ8L13SaUYA//LqBuKfO6tXYt6nqpi+0tcvcDi7nXZeYXEMC5H/b1ubm0VnOmQDqyvlm2Y4eDzIPvaJy91jbv4TOX383opIOdgC5RBDbyh2ColyE7uBajQrrKO/GEHfBBwUeD4FeCO8k4jMBa4HTjXGbC9NeINAwEsX22b+rh1w6y2wfXtmoph8gt3etadvVTGDTTi7hfzimEuIMtt928fvmDW5pxmG7g5dJ+Vm9EjPqQnmXJp/Rsveikg52ALFxFAuGXxv1wAoF7GrtEZzAClG0J8EZojINOB14GzgnOAOInIwcCfwOWPMhpJHOZD4XvrNN8ONN8K112ZGkqZra/nFDf/Hf02cRtpEC/aKzt0kHdP/gUkDTTi7nXZu/pr1XEKU2e55+GOaoHN19+pK4WmGM5N7BUaxOoFO3eCxixHssIgUM4J2KG2BQjGUkxiF/+aDvQZAXyllw10ujWsfKSjoxpiUiFwEPIhbtniDMWatiFzovX4N8G/AWODnIgKQMsY0DVzYJaa52bVeUqlusXEczN4Eby9fTuKcL3nbsgW7vWsPm95LYIuAJ/h9rooZaHKJZaEO2LAQhbcfc5W7PXhcv2QzPjZQK+87bAJWzH2MWmKvN1lfeFKy8MjYcJ+Df7cwmD/UQo3UYN1FFCNUuSqghtonL0SpGu5iGtcyF/yi6tCNMfcB94W2XRP4/5eBL5c2tEEmNNeLY1kkYzX8rcFtlwSyRpL+bvMOlr65g7Qx2AKfnTSWT00cU57ZuU9vxDJfA1CoYfD/n2XNWGDVdIsu9P+HkSWGAR8/+EPMimWIsuBg38PaK7KveaDvIra1Zdtd+a4/6m9bzHiKwVp7d6D9/EKNazndTeVg+I4UDRMqY3zj9Tf52oRpdMycS1zg7ANdwQZ3JOlex3T3DBuYMiJe3mLeF3I1AMU0DGFrJqpWvjc/hijR2POq20g4Dlk+fviH2NsseCAEKpcYDGTnYuacQburwPWH/7aF/taDtfZuITEthZ9fqHGN+h7528skY1dBDxKY62UK8IOHHmbn/bcx+uQFzDyiEYCfvrKFZEDMBTh+/dN8ovUPsPCUAVgLr0IJ/zh6UysfJp9oWDZMXuTO/x5l4UTFki8L7o9A9WU+Hhi4zsXMOYN2V4nvAoLXld4L66/sOR1FKa5tMKypQo1r+HtU7OIvg2jTqKDnoq2NmYvOcEeS/mQxfPGLrP/EJ9l08IyMZ37sutV8+9GHOP6u27FSKbjyR8WvWO0xqCWPg0mxmWcxX/bwjznYWWeAA46DWd/Kn1UXmwXnO1c+IenNfDyD5UcHz1nK2TejzuGL+O4XKHo6it4I3WB9fvnssfD3qJhGZpBtGhX0XARXok6nMUuWMPXGG3l28RKOFrqFPJEA42VAvVyxumJKHvtKb2/Xc33Zi+msC54rn71RiL52DBb6cQ+ktZKLYs7Z3+zRP0dwDEHYYouazRN63wE5WJ9fvu9l+HtUqJEZjDuLACroufA7Sd97D4xBjKEmmeITy+7h08vupTaRwDKBsVEivVqxur1rT+REYEB1ZuxRFJvhbG3tudJTrs66HoOeevEjCp+r0CpT4ZG3xczHM9g+a75zlip7HNfccwxBUMzDneN2beGBbbkqmGZ/Jzp7LiXFinCuRqa334sSooKei1B9ukmlcCyLuc8/RzyR7BZzEYjF4ItfhHPP7ZGdR1kqfmYenghsdI1d3Rl7mHxf9kLVGVFCFSUexf6IeuOd54qtrwt6DxVh/zvXHDrFkEvcMucIzeYJ+f/2fRmJXCp6I8Lh72FUIzmId2Yq6PnwO0nPPRe5+WZiN95I43PrEOMOPKKmJkvI27v2sOKVLZla9GBpY1Cg/cFIDnDs2tWc89waGv/uNB5KTRyQQUpl69Pny3B6W50BxVXWFHxvAe88X2yF5qLPurZ+ClKuRqE3i5WE/e/gHDr+ZxL1/nzlg+FYMpVIhszfRWz39ag7n76MRB7sztF8RGX3xXwvSoQKejF4A4+sVAp8MT/pJLj00kxGHvTDbQEQkqa7GsY4hsUvbeaSaROZXz+K49c/zcceuJtPP3APcSeN/PLn8Id7+Z/4WPCOsem9BL9+/S12ptJ9FuOy9+mjMu3Mj6KX1Rn9qawpxjvPyhz7WDnSW081qoQyPI1CsLY/yt4oVHcenkPnpZvdbDjX3UmxlR1By2T6Be4iL/4yiRuviz52MDMvZiRyrs+9v6twRTVO4b9DlOAPRed3ABX0YgnNnx4Uc8ieAsAxcMzaDppXt7OioYmVsxtwgEd27mJF5y6+tfllfvfPX8Hauxcxxp39LJFgZvvj3N90PC/f/yBLDpvNb8zcjCUT76MYV8zUBEH6Wp3Rn8wq6r1Bnx5yD5QaPa/4UajF/uCjbJ2M3RC8O/DsiBdvgPrGQHxQVB9ClP8NuRudYhuk4H4GGHkwzLjA9b6dVPb7IXCtoQXV841EznX3kOtYUXcexQh/MXZc8Li5RicPgtWmgl4swYFH3qRdQfys+9hVT9K5fz2XXn0lsWSStG3z0EfPZMmHF/Lk7AYSBt5evhyCnaoipONxWonRsugMjtiboCVWw1mLl2Qag0Qgwy9GkH2bZXSNnYnryXnHMv/oGSX/aEpOf4W5rz+W8HuDz9deEW3nQM+MFXKLRPDacv3g/SUMw7ZOxm4IT1Tq2RE7nvCe51nJKkpUwp83uBl6uNEJWiiG/A1SroYrvD1qrp8ouyyftRO8th7HAnDchm7lxW6236OBLHBHU8iOi7qjGd+Sf+zEQJSQooLeO8KLTAdoWvc0t11yIbI3AZYgjoM4DrbjcPrvb+Xke+5g6cJFPD3jSCZteZO0XYMlaSzbZutnP8eFTSdw7Kon+dDevdiOQyxhuOSma7j3hJMY+3Ynf2to4pHZDTzWtTtvph6eluD49U+7dwOJJKb2l9jzZhQsq8zluVfENMEDRS47JyP04R93HpHwH3P94EVcay9o64gN9r7dYio2HHiaO6DKCYq8J4YHnZnHn44QlfDnHW5Qw+897Pz8gpSrUQ5vj7LXertoi0+PY/kI4MCOJ4luIAvc0RSy4yB6BGnORiDHNBUlQAW9VLS2YicS7h8LC2w7U+6IMcSTST53z+3uV0uEVE0Nb537ecaf/yWWTjqUx198k72O4eJYDCuZxHYcPrTycVraH8MRi0Q8xlmLl7BqdgOLX9rM6ePq2JlKM7rGzjyu2f0uS9/ckeXdH7vqSaxEEstJF1Unn8tzD/cR+FMhDJngDza5BCr8Y4fiRCJf1mf8jkPJFu7X7+kppkGLwfeZc4lh1jkLiEpY4KMslGIssFzefXB7qQY/RVl1o+e5n22mRh56NJDBDttip20O23FRdzSRSz36dw85pqnoJxUn6B1bO2jf0k7ThCYaxzcOdTjdhD32q66CVavcKXkTiYxXbgDbGCzHYfyMw6C5mflde1xbZHU7/3HRt/jOqhXUtf4Z8WZ+tI1DLJHkkpuuYfF5F/KX2Q207tyVOV74EeCYtat5/+p2uupGY2oDcbW05M3Acy2SHfTi0wZ+/cZ2lr65vcccN2Xb+VoKogQql11RqHSyUNYXrADZ2uqKeZSY+jHlmwo5fM6+iMpAdfb1x14r9lj1c7o/26gG0u+wzdd5ms+Og+jzRjUC4ca3xJ2mYkz49mRwaGpqMu3t7b16T8fWDs5fdj6JdIK4Hee6U64rL1Fva+vpsbe1dc+1nkziz7VObW33NAFtbaQXLED2JjC1ceyrroKLL87M/IgIxhgcsUjV2Px24SLWzDiSMW93smP/+h6Pc59fz9kP3ENNOg3+8bZvZ33T8dx48IzIUspgbfw8rzFon3cs//rZMwG3BHPDn/7McR1P8jevoxfcRqTWEj41cQy3vLGdNO4cy98+9EC+dsiEwf4LFGRQ7iKKrbAotnLCtzt8Me3PbXpURl/s8cqlfr4vBGPf2gqrvwekXYGfe5lbWjgUsfThcxSRlbmmJ6+oDL19SzuJdAIHh6STpH1Le3kJepTHHqhl92dyDK6GBGTbNYmE+3pg5kfuuAP505+wHQcr6XCub93gVsCEH41IVvUM27fT/v++5s4S+cb2HqWUp4+r44/bujJifvslXyGeTEJtLRtmTOG7z2/iYw/czWUP3ENNOkXKtlm6cBG/O+UMVs5uIOEY1ux6l2PXPc3xHU/yeGMTmyaNpb1rT9GiORhCO2glnMX6/4WyvuD2UmaxxWb0heKtJHLZPEMxz/sAfo4VJehNE5qI23GSTpKYFaNpQuWsoZGvQ7WHXeOLvb//nDnw6KOZaQgsPOsmx2PWKFbveCs6dzN3TQfHr27PZPMrGpr4y+wGdv31r7x/dTuT96/njEf+RDyRxDYOJpHgtSXXc8u9dxL3qnIEsByHz917B2c/8IeMsFvAb72GIBGL8enFS/jEUQ2cfeBY5ozaJ8vv9wde+QIOFPTn+0qwoQh+Bo83NLFi6iAvBN4fSi0ClSzO/aWUDWSZUVGWC5Sxh95fouya8Oth68ab9bHHY8Qo1vUPPczUj55GLJHADtg3fzruAyx44m/UpJLudgQL9xgJ2+aZw46g8bm12CZ7ymDwXFgRkjU1WfulxOLRY45j8XkXZlkzXiEdNd7AK9/2Cdo1/r61OTLocCYfLM8MD8AKZ+S/6HqDlk99nFgySTIW45e/WkrdBz8Q+b6q7dxVekU5fheqxnIBaBzfWF1C7pMvgw++HrZucj2GGoaZ7Y9jUinEE2bbuPbNqX/7M0K34NqemL85p4Ex69bS8NxaLGNIe42EfdppcP/93R29XgXPvGefyVTw2MbhhJWPM3/1Sn67cBG3edYMuLZQMjDw6rGGJtaMbObYdU9zXMeTbPfuHqIy6LBAXzZ9Mt/b+HrWnDjxiCkW/EFVdX/9KyNSKbezOZni7eXL+f6EqVnvg8Hv3C1H0VAqYJR1BBUn6MOeQsKfi5YWxFtiTwLZvW/f+LNm+B229tFHY69ZTY2Xce9p+TB1378s04mbuVvwpg+2AGNZWIceinnxRc/vT3LuvXfwGc+aeXrGkYx9u5OuwMCrZCzG9776TS67+sqMreOIkIjH+eUBS/l1TXcGHRRo4xh+/upW5qzp4H0BG8lvCMCdOiHo6z919LEcH48jiQRpy2LSls189t47Mu/73cQxvPpuInPM4LH6I7j5BHuwRKPYRiPXHVC5Njb57tD6SyWOsq44y0XpB76tM3Zsd0llKuXWzH/xizBvXnd2D27lTSKBicexoxbuCAp7KtVdrnnxxfDee5hAqaaPsSywrMzAq5RYvDJpMlPf2JSxdQRIWTb/9cV/4KfnfClj0ywYuz87H/lrptJGgNsu+UrARnIbgv++/jdcO2UGjc90cGvI16+xhCsee5ipS3+L7dtM3vsu/eq3mPX8Oj7tdf4mYzFaf/d7/qFuUpbgApEikmtmzctvuYNjV7mNyuEnnZjpU/AbqR+9+OaAVQeFB5qFG41gzEDkHVCuxmawB6BFNTbhWUv7OkVGrvP5f7snvYqvchD0qrJclH4Qzu59+yaHb2/nmeog63jh48yZAzffjARq8H382nps27V4HIepr7+WVZ3jWBZpOzuDzi7HdCttnjnsCOKemPv1/bFkCusvf2FOwx6+ftM1mddjiSRfv+kafnzehWyfOJnDnXTAfnJto+//5AosJ93d+ZtK0fnQcuYcMYf3efbQ90buw9rd75Iy3eWdl887lo8vOi1S/F5c3sot37ggczdy1uIl3Dy7ISM+l02fHDk1QykyT1/w9jqGo9euZr53DYvrRnHJNPfOIyjgn5o4Jisj/eO2rpwZalDsvh9oqPzBbVGNR3+IupMJzlp6jHd9/e3szmo0AqO//VHW7bPmlvUdiwr6cKZY3763xwkKfa4a/KuuQu64A+OVYxrLQk46Cc48k7cee4K63/yav7/3Dq+jlow15Pv9luNkfPtwQ9Cwfi3fuOlaalLJjIXj+/rvW7OKLT+6Mtt+sizEsrDSTqZxMCIkLYu6VU9x25L/zWTsn1y8hDnAJ5fdk2lckrEYX6u5gTmJZEb4F9eN4vRxdRzw10eIJZPUOA54g8PuPeEkxrzdyWMNTWx4eUNmagYnfj13ThzL+uOPz9k3ANl3B7nuEkbX2PxxWxdz1nRwZijWTy1ewlldjXxq4pisqh8mnphpXB5vbGKfEz6YsazCjc0f7rovsqESyDQe/bWswtcSHvDmz5+0yJu11L++l4+4j3bv7qeYu6jg+YLVVkvuvouFiQTilRNvemAZZyX2LWtPXQVdGTgK1eDPmYM8+qibxQdmsBy/fTsmmEHTXSHjjW8E/3lEQ7BwRWsmyw77+iNSKaa+szu7zn/7dqyxY0lffDFOIoGxbV5qWcBBf17ORwLHIpHkP66+ktkvPE/Mayz87Rf++HJmv/B8lvAvBT618QXSdg1CCttxOKH9MW86B3f6h2cOOwLZm8AyDk4CXrj/Aa4eM5l5z6zO6hvw7w5qH3+M4zqe5JX96xn7dicPNjSxanZDxpJavn0XKa8RbFq7mtsu+UpWySmJJN+46Rr+57wLSb4wglsv+UpGlP/4r5dy+RX/gSSSpGyL5X4FVDoNtb9kw4wpfOvgGWz405/5p1/9InP3E2yo5gTuopKxGP9d/xu+3/UOx3d0Z/J+SWo+y8a3ifxr8RuK969u5wlvnMPIJx53M+j33suMu7BSKV6+/0G+EhubeW/Qslu+fVfOu4fwaOirD51FS00NIwCJx1nR0JRpAIONdqEqqcHsl1APXRlaco2uXbCgx0hZY1mYYKWN79v7/v4VV2C+9z0knXazeREYMaLb1/fr/HMt5B2IZdMDy5j4/f+kxnGy+wAgqyooPPWCACmxWH3EzIzAp22bHbOOYuKa1Yh3vPD7HcsiZbujgJ+ZcSSXXX1lVt9AqqaG5cd9gBODJabe9mAlkW897PDGFJyw8vGsklMgqzEJlpq+Omkyh7zxujsGIXytts0tp3+ctDFuNhyIwcpxFxX1OSxduIjff+QMRp/wwYy4+mMPgpZN0nTbRDv2r2fO8+v5TMBuu3XhIiyBz/7x90jaLXg1IrwXj/PJxUswkHlv1IjqxxqaGPmBD2TNibRm97uZ0dB+tVXn/vV83HYYffIClm9/my99/uxMA+ifx29kfNvp93+4L2OhBa04u4hGpRjyeegq6Ep5EuzAjSrHzNcQJBLdHb3+soCF6vwjzp/pFLZt7MZGTHt7tyB7Sw/unjOXfZ56KiOCPlnCZtnsOO/zjF/6f5lGynhVRr7w7WxoYP+1a7HTKYxIZr6fKPEPz93jjwXoOabAFVksC6mpgdA1ZBqTsCgHRhobcKd3jsVIGzI2lh93ato0al9+KWdD1aMB9GINTl/hrxngzz+03RPwswMNR67j2CLuHYRt8/Qnz+bXEw5mdui9PUZUew1asPJqe7CPJmDVJeJxWn/3e67btI2v3fgLPug1kLkaq2dmHMl/Biq4vvbTG9jsWXHB6/QtqZM+urDXneD5BN3NfIbg3zHHHGMUpeSsWGHM5Ze7j6U81ooVxuyzjzG2bUw8bsyFF2a2p/bZx6Rt26TicWOOO844lmWMOwGuSYuY1D77dB/j8suNWbLEmAsvNOnaWvd9++xjzIUXGse23fdZljGxmDH+cUSM4w4Zc48ZOHbUdhN8tCxjTjkl6xqCx0uKZbqmTTMpsTLPdx3T1B2bd61bvvAlk4y6riVL3M/Fe81YljHxuNl1TFPmmFGx+c9TIua9WMzc+/4Pm3djcZP0You8lsA/AyZp2WbLF76U+RutW7bcvFNba1KhzyXXYyaGfJ+fbZvV55xr3qmt7Y4tR4xpEZOwbZMSyXyW7UfOdq/LsrKv07LMO7W1Zt2y5b3+WgLtJoeuqqArSrHkaiyKEf5Cxwu+zxfKKPH3j+ltN7W1xkQIWEZc/cYkeM5wY7JkSXejFG58/PeGG67gdQUbqsD1ZO3/sY+558whtrkEPOtagscJxupz+eXdjWLwvSEhDjeQUY1G8PMLNmZJscy7hx6W3WgHGxmxTLqmxqQsK29D5jcW5vLLe/01zCfoarkoSqnprb1TzPtyvRYeCxA1pqBAf0HRllQfbKse57j5Zhyv6kkcB/H6R3rYSsHpK8LXku+zCFtu/nuD1t2qVVkx+BZTpo+m0JiMYJ+MbcNpp+F4fTr+6123/o5Rf/5zDysuY0P5/Tu5+nPyUFUeel9/K4pS1VTSD6M3A9x6ey3Ffg6F+mgKHbeI51l9MMGO/HD/Ti+pGkEPNsD5ihUURakgKqkx6g19uQsqgqoZKdra6op5urjV1BRFqQT6Oj9RuZNrwN0AYhWzk4gsFJHnRGSjiPxLxOsiIj/1Xn9aRI4ufajd04bbdve04YqiKIpLwQxdRGzgauBkYBPwpIjcbYxZF9jtVGCG9+944BfeY0lpboaHrupi4x2dTD+znubmulKfQlEUpWIpxnI5DthojHkRQESWAouAoKAvAm72SmoeE5F6ETnQGPNmKYPtausiffFqDkk4pB+16JrTQJ2KuqIoClCc5TIZeC3wfJO3rbf7ICIXiEi7iLRv27att7HS2dqJk3AgDU7CobO1s9fHUBRFqVaKEXSJ2BYujSlmH4wx1xpjmowxTePGjSsmvizqW+qx4hbYYMUt6lvqe30MRVGUaqUYy2UTcFDg+RTgjT7s02/qmutoWN5AZ2sn9S31arcoiqIEKEbQnwRmiMg04HXgbOCc0D53Axd5/vrxQFep/XOfuuY6FXJFUZQICgq6MSYlIhcBD+JOTX2DMWatiFzovX4NcB9wGrAReAf4wsCFrCiKokRR1MAiY8x9uKId3HZN4P8G+GppQ1MURVF6Q1EDixRFUZTyRwVdURSlSlBBVxRFqRJU0BVFUaqEIZs+V0S2Aa/08m0HAG8NQDilpBJihMqIU2MsDRpjaSiXGA8xxkSOzBwyQe8LItKeax7gcqESYoTKiFNjLA0aY2mohBjVclEURakSVNAVRVGqhEoT9GuHOoAiqIQYoTLi1BhLg8ZYGso+xory0BVFUZTcVFqGriiKouRABV1RFKVKqBhBL7RQ9VAgIgeJyJ9FZL2IrBWRf/K2jxGRh0Tkee9xdBnEaovIKhG5txxj9JYtvF1EnvU+z+YyjPHr3t/5GRH5rYiMKIcYReQGEdkqIs8EtuWMS0S+4/2OnhORjwxhjFd6f++nReT3IlJfbjEGXrtERIyIHDCUMRaiIgQ9sFD1qcAs4DMiMmtoowIgBfyzMWYm8D7gq15c/wIsN8bMAJZ7z4eafwLWB56XW4w/AR4wxhwJNODGWjYxishk4GtAkzHmKNyppM8ukxh/BSwMbYuMy/t+ng3M9t7zc+/3NRQxPgQcZYyZC2wAvlOGMSIiBwEnA68Gtg1VjHmpCEEnsFC1MSYB+AtVDynGmDeNMU95/9+FK0KTcWO7ydvtJuBjQxKgh4hMAU4Hrg9sLpsYRWR/4ATglwDGmIQxppMyitGjBthHRGqAfXFX5RryGI0xjwA7QptzxbUIWGqM2WuMeQl3DYPjhiJGY8wyY0zKe/oY7kpnZRWjx4+Bb5G9rOaQxFiIShH0ohahHkpEZCowD3gcmOCv2OQ9jh/C0ACuwv1COoFt5RTjocA24EbPFrpeREaWU4zGmNeBxbhZ2pu4q3ItK6cYQ+SKq1x/S18E7vf+XzYxisjfAa8bY1aHXiqbGINUiqAXtQj1UCEio4A7gIuNMW8PdTxBROQMYKsxZuVQx5KHGuBo4BfGmHnAHobeAsrC86AXAdOAScBIEfn7oY2qT5Tdb0lEvotrX97ib4rYbdBjFJF9ge8C/xb1csS2IdekShH0QVmEui+ISAxXzG8xxtzpbd4iIgd6rx8IbB2q+ID3A38nIi/jWlUnishvKK8YNwGbjDGPe89vxxX4corxJOAlY8w2Y0wSuBOYX2YxBskVV1n9lkTkPOAM4LOme1BMucR4GG4Dvtr7/UwBnhKRiZRPjFlUiqBnFqoWkThuZ8TdQxwTIiK4vu96Y8z/BF66GzjP+/95wB8GOzYfY8x3jDFTjDFTcT+3h40xf095xbgZeE1EjvA2LQDWUUYx4lot7xORfb2/+wLcPpNyijFIrrjuBs4WkVpxF36fATwxBPEhIguBbwN/Z4x5J/BSWcRojFljjBlvjJnq/X42AUd739eyiLEHxpiK+Ie7CPUG4AXgu0MdjxfTB3Bvs54GOrx/pwFjcSsLnvcexwx1rF68LcC93v/LKkagEWj3Psu7gNFlGON/AM8CzwC/BmrLIUbgt7i+fhJXdL6ULy5cG+EF4Dng1CGMcSOuD+3/dq4ptxhDr78MHDCUMRb6p0P/FUVRqoRKsVwURVGUAqigK4qiVAkq6IqiKFWCCrqiKEqVoIKuKIpSJaigKxWDiEwUkaUi8oKIrBOR+0TkcG8WvH8M7PczEfm89/9ficjrIlLrPT/AGyQSdfxLvWNND2z7uretyXv+sojcEXj9LBH5lff/z4vIz7z/HyEirSLSIe7skdeKyEe85x0istubpa9DRG4u+YelDEtU0JWKwBvM83ug1RhzmDFmFvCvwATcUZD/5A06iyKNO1dIMazBHYDlcxbuIKcgTSIyu8Bxfgr82BjTaNzZOP/XGPOg97wRt+b+s97zc4uMTVHyooKuVAofBpLGmGv8DcaYDtyBKdtwB8+cF/1WrgK+7s2SWIi78GbyFJFDgS7v+EEW4zYm+TgQd3CKH+uaIs6tKP1CBV2pFI4C8k0w9kPgn3PMSf0q8Ffgc0Wc523caQiOAj4D3Bqxz++Ao4PWTAQ/Bh4Wkfs926a+iHMrSr9QQVeqAuPOSf0EcE6OXS4Hvklx3/mluLbLx3BtnjBp4Eq8BRlyxHMjMBO4DXfKhcd8H19RBgoVdKVSWAscU2Cfy3Ene+rxvTbGbMSdL+RT/jYR+YHfSRna/R7cbP5Vk3s65F/jLspxcK5gjDFvGGNuMMYswp0e9qgC8StKv1BBVyqFh4FaETnf3yAixwKH+M+NMc/idmCekeMYPwAuCez/3UAnJYHt7+I2DD/IFYxxp9D9MXBx1OviroEb8/4/EXeyrNdzX56i9B8VdKUiMO4sch8HTvbKFtcCl9JzDuof0L2UWfgYa4GnijzfUuMtL5iHX+IuzhHFKcAzIrIaeBD4pnGnXVWUAUNnW1QURakSNENXFEWpElTQFUVRqgQVdEVRlCpBBV1RFKVKUEFXFEWpElTQFUVRqgQVdEVRlCrh/wcN1eo/4YmkigAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "max_loss = 0\n",
    "max_MS_loss = 0\n",
    "max_train_acc = 0\n",
    "max_F1 = 0\n",
    "max_MCC = 0\n",
    "max_Score = 0\n",
    "\n",
    "for i in range(len(turns_record)):\n",
    "    if loss_record[i] > max_loss:\n",
    "        max_loss = loss_record[i]\n",
    "    if MS_loss_record[i] > max_MS_loss:\n",
    "        max_MS_loss = MS_loss_record[i]\n",
    "    if Score_record[i] > max_Score:\n",
    "        max_Score = Score_record[i]    \n",
    "\n",
    "for i in range(len(loss_record)):\n",
    "    loss_record[i] = loss_record[i]/max_loss\n",
    "    MS_loss_record[i] = MS_loss_record[i]/max_MS_loss\n",
    "    Score_record[i] = Score_record[i]/max_Score\n",
    "    \n",
    "plt.plot(turns_record, train_acc_record, '.', color = 'tab:green', label='ACC')\n",
    "plt.plot(turns_record, F1_record, '.', color = 'b', label='F1')\n",
    "plt.plot(turns_record, MCC_record, '.', color = 'm', label='MCC')\n",
    "plt.plot(turns_record, MS_loss_record, '.', color = 'tab:cyan', label='MS')\n",
    "plt.plot(turns_record, loss_record, '.', color = 'r', label='CE')\n",
    "plt.plot(turns_record, Score_record, '.', color = 'orange', label='MPWS')\n",
    "\n",
    "plt.xlabel(f\"{NAME}\")\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a20e150e",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}