Evaluation.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "%reload_ext autoreload\n",
    "%autoreload 2\n",
    "%matplotlib inline\n",
    "\n",
    "\n",
    "import pickle\n",
    "import numpy as np\n",
    "\n",
    "from SlideRunner.dataAccess.database import Database\n",
    "from lib.calculate_F1 import *\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Object detection (single shot) based on RetinaNet"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## First: Object-Detection-augmented Expert Labelling (ODAEL)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Optimize threshold value"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Optimizing threshold for validation set of 21 files: \n",
      "Best threshold: F1= 0.6285386302605416 Threshold= 0.7900000000000003\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYUAAAEWCAYAAACJ0YulAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4xLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvAOZPmwAAIABJREFUeJzs3Xd4FWX2wPHvSUISeoDQCT30AELoiAqigNgVRCysq1jXrqv+1EV0197XApZFXRURGyqCoCIgKASlBkJHQgmEDgHSzu+PmWQvIeUCuZnc5HyeJ0/u3Hln5ky5c2bemXlHVBVjjDEGIMTrAIwxxpQelhSMMcbksqRgjDEmlyUFY4wxuSwpGGOMyWVJwRhjTC5LCgEiIo1F5KCIhJ7k8AdFpHlpiukUpltXRGaLyAERed6P8qNEZG5JxFZEHKeLSFIJTavE5llEVERansRwTd1hwwroP0ZE/lvI8BEikigi9U502oWMc6SIfF9c4ytkOrNE5PoATyNCRFaJSJ1ATqcolhRc7o9ymYikich2EXlDRKJOYPiNInJ2Treq/qmqVVQ162TicYddfzLDBiqmUzAaSAWqqeo9xTliEZkgIk8U07iO2Vmq6hxVbV0c484znUJ3rmXYaGC2qm6H4ll3qvqhqp5TLNGVsLyJRlWPAu8Cf/cuKksKAIjIPcDTwH1AdaAn0ASYISLhXsZWRjQBEtWelCwW4gjG3+6NwAf+Fi6HSRPgI+BaEYnwLAJVLdd/QDXgIDAsz/dVgB3AdW73GGAy8AlwAPgd6OT2+wDIBg6747ofaAooEOaWmQU8Acxzy3wN1AI+BPYDC4GmPtNXoCXQwC2f85fmrDYFaAH8COzCORL/EIg6gZgaAFOA3cBa4Aaf6Y8BJgHvu/O7AogvZDn2dudhn/u/t/v9BCADSHfjODufYWu5cewHFgCPA3N9+rcBZrhxJuWsK5wjT99xf+0zX58BO4ENwO0+4woFHgLWufO1CIgBZrvL5pA7ruHAmUCyz7Bt3fW4110eF/j0mwC8Bnzrjvc3oEUBy+pPd1o567QXMAqYCzwH7HHjHuwzzCzgn8Av7jptiXMA8w6wDdiCs32FuuVbAj+76yMV+CTPtnUTsMad1muAuP1CgIeBTTjb//tAdbdfU47dfpq50zjgrp9/A/8tYJ4bu3GHFbHuNuIcKS8FjgJhwAM+6ysRuNhnvKM4dlvJd96ACHe9dfApW9uNqQ5QA/gGZ5vZ435ulGf5X1/AvHUHEnC23xTgBZ9+PXF+83uBJcCZ7vf/BLKAI+78/9tnmDXAGZ7tE72acGn5AwYBmTkba55+7wEfu5/HuBvxZUAF4F6cH24Fn435bJ9h8/6AZuHseFvg/JgTgdXA2e6G/z7wnzwbd8t8YvrQJ6aWwEB3g6+Ns2N7yadsUTH9DLwORAKd3R/EAJ/5PQIMwdmRPgn8WsAyrOn+kK5252WE213L7T8BeKKQdTARJwFVBjrg7ODmuv0qA5uBv7jj7oKzk2uf37hxdmqLgEeBcKA5sB441+1/H7AMaI2zs+jkE+cxyxyfpOCu87U4CSUc6I+zk2rtE8dunB1EmLueJhYwv8esB/e7UTjb1w3u8r4Z2Mr/dtazcJJJe3f8FYAvgXHuMqqDk1BvdMt/DPyfuzwigb55tq1vgCicnfVOYJDb7zp3PpvjHBh9DnxQwPYzH3gBZ/vr5y6PgpLCecCKPN8dt13gbLOLcRJ1Rfe7y3ESfQhOsj4E1PdZbnmTQkHz9i7wT5+ytwLT3M+1gEuBSkBV4FPgS5+ysyg4KcwHrnY/VwF6up8b4hywDXFjH+h21y5snDgHSLfnN62S+AvGU9DiFg2kqmpmPv22uf1zLFLVyaqagfNjiMQ5EvDXf1R1naruA74D1qnqTHfanwKnFTawiPwd56j5OgBVXauqM1T1qKrudGM6w59ARCQG6Av8XVWPqOpi4G2cHXuOuao6VZ1rEB/g7EDzcx6wRlU/UNVMVf0YWAWc70ccoTg/xkdV9ZCqLsdJxjmGAhtV9T/uuH/HOQu4rIBRdsP50Y1V1XR1rsu8BVzh9r8eeFhVk9SxRFV3FRUnznquAjzljvdHnJ3PCJ8yn6vqAnd9foiTaE/EJlV9y13e7wH1gbo+/Seo6gp3/DWBwcCd7nLbAbzoM58ZONV2Ddz1m/ci9lOquldV/wR+8ol1JM6R7npVPQg8CFyRtypHRBrjLOtH3O1vNs7Zb0GicJKGP15R1c2qehhAVT9V1a2qmq2qn+AcSXcvZPiC5u0jjl1fV7rfoaq7VPUzVU1T1QM4R/J+/ZZwlnVLEYlW1YOq+qv7/VXAVPc3lK2qM3DOKIYUMb4DOMvLE5YUnKPO6ALqL+u7/XNszvmgqtlAMs4RjL9SfD4fzqe7SkEDishg4A7gopwfi4jUEZGJIrJFRPYD/+XYJFaYBsBu9weQYxPO0U2O7T6f04DIApZTA3dYX3nHVZDaOEe+m32+8x1XE6CHiOzN+cPZcRV0B0sToEGe8g/xv51rDE5VxIlqAGx217tvnIUtrwLXZwFyh1fVNPej7zh8l1ETnLOFbT7zOQ7njAGc6kIBFojIChG5rqBp5Yk177rchLN+fJNTTrk9qnooT9mC7ME5AveH73wiIteIyGKf+exA4dt5QfP2I1BRRHqISBOcZPGFO41KIjJORDa5v6XZQJSfd+r9FWgFrBKRhSIy1P2+CXB5nm2xL85+pTBVcaqbPFEeL+TkNR+n7vISnCoMAESkMs6R2EM+ZWN8+ocAjXBO8cE5bQ0IEWmNc+R4iar6/mCedKfbUVV3ichFOPW6OQqLaStQU0Sq+iSGxjhVNydqK84PwFdjYJofw+7Eqb6LwTm7yBk2x2bgZ1UdWMDweedxM7BBVWMLKL8ZpwpvuR+x+doKxIhIiE9iaIxTBXiiTnZb8R1uM852G53fWa46d/jcACAifYGZIjJbVdcWMY2867IxzvpJwdnec2wDaohIZZ/E0JiC520p0FxEwnziLahs7vfuzvstYAAwX1WzRGQxTsI7IaqaLSKTcM4WUoBvfLb9e3CqFHuo6nYR6Qz84c90VHUNMMLdJ1wCTBaRWjjr6ANVvaGo+cyjLVDkrduBUu7PFNyqnMeAV0VkkIhUEJGmONU5yRx7t0RXEbnEPVq+E+dHmXOqmIJTD1usRKQa8BVOlUfeKoCqOBep9opIQ5z6cl8FxuQml3nAkyISKSIdcY54PjyJMKcCrUTkShEJE5HhQDuc6pVCuVUlnwNj3KO1dsC1PkW+ccd9tbtuKohINxFpW8A8LgD2i8jfRaSiiISKSAcR6eb2fxt4XERi3bt4Oro/4PzG5es3nLrs+90YzsSpHptY1DzmYyfOTQAnvb2o6jbge+B5EakmIiEi0kJEzgAQkctFJGcnvgdnB+TPrcgfA3eJSDMRqQL8C+ci9TGJR1U34VSFPCYi4W7iKbC6UFWTOb7ax5/fTGU39p3ufP0F50zhZH2Ec11ipPs5R1Wcs/W9IlIT+Ie/IxSRq0SktnuwkHOEn4Vz5n6+iJzrboeRInKmz3o5bv7d33FN/rdfKXHlPikAqOozOGcEz+HcQfAbTpYfoM69wzm+wtmgci6qXuJeXwDnqP1h9zTx3mIMrwvOEcwL4jx4dlBEDrr9HnP778O56+XzPMMWFdMInIuHW3FOo//h1nueELdOfijO0dYunKqLoaqaWuiA/3Mbzin+dpyLj//xGfcB4BycuvKtbpmncS5ugnP3TTt3Hr90k8z5OFUDG3Cq/97GubgPznWXSTg71P3u8BXdfmOA99xxDcszj+nABThnj6k4F+ivUdVVnCC3auifwC/utE7kupSva3AueifibJOT+V/VRDfgN3dbmQLcoaob/BjnuzgHQrNxlt8R4G8FlL0S6IFzgf0fODdLFGYcx16zOmbd5TeAqibiHDXPx9mJxuHcgXVSVDUnuTfAua6X4yWc7SAVZ4fsz1lujkHACndZvwxc4V7H2QxciLNv2YmzT7mP/+13XwYuE5E9IvKK+92VwHt59jslKufOBlMEERmDc2fKVV7HYkwwEufe+z9wDra2eR1PaeMunyVAP/fGAU/YNQVjTIlwj37beR1HaeUunzZex2HVR8YYY3JZ9ZExxphcdqZgjDEmV9BdU4iOjtamTZt6HYYxxgSVRYsWpapq7aLKBV1SaNq0KQkJCV6HYYwxQUVECnviPJdVHxljjMllScEYY0wuSwrGGGNyWVIwxhiTy5KCMcaYXJYUjDHG5LKkYIwxJlfQPadwSs488/jvhg2DW26BtDQYks9b8kaNcv5SU+GyfN4AefPNMHw4bN4MV199fP977oHzz4ekJLjxxuP7P/wwnH02LF4Md955fP9//Qt694Z58+Chh47v/9JL0LkzzJwJTzxxfP9x46B1a/j6a3g+n/d2fPABxMTAJ5/AG28c33/yZIiOhgkTnL+8pk6FSpXg9ddh0qTj+8+a5fx/7jn4Js/rFSpWhO/c1osffxx++OHY/rVqwWefOZ8ffBDmzz+2f6NG8N//Op/vvNNZhr5atYLx453Po0fD6jzvw+nc2Vl+AFddBcnJx/bv1QuefNL5fOmlsCvPWzsHDIBHHnE+Dx4Mhw8f23/oULjXbbG8kG1PDx0ic9AQ0jOzOJqVTXpmNtkKSedeQuK5lxC+ZxeDH7sdBSqFhxJVqQKhIrbtlcdtL2eeAqh8JQVjPKZA2tFM9h7OYOa8jUxIm8XunXsYt2n3cWW/XbaNySRRI20fcXvScr8PESGqUjgHN+2hTXomlUowflP2BV2DePHx8WpPNJtgsn3fEeas2cmcNan8sjaVXYfSAWhdtyot61ahQfVI6levSIMo53/96pFUDA8lNEQIESE0RAgVIVuVBRt28+2ybUxfsZ3Ug+lEVgjhrNZ1uPi0hgxoW5fQkBN+S6UpJ0RkkarGF1nOkoIxxW/z7jS+W76Nqcu2s3iz84bG6CoR9IuNpm9sNH1bRlOnWuRJjz8r20kQU5dt47vl20k9eJSGURW5qmcTrugWQ43K4cU1K6aMsKRgTAnbtOsQ3y7bxnfLtrNsyz4A4hpWZ1CHevRvU4c29aoiUvxH8plZ2cxcmcJ78zYxf/0uIsJCuKBTA67t3ZQODasXPQJTLlhSMKaELNiwm9dnrWVW0k4AOsdEMSSuHoM71CemZsnW+K9OOcB78zby+e9bOJyRRf82dXju8k7UtDOHcs+SgjEBpKrMStrJ67PWsnDjHmpWDucvvZtySddGNIyq6HV47DucwccL/uSFGauJrhzOayO7cFrjGl6HZTxUKpKCiAwCXgZCgbdV9al8ygwDxuDcmLFEVa8sbJyWFIyXsrKVqcu28fqsdazctp8G1SMZ3a85w7s1pmJ4qNfhHWdZ8j5u/nARKfuP8PB57bimV5OAVGGZ0s/zpCAiocBqYCCQDCwERqhqok+ZWGAS0F9V94hIHVXdUdh4LSkYL2RmZTNlyVb+/eNa1qceonntytx8Rgsu7NyQ8LDS/QzovrQM7p60mB9W7eD8Tg146pI4KkfY3ejljb9JIZBbRndgraqudwOaCFwIJPqUuQF4TVX3ABSVEIwpaRlZ2Xzx+xZem7WWTbvSaFOvKq+P7MK57esFze2f1StV4K1r4nnj53U8/30SK7ft582rutCyTlWvQzOlUCCTQkNgs093MtAjT5lWACLyC04V0xhVnZZ3RCIyGhgN0Lhx44AEa4yv9MxsJi9K5vVZa0nec5i4htUZf3VXzm5bl5AgSQa+QkKEW89qyWkxUdw+8Q8ufWM+H9/Qk3YNqnkdmillAnnem98vJ29dVRgQC5wJjADeFpGo4wZSHa+q8aoaX7t2ka8YNeakZWUrny1Kpv/zs3joi2VEV4ngP6O6MeW2PpzTvl5QJgRfvVtG88UtfagUHsrV7/zGmpQDXodkSplAJoVkIManuxGwNZ8yX6lqhqpuAJJwkoQxJUpVmb5iO4Nfns09ny4hqlIFJvylG1/c0puz2tQpUxdnY2pW4qMbehISIox8+zc2ph7yOiRTigQyKSwEYkWkmYiEA1cAU/KU+RI4C0BEonGqk9YHMCZjjjNvbSoXvz6PGz9YRGa28tqVXZhya1/ObF22koGvZtGV+fD6HmRmK1e+9Subd6cVPZApFwKWFFQ1E7gNmA6sBCap6goRGSsiF7jFpgO7RCQR+Am4T1V35T9GY4rX2h0HGPWfBVz59m+k7D/C05fG8f2d/TivY/2grybyR6u6VXn/uu4cPJrJyLd/Y/u+I16HZEoBe3jNlDt709J5aeYaPvh1E5XCQ7m9fyxX92pCZIXS95xBSfjjzz1c/c4C6lSL4JPRvahdNcLrkEwA+HtLaum+wdqYYpSRlc2EXzZwxrOzeH/+RkZ0j2HWvWdyQ7/m5TYhAJzWuAbvjurG1r2HufbdBRzJyPI6JOMhSwqmXJi9eieDXprNmK8TiWtYne/u6McTF8VRq4odFQN0b1aT10d2IXHbfp76bpXX4RgP2WONpkzbvu8Ij3+TyLfLttG0ViXeviaeAW3L7gXkU9G/TV2u69OMd3/ZQL9W0fRvU9frkIwHLCmYMikzK5sJ8zby4ozVZGYr9wxsxegzmhMRVn6rifxx/6DWzFuXyr2fLmXanadTp+rJv/PBBCerPjJlTsLG3Qx9dS5PfLuS7s1qMuOuM/jbgFhLCH6IrBDKqyNO49DRTO6ZtITs7OC6EcWcOksKpszYl5bBA58t5bI357PvcAZvXtWVd0d1o3Ete4vxiYitW5VHhrZjzppU3v1lg9fhmBJm1UemTPhu2TYenbKC3YfSGd2vOXcMiLWWQE/ByB6N+Xn1Tp6etoqezWvZG9zKETtTMEEtZf8RbvwggZs//J06VSP46tY+PDSkrSWEUyQiPH1pR2pUCueOiX9wON1uUy0vLCmYoJSdrXz025+c/fzPzErayYOD2/DVrX3siLYY1awczovDO7M+9RBjv0ksegBTJtjhlAk6m3encd/kJfy6fje9W9TiXxfH0TS6stdhlUl9WkZzfd9mvDVnA1d2b0xcI0u6ZZ2dKZigoap8vOBPBr00m+Vb9vPUJXF8eH0PSwgB9rcBsVSvWIEXZ672OhRTAuxMwQSFlP1H+PtnS5mVtJPeLWrx7OWdaBhV0euwyoVqkRUY3a85z05P4o8/93Ba4xpeh2QCyM4UTKmmqny1eAvnvDibX9fv4rEL2vPfv/awhFDCRvVuSs3K4bwww84WyjpLCqbU2nXwKLd+9Dt3TFxMi9qV+e6Oflzbu2m5aNa6tKkcEcZNZzRnzppUFm7c7XU4JoAsKZhSadrybZzz4mxmJu7g/kGt+fSm3jSzaweeurpnU6KrRPDC93a2UJZZUjClyp5D6dz+8R/c9N/fqR8Vydd/68stZ7Yk1M4OPFcxPJRbzmzB/PW7mLcu1etwTIBYUjClxszEFM55aTZTl23j7oGt+OKWPrSuV9XrsIyPK3s0pm61CF6csZpge0GX8Y8lBeO5/UcyuGfSEq5/P4FalcP56rY+3D4glgqhtnmWNpEVQrntrJYs3LiHuWvtbKEssl+d8dSCDbsZ/NIcvly8hb/1b8mU2/rSvoE9IFWaDesWQ4PqkTz/vZ0tlEWWFIwn0jOzeXb6Kq4YP5/QEGHSjb2455zWhIcF5yY5bdo0WrduTcuWLXnqqacKLDdp0iTatWtH+/btufLKKwFYvHgxvXr1on379nTs2JFPPvkkt/yPP/5Ily5d6NChA9deey2ZmZkFjvuPP/5g+PDhxMXF0a1bN8aMGcPhw4cLLP/ee+8RGxtLbGws7733Xr5lhg8fTufOnencuTNNmzalc+fORISFcnO/Jsx8cwwtWrejU6dOzJo1q4glZIKGqgbVX9euXdUEtzUpB3ToK3O0yd+/0fs/XaIHjmR4HdIpyczM1ObNm+u6dev06NGj2rFjR12xYsVx5VavXq2dO3fW3bt3q6pqSkqKqqomJSXp6tWrVVV1y5YtWq9ePd2zZ49mZWVpo0aNNCkpSVVVH3nkEX377bfzjeGrr77S3r1767x58zQ7O1uPHj2q48aN07POOkuPHDlyXPldu3Zps2bNdNeuXbp7925t1qxZblwFufvuu/Wxxx5TVdWXX3lV63Q9V4e+Mke3b9+uXbp00aysLD+XmPECkKB+7GOD87DMBCVV5YNfNzH01Tkk70njzau68vRlHalSzC2abty4kTZt2nD99dfToUMHRo4cycyZM+nTpw+xsbEsWLAAgAULFtC7d29OO+00evfuTVJSEgAvvPAC1113HQDLli2jQ4cOpKWlFTi9BQsW0LJlS5o3b054eDhXXHEFX3311XHl3nrrLW699VZq1HCeCK5Tpw4ArVq1IjY2FoAGDRpQp04ddu7cya5du4iIiKBVq1YADBw4kM8+++y48e7du5exY8cyffp0evXqhYgQHh7O6NGjGTlyJK+88spxw0yfPp2BAwdSs2ZNatSowcCBA5k2bVqB86iqTJo0iREjRgCQtGolwy4YzLIt+1h3MJSoqCgSEhIKHN4ED0sKpkTsOniUG95P4JEvl9OtaU2m3dmPQR3qBWx6a9eu5Y477mDp0qWsWrWKjz76iLlz5/Lcc8/xr3/9C4A2bdowe/Zs/vjjD8aOHctDDz0EwJ133snatWv54osv+Mtf/sK4ceOoVKkSCQkJXH/99cdNa8uWLcTExOR2N2rUiC1bthxXbvXq1axevZo+ffrQs2fPfHfCCxYsID09nRYtWhAdHU1GRkbuznby5Mls3rz5uGEmTZrEjTfeSJUqVXj88cfp0qUL9913H3/961+59tpr+e6770465hxz5syhbt26ucmrU6dObFkyh+oRIbz21TwWLVqUb2wm+AS07SMRGQS8DIQCb6vqU3n6jwKeBXK2xn+r6tuBjMmUvDlrdnL3pCXsS8vg0aHtGFUCTyU3a9aMuLg4ANq3b8+AAQMQEeLi4ti4cSMA+/bt49prr2XNmjWICBkZGQCEhIQwYcIEOnbsyI033kifPn0AiI+P5+23j988NZ+LrSLHz19mZiZr1qxh1qxZJCcnc/rpp7N8+XKioqIA2LZtG1dffTXvvfceISHO8drEiRO56667OHr0KOeccw5hYcf/ZJcsWcJNN93EkiVLWLx4MQkJCXz55Ze88sor+ZY/kZhzfPzxx7lnCQDXXXcdK1eu5NcP7uLLsCh6du9Z4LRMcAnYmYKIhAKvAYOBdsAIEWmXT9FPVLWz+2cJoQw5mpnFP79N5Op3FlC9YgW+vLUP1/VtViLNVEREROR+DgkJye0OCQnJvVj7yCOPcNZZZ7F8+XK+/vprjhw5kjvMmjVrqFKlClu3bi1yWo0aNTrmKDk5OZkGDRrkW+7CCy+kQoUKNGvWjNatW7NmzRoA9u/fz3nnnccTTzxBz549c4fp1asXc+bMYcGCBfTr1y/3SN2XqhIaGsqqVasYOHAgISEhDB48uFhiBieZff755wwfPjz3u7CwMF588UXmL1xEnUsfYf2WHfnGZoJPIKuPugNrVXW9qqYDE4ELAzg9U4qs3XGQS16fx1tzNnBVz8Z8fVtf2jWo5nVYx9i3bx8NGzYEYMKECcd8f8cddzB79mx27drF5MmTCx1Pt27dWLNmDRs2bCA9PZ2JEydywQUXHFfuoosu4qeffgIgNTWV1atX07x5c9LT07n44ou55ppruPzyy48ZZseOHQAcPXqUp59+mptuuum48cbFxTF//nxat27NDz/8QHZ2NtOnTwecO4xyznR8nXvuuXz//ffs2bOHPXv28P3333PuuefmO38zZ86kTZs2NGrUKPe7tLQ0Dh06RJNalYnNWE/qoUxiW7cpdDmZ4BDIpNAQ8K1kTHa/y+tSEVkqIpNFJCaf/ojIaBFJEJGEnTt3BiJWU0yys5X352/k/FfnsnXvYcZf3ZUnLoqjYnio16Ed5/777+fBBx+kT58+ZGX973WTd911F7fccgutWrXinXfe4YEHHmDHjh0FXlMICwvj3//+N+eeey5t27Zl2LBhtG/fHoBHH32UKVOmAM6OuFatWrRr146zzjqLZ599llq1ajFp0iRmz57NhAkTcm//XLx4MQDPPvssbdu2pWPHjpx//vn079//uOkPGzaM559/npYtW9K+fXvi4+P55ZdfUFXWrFnDI488ctwwNWvW5JFHHqFbt25069aNRx99lJo1awJw/fXXH3PReOLEicdUHYGTrLp06ULbtm1J/eVTqg++ixmJKSe6CkwpJPnVLRbLiEUuB85V1evd7quB7qr6N58ytYCDqnpURG4Chqnq8Vu9j/j4eLW7HEqnrXsPc//kpcxdm0q/VrV59rKO1K0W6XVY5cInn3zCuHHjeO2112jbti0ZGRlMmzaNJk2a0LFjx4BOOytb6ffMTzSuWYmPR/csegDjCRFZpKrxRZUL5JWhZMD3yL8RcEwFraru8ul8C3g6gPGYAFFVPvt9C49NWUGWKv+8uANXdm9c6IVLU7yGDx9OkyZNePDBB9m0aRNVqlThvPPO45xzzgn4tENDhJE9G/PMtCTW7jhAyzrWXlUwC2RSWAjEikgznLuLrgCu9C0gIvVVdZvbeQGwMoDxmADYceAID32+nJkrU+jetCbPXd6JxrUqeR1WudSzZ0++/PJLT6Y9LD6Gl2as4b+//smYC9p7EoMpHgFLCqqaKSK3AdNxbkl9V1VXiMhYnCfrpgC3i8gFQCawGxgVqHhM8Zu2fBsPfr6MQ+lZ/N+QtlzXt5k1cV1ORVeJYEhcPT5blMx957amcjE/kGhKTkDXnKpOBabm+e5Rn88PAg8GMgZT/A4cyeCxrxOZvCiZDg2r8eKwzsTWtSqD8u7qXk34cvFWvlq8lSt7NPY6HHOSLJ2bE7Jgw27unrSYrXsPc9tZLbl9QGzQNmJnileXxjVoW78a78/fyIjuMXZNKUjZr9n4JT0zm6enrWL4+PmEiNOq6b3nBm+rpqb4iQhX92zCqu0H+P3PPV6HY06S/aJNkVZt389Fr/3CG7PWMaxrDFPvOJ34pjW9DsuUQhd2bkDViDA+mL/J61DMSbKkYAqUmZXNv38WBhWxAAAgAElEQVRcw/mvziVl/xHGXR2YVk1N2VE5IoxLuzZi6rLt7Dp41OtwzEmwpGDylbT9ABe/Po/nvl/NOe3r8f1d/Ti3feBaNTVlx/BuMaRnZTNtxXavQzEnwQ75zDEys7IZN3s9L89cQ5XIMF4f2YUhcfW9DssEkTb1qtI8ujLfLdvOyB5NvA7HnCBLCibX6pQD3PvpEpYm7+O8uPqMvbA9tapEFD2gMT5EhMFx9Xjz5/XsPpROzcrhXodkToBVHxmyspU3f17H0FfmkrznMK9d2YXXRnaxhGBO2pC4+mRlK9OtCino2JlCObd+50Hu/XQJv/+5l0Ht6/HExR2ItmRgTlG7+tVoUqsSU5dtY0R3e5AtmFhSKKeys5UJ8zbyzPRVRISF8vIVnbmgUwN74MgUCxFhSFx9xs9ez55D6dSwKqSgYdVH5dDm3WmMeOtXxn6TSO8W0Xx/Vz8u7NzQEoIpVkM6OFVI9p6F4GJnCuWIqjIpYTNjv04kRIRnLuvI5V0bWTIwAdGhYTVialbk22XbGNYt3/dnmVLIkkI5sfPAUR78fCkzV+6gV/NaPDesEw2jKnodlinDcqqQ3pmzgb1p6URVsiqkYGDVR+XAtOXbOPel2cxek8ojQ9vx4fU9LCGYEjGkQ30yrQopqNiZQhm2/0gGY6as4PPftxDXsDovDOtkTVybEtWxUXUaRlVk6rJtXB5vVUjBwJJCGfXjqhQe+nw5Ow8e5fYBsfytf0sqhNqJoSlZThVSPSbM28i+wxlUr1jB65BMEWwvUcbsTUvn7k8Wc92EBKpVDOPzm3tz98BWlhCMZ4bE1ScjS5lpVUhBwc4UypBpy7fz8JfL2ZuWzu39W3Jr/5ZEhIV6HZYp5zrHRNGgeiRTl23j0q6NvA7HFMGSQhmw6+BRHp2ygm+XbqNd/Wq8d1032jeo7nVYxgA5bSHV54P5m9h/JINqkVaFVJpZnUIQU1WmLNnKwBdn8/2K7dx7Tiu+uq2PJQRT6gyJq096VjY/rLQqpNLOzhSCVMr+Izz85XJmJKbQqVF1nrmsJ63r2Z1FpnQ6LSaKetUimbpsOxefZlVIpZklhSCjqny6KJnHv0kkPTObh4a04bo+zQizC8mmFAsJcZrT/vC3PzlwJIOqVoVUagV0TyIig0QkSUTWisgDhZS7TERUROIDGU+wS96TxjXvLuD+yUtpW68a0+7sx+h+LSwhmKAwJK4+6ZnZ/JS00+tQTCECdqYgIqHAa8BAIBlYKCJTVDUxT7mqwO3Ab4GKJdhlZysf/raJp75bhQJjL2zPVT2aEBJibRaZ4NGlcQ1qVQ5nRmIKF3Rq4HU4pgCBrD7qDqxV1fUAIjIRuBBIzFPuceAZ4N4AxhK0NqQe4u+fLWXBht30bRnNk5fEEVOzktdhGXPCQkOEAW3r8N2y7aRnZhMeZme4pVEg10pDYLNPd7L7XS4ROQ2IUdVvChuRiIwWkQQRSdi5s3ycemZlK+Nnr2PQS7NZuW0/z1zakQ/+2t0SgglqA9vV48DRTBZs2O11KKYAgTxTyK9uQ3N7ioQALwKjihqRqo4HxgPEx8drEcWDXtL2A9z/2VKWbN7LwHZ1eeKiDtStFul1WMacsr4to4msEMKMxO30jY32OhyTj0CeKSQDvi1gNQK2+nRXBToAs0RkI9ATmFKeLzYfycji+e+TGPrqHDbvTuPVEacx/uqulhBMmVExPJS+LWszIzEF1TJ/fBeUAnmmsBCIFZFmwBbgCuDKnJ6qug/IPVQQkVnAvaqaEMCYSq1561L5vy+WsyH1EJd0acjD57Wjpr3C0JRB57Sry8yVKazYup8ODe1By9ImYElBVTNF5DZgOhAKvKuqK0RkLJCgqlMCNe1gsudQOv+aupJPFyXTpFYl/vvXHnZabcq0/m3rIAIzElMsKZRCAX14TVWnAlPzfPdoAWXPDGQspY2q8tXirYz9JpH9hzO45cwW3D4glsgK1oCdKduiq0TQtXENZiSmcNfAVl6HY/KwJ5o9sDH1EA9/uZy5a1PpHBPFU5fG0aZeNa/DMqbEDGxXlye/W0XynjQa1bA76koTu1G4BKVnZvPqD2s456XZLNm8l8cvbM9nN/e2hGDKnYHt6gLYOxZKoSLPFESkEnAP0FhVbxCRWKB1Uc8WmGMt2LCbh75YxtodBzmvY30eHdrO7ioy5Vbz2lVoUbsyM1amMKpPM6/DMT78qT76D7AI6OV2JwOfApYU/LDnUDpPfbeKTxI20zCqIv8Z1Y2z2tTxOixjPDewXT3enrPeXtNZyvhTfdRCVZ8BMgBU9TD5P5hmfKgqnyZsZsALPzP592Ru7NecGXf3s4RgjGtgu7pkZiuzknZ4HYrx4c+ZQrqIVMR9GllEWgBHAxpVkFuTcoD/+3I5CzbspmuTGvzz4g523cCYPE6LiSK6itNA3oWdGxY9gCkR/iSFfwDTgBgR+RDogx9NU5RHh9OzePXHNYyfvZ4qkWE8fWkcl3eNsdZMjclHSIgwoE1dpi7bZg3klSKFJgUREWAVcAlOMxQC3KGqqSUQW9BQVWau3MFjX68gec9hLuvaiAcHt6FWlQivQzOmVBvYri6fJGzm1/W76NeqttfhGIpICqqqIvKlqnYFvi2hmILKxtRDjPl6BbOSdhJbpwoTR/ekZ/NaXodlTFDoGxtNxQqhzEhMsaRQSvhTffSriHRT1YUBjyaIpKVn8vpP6xg/ez3hYSE8fF5bru3dlAr2FjRj/BZZIZTTY6OZuTKFsRe2x6mcMF7yJymcBdwoIpuAQzhVSKqqHQMaWSmlqkxbvp3Hv0lk674jXHJaQx4Y3IY69syBMSdlYLu6fJ+YwvIt+4lrZG0hec2fpDA44FEEiVXb9zP260TmrdtFm3pVeXnEaXRrWtPrsIwJav3d27RnJe2wpFAKFJkUVHWTiHQCTne/mqOqSwIbVumyNy2dF2as5r+/bqJqZAXGXtieK7s3Jsyqiow5ZbWqRNC+QTXmrE3lbwNivQ6n3POnmYs7gBuAz92v/isi41X11YBGVgpkZSsfLfiT579PYv/hDEb2aMLdA1tRw95zYEyx6hsbzbtzN3DoaCaVI6ydTi/5s/T/CvRQ1UMAIvI0MB8o00nh1/W7GDNlBau2H6Bn85r84/z2tK1vD6AZEwint6zNuJ/X89uGXfRvU9frcMo1f5KCAFk+3VmU4WYukvek8eTUVXy7bBsNoyry+sguDO5Qz+6KMCaA4pvWICIshLlrLCl4zd8G8X4TkS/c7ouAdwIXkjcOp2fx5s/rePPndYjAXWe34sYzmttLb4wpAZEVQunerCZz1+70OpRyz58LzS+470/ui3OG8BdV/SPQgZUUVeXbZdv417cr2brvCEM71ufBIW1pGFXR69CMKVf6tozmye9WkbL/iDUr7yF/LjT3BFao6u9ud1UR6aGqvwU8ugBbmryXx79JZOHGPbSrX42XrjiN7s3sFlNjvNA3Nhq+g7lrUrm0ayOvwym3/Kk+egPo4tN9KJ/vgsr2fUd4ZvoqPv99C9FVwvnXxXEM7xZDqDVcZ4xn2tarRq3K4cxda0nBS35daFZVzelQ1WwRCcp7xg6nZzF+9nre/HkdWdnKTWe04NazWlA10l7wYYzXQkKE3i2jmbs2FVW1mzs84s/Ofb2I3I5zdgBwC7A+cCEFxozEFB79ajnb9h1hSFw9HhjUlsa17IXhxpQmp7eM5uslW0lKOWDvIPGIP4/k3gT0BrbgvIqzBzDan5GLyCARSRKRtSLyQD79bxKRZSKyWETmiki7Ewn+RGRlZxNdJYJJN/bi9ZFdLSEYUwr1jY0GnOsKxhviUzNUvCMWCQVWAwNxkslCYISqJvqUqaaq+93PFwC3qOqgwsYbHx+vCQkJJxyPqqKKvfDGmFKu//OziKlRifeu6+51KGWKiCxS1fiiyhV5piAiz4hINRGpICI/iEiqiFzlRwzdgbWqul5V04GJwIW+BXISgqsy7is/A0FELCEYEwRObxnNgg27OZqZVXRhU+z8qT46x915D8U54m8F3OfHcA2BzT7dye53xxCRW0VkHfAMcLsf4zXGlGF9Y2tzOCOL3zft9TqUcsmfpJBza84Q4GNV3e3nuPM7LD/uTEBVX1PVFsDfgYfzHZHIaBFJEJGEnTvtiUdjyrIezWsSGiL2dLNH/EkKX4vIKiAe+EFEagNH/BguGYjx6W4EbC2k/EScJjSOo6rjVTVeVeNr17ZX9hlTllWLrEDnmCi72OyRIpOCqj4A9ALiVTUDSCPPtYECLARiRaSZiIQDVwBTfAuIiG/j6ecBa/wN3BhTdvVtGc3SLfvYm5budSjljl9viVHVPaqa5X4+pKrb/RgmE7gNmA6sBCap6goRGeveaQRwm4isEJHFwN3AtSc1F8aYMuX02GhUYd66XV6HUu4E9MlkVZ0KTM3z3aM+n+8I5PSNMcGpU0wUVSLCmLMmlSFx9b0Op1yx90kaY0qdCqEh9Gxeyy42e+CkkoKItCnuQIwxxlfflrXYvPswf+5K8zqUcuVkzxS+L9YojDEmj76xzp2Gs9fY2UJJKvCagoi8UlAvICow4RhjjKNF7co0qB7J3DWpXNWzidfhlBuFXWj+C3APcDSffiMCE44xxjhEhNNjazN1+TYys7IJC7VLoCWhsKSwEFiuqvPy9hCRMQGLyBhjXKe3iuaThM0sSd5H1yY1vA6nXCgs9V4GLM6vh6o2C0w4xhjzP31aRCMCc+y6QokpLClUUVW77G+M8UyNyuF0bBTFHGvyosQUlhS+zPkgIp+VQCzGGHOcfrHRLN68l32HM7wOpVwoLCn4tnLaPNCBGGNMfk6PrU1WtjLfmrwoEYUlBS3gszHGlJjTGkdROTzUriuUkMLuPuokIvtxzhgqup9xu1VV7a3axpiAqxAaQq8W0XZdoYQUeKagqqGqWk1Vq6pqmPs5p9sSgjGmxPRrFc2fu9PYtOuQ16GUefY0iDGm1Ds9t8kLO1sINEsKxphSr2mtSjSqUZE5q+26QqBZUjDGlHo5TV7MX7eLjKxsr8Mp0ywpGGOCQr/YaA4czWTJ5r1eh1KmWVIwxgSF3i2iCRG7rhBolhSMMUGheqUKdIqJsucVAsySgjEmaJweW5slm/eyL82avAgUSwrGmKDRLzaabIVf1lkVUqBYUjDGBI1OMVFUjQizKqQAsqRgjAkaFUJD6N2yFrNXp6JqTbIFQkCTgogMEpEkEVkrIg/k0/9uEUkUkaUi8oOI2ItYjTGFOj22Nlv2HmZDqjV5EQgBSwoiEgq8BgwG2gEjRKRdnmJ/APGq2hGYDDwTqHiMMWVDv5wmL+zp5oAI5JlCd2Ctqq5X1XRgInChbwFV/cnn7W6/Ao0CGI8xpgxoXKsSzaIr81OSJYVACGRSaAhs9ulOdr8ryF+B7/LrISKjRSRBRBJ27rQNwZjyrn+bOsxft4tDRzO9DqXMCWRSkHy+y/fKkIhcBcQDz+bXX1XHq2q8qsbXrl27GEM0xgSjAW3rkJ6Vbe9YCIBAJoVkIManuxGwNW8hETkb+D/gAlU9GsB4jDFlRLemNakaGcYPK1O8DqXMCWRSWAjEikgzEQkHrgCm+BYQkdOAcTgJYUcAYzHGlCEVQkM4s3UdfkraQXa23ZpanAKWFFQ1E7gNmA6sBCap6goRGSsiF7jFngWqAJ+KyGIRmVLA6Iwx5hhnt61D6sF0Fidbq6nFqbB3NJ8yVZ0KTM3z3aM+n88O5PSNMWXXma3qEBoi/LAyhS6Na3gdTplhTzQbY4JS9UoViG9Sgx9WWs1zcbKkYIwJWme3rcuq7QdI3pNWdGHjF0sKxpigNaBtHQA7WyhGlhSMMUGree0qNI+uzEy7NbXYWFIwxgS1AW3r8Nv63Ry0p5uLhSUFY0xQG9C2rvN0szWQVywsKRhjglp8kxpUr1iBmXZdoVhYUjDGBLWw0BDObF2bn5J2kGVPN58ySwrGmKA3oG1ddh9KZ/HmPV6HEvQsKRhjgt4ZrWoTFiJWhVQMLCkYY4Je9YoV6Na0prWaWgwsKRhjyoQBbeuwOuUgm3fb082nwpKCMaZMOLttXQB7kO0UWVIwxpQJTaMrE1unCtOWb/c6lKBmScEYU2YMiavPgo27Sdl/xOtQgpYlBWNMmXF+p/qowtRl27wOJWhZUjDGlBkt61SlTb2qfLPUksLJsqRgjClThnasz6JNe9i697DXoQQlSwrGmDJlaMcGAHxrZwsnxZKCMaZMaRpdmQ4Nq/HN0q1ehxKULCkYY8qcoR0bsCR5H3/usgfZTpQlBWNMmXNeXH0AvllmZwsnKqBJQUQGiUiSiKwVkQfy6d9PRH4XkUwRuSyQsRhjyo+YmpXoHBPFN0vsusKJClhSEJFQ4DVgMNAOGCEi7fIU+xMYBXwUqDiMMeXT0I71Sdy2n/U7D3odSlAJ5JlCd2Ctqq5X1XRgInChbwFV3aiqS4HsAMZhjCmHzuvoViHZXUgnJJBJoSGw2ac72f3OGGMCrn71inRrWsPuQjpBgUwKks93J/WuPBEZLSIJIpKwc6e9nNsY45+hHRuwOuUgq1MOeB1K0AhkUkgGYny6GwEnlbJVdbyqxqtqfO3atYslOGNM2Tc4rh4hAt8ssbMFfwUyKSwEYkWkmYiEA1cAUwI4PWOMOUadqpH0aFaLb5ZuQ/WkKirKnYAlBVXNBG4DpgMrgUmqukJExorIBQAi0k1EkoHLgXEisiJQ8RhjyqehneqzPvUQidv2ex1KUAjocwqqOlVVW6lqC1X9p/vdo6o6xf28UFUbqWplVa2lqu0DGY8xpvwZ3KE+oSHC1/bMgl/siWZjTJlWs3I4Z7WuzccL/mT/kQyvwyn1LCkYY8q8Owa0Yt/hDN6es8HrUEo9SwrGmDIvrlF1BrWvxztz1rP7ULrX4ZRqlhSMMeXC3ee0Ii0jizd/Xud1KKWaJQVjTLnQqm5VLurckPfmbSRl/xGvwym1LCkYY8qNO8+OJStb+fePa70OpdSypGCMKTea1KrM5fExTFz4J5t32wt48mNJwRhTrtw+oCUiwss/rPE6lFLJkoIxplypX70iV/dswue/J7N2h71rIS9LCsaYcufmM1sQWSGUF2eu9jqUUseSgjGm3ImuEsF1fZrx7dJtrNi6z+twShVLCsaYcumGfs2pFhnGk1NXkZ1tLajmsKRgjCmXqleswH2D2jB3bSpv2ANtuSwpGGPKrat6NOb8Tg14/vsk5q1L9TqcUsGSgjGm3BIRnrokjua1q3D7x3/Yk85YUjDGlHOVI8J4Y2QXDh3N4raPficjK9vrkDxlScEYU+7F1q3KU5fGsXDjHp6dnuR1OJ6ypGCMMcCFnRtydc8mjJ+9nmnLt3sdjmcsKRhjjOvhoW3p1Kg69326hI2ph7wOxxOWFIwxxhURFsprI7sQEiJc/e5vTF+xHdXy9QyDJQVjjPHRqEYl3rk2nvDQEG78YBGXvjGPBRt2ex1WibGkYIwxecQ3rcn0O/vx1CVxbNl7mGHj5vPXCQtJ2n7A69ACToLt1Cg+Pl4TEhK8DsMYU04cTs/iP/M28MasdRw8msn5HRtwXsf69IutTcXwUK/D85uILFLV+CLLBTIpiMgg4GUgFHhbVZ/K0z8CeB/oCuwChqvqxsLGaUnBGOOFPYfSeX3WWiYu3MyBI5lEVgihb8vanNOuLgPa1qFWlQivQyyU50lBREKB1cBAIBlYCIxQ1USfMrcAHVX1JhG5ArhYVYcXNl5LCsYYL2VkZbNgw25mJKbw/YrtbN13BBGIa1idhlEVqVstkjrVIqhbNZK61SKpXTWCqpFhVIkMo3J4GKEh4kncpSEp9ALGqOq5bveDAKr6pE+Z6W6Z+SISBmwHamshQVlSMMaUFqrKiq37mZGYQsKm3aTsP0rK/iMcOJJZ4DCVw0OpGlmBKpFhvDCsEx0bRZVIrP4mhbAAxtAQ2OzTnQz0KKiMqmaKyD6gFnBMy1QiMhoYDdC4ceNAxWuMMSdEROjQsDodGlY/5vu09Ex2uAli58GjHDySycGjmRzI/Z/BwaOZVIkI5C745AQyovzOkfKeAfhTBlUdD4wH50zh1EMzxpjAqRQeRtPoMJpGV/Y6lBMWyFtSk4EYn+5GwNaCyrjVR9WB8nNDsDHGlDKBTAoLgVgRaSYi4cAVwJQ8ZaYA17qfLwN+LOx6gjHGmMAKWPWRe43gNmA6zi2p76rqChEZCySo6hTgHeADEVmLc4ZwRaDiMcYYU7SAXuVQ1anA1DzfPerz+QhweSBjMMYY4z9r5sIYY0wuSwrGGGNyWVIwxhiTy5KCMcaYXEHXSqqI7AQ2neTg0eR5WrocsHkuH2yey4dTmecmqlq7qEJBlxROhYgk+NP2R1li81w+2DyXDyUxz1Z9ZIwxJpclBWOMMbnKW1IY73UAHrB5Lh9snsuHgM9zubqmYIwxpnDl7UzBGGNMISwpGGOMyVUmk4KIDBKRJBFZKyIP5NN/lIjsFJHF7t/1XsRZnIqaZ7fMMBFJFJEVIvJRScdY3PxYzy/6rOPVIrLXiziLkx/z3FhEfhKRP0RkqYgM8SLO4uLH/DYRkR/ceZ0lIo28iLM4ici7IrJDRJYX0F9E5BV3mSwVkS7FGoCqlqk/nGa61wHNgXBgCdAuT5lRwL+9jrWE5zkW+AOo4XbX8TruQM9znvJ/w2m+3fPYA7yexwM3u5/bARu9jjvA8/spcK37uT/wgddxF8N89wO6AMsL6D8E+A7nzZU9gd+Kc/pl8UyhO7BWVderajowEbjQ45gCzZ95vgF4TVX3AKjqjhKOsbid6HoeAXxcIpEFjj/zrEA193N1jn/bYTDxZ37bAT+4n3/Kp3/QUdXZFP4GyguB99XxKxAlIvWLa/plMSk0BDb7dCe73+V1qXvqNVlEYvLpH0z8medWQCsR+UVEfhWRQSUWXWD4u54RkSZAM+DHEogrkPyZ5zHAVSKSjPMuk7+VTGgB4c/8LgEudT9fDFQVkVolEJuX/N72T0ZZTAqSz3d577v9Gmiqqh2BmcB7AY8qsPyZ5zCcKqQzcY6a3xaRqADHFUj+zHOOK4DJqpoVwHhKgj/zPAKYoKqNcKoZPhCRYP2d+zO/9wJniMgfwBnAFiAz0IF57ES2/RMWrBtLYZIB3yP/RuQ5hVbVXap61O18C+haQrEFSpHz7Jb5SlUzVHUDkISTJIKVP/Oc4wqCv+oI/JvnvwKTAFR1PhCJ04haMPLnt7xVVS9R1dOA/3O/21dyIXriRLb9E1YWk8JCIFZEmolIOM4OYYpvgTz1bxcAK0swvkAocp6BL4GzAEQkGqc6aX2JRlm8/JlnRKQ1UAOYX8LxBYI/8/wnMABARNriJIWdJRpl8fHntxztcyb0IPBuCcfohSnANe5dSD2Bfaq6rbhGHtB3NHtBVTNF5DZgOs7dC++q6goRGQskqOoU4HYRuQDnNHM3zt1IQcvPeZ4OnCMiiUAWcJ+q7vIu6lPj5zyDU50yUd3bNoKZn/N8D/CWiNyFU6UwKljn3c/5PRN4UkQUmA3c6lnAxUREPsaZr2j32tA/gAoAqvomzrWiIcBaIA34S7FOP0i3F2OMMQFQFquPjDHGnCRLCsYYY3JZUjDGGJPLkoIxxphclhSMMcbksqRQDolIUxG50s+yH7vNgdwlIm3cFkf/EJEWpxjDRvd5iVMiIhEiMtONa/ipjq+IaU0VkSj37xY/yh+znEUkXkReOYHpne62aLtYRBqKyOSTjb04iMgEEbmsiDKjRKTBSY7/lJaXKR6WFMqnpkCRSUFE6gG9VbWjqr4IXITzVPRpqrouwDH66zSggqp2VtVP/BlAREJPZkKqOkRV9wJRQJFJgTzLWVUTVPX2E5jkSOA5d962qGqhO2R/iUggn08aBZxUUuDUl5cpDl43E2t/xfMHXAMsxWkg7AP3uwnAZT5lDrr/fwX2AYuBu3Ceev0PsAynee2z3HJLgcNuuX8A23Halvkpz7RvBp7x6R4FvOp+/hJYBKwARvuU2YjT/EJTfJoIxmnLZoz7uQUwzR1+DtAmz3Tr4DzAkzMvLXCe5v3DnZd3gQif6T0KzAWuyDOeCcAbOK1srsdpQ+ddnCfdJ+QT80Sf5fIsTls0zwLL3ekOL2A5nwl84/ar4rPMlwKX5onpepwHKzcAH/ouJ6ASTlMWS4FPgN+AeN917H6+LCd+dx5fcOfxeaCyO48L3eV1YT7blAD/BhKBb3EemrrM7dcV+NldN9OB+u70DuI0obIYqJhfOXf4ljjtji0BfnfXXWHLqybOtrTULdfR/X6MOx+z3HV3u9e/xWD/8zwA+yuGlQjt3R9itNtd0/0/gfyTQu6Pze2+B/iP+7kNTlMJkRy/wx4D3JvP9GvjNHGc0/0d0DdPLBVxdpq13O6NFJ0UfgBi3c89gB/zmbbvjiMSp/XIVm73+8CdPtO7v4DlNwFnRy84zRLvB+JwzqQXAZ2LiPlSYAbOU7d13eVXP5/l7Bvr08BLPv1qFBBXzk44d5ruMhrnfu6A82S+P0nhGyDU7f4XcJX7OQpYDVTOM/1LfOarAbDXHWcFYB5Q2y03HPddFTg755xYCiv3G3Cxz3qrVMTyehX4h/u5P7DYZ5ucB0S462YXzpmj57/LYP0rc81clFP9cVoBTQVQ1cLaYs9PX5wfHaq6SkQ24bSNtN+fgVV1p4isd9thWQO0Bn5xe98uIhe7n2NwGuErsnkNEakC9AY+FcltFDKiiMFaAxtUdbXb/R5Oswcvud2FVS99raoqIsuAFFVd5saxAmeHvLiQYfsCH6vTCmuKiPwMdKPw5Xc2Tls+AKj7ngs/9QVedodbLiJL/RzuU/1fS7HnABeIyL1udyTQmGPbAevH/+Zrq4jkND3eGicZzXDXTSiQX9s7+ZYTkapAQ1X9wp2HIwA+67mgeb7ULf+jiNQSkWgvnLUAAALnSURBVOpuv2/VaeDyqIjswEnMyUUtDJM/Swplg5B/07mZuNeNxPnFhRcy/Kn6BBgGrAK+cHewZ+Ls/HqpapqIzMLZ+eQboyunfwiwV1U7n0AMRc3HoUL65bSam+3zOae7qN/JySy/gtaZv8MWxHeceZe17/wLTpVVUhHTyi9GAVaoaq8ihs23nIhU+//27p41qiAK4/j/IYZAQCRCRCstRHwpRLCx0g8gqIVvSBCxErTWUoyCYh/yFWxMY6FCIgnabJVsjNhpYWFAELTRqIzFOXd3WW+yCQlIwvPr9mbu3ZnZZWZyTjKzTPlez1qubp2f1x88rq2LE81bwyRwoTpcRNLOvP6R9rbgZ8hNtYDvwPaO+2eIpCaSDhArxl6DRbenRCL6Mu0V+Q7ga04IB4mjA7stArty5TcAnAYopXwDPkg6n/WSpKM96vAe2Cdpf74eIeLZG62u/y5K6pM0TKywGzXlOr0EblYvJA2t4f1fExMwkg4Toa7KoqRDuXPoubqb0wvgVi4WkHSspswMcCnbtYfcZZf4bgxLOpH39ks6kj/rbHNtufxsP0k6m9cHJA2ycn91fkdPAV/yObbBPClsAaWUBeABMC1pjkgoQpwVcVJSg4jJVyvFJvBb0lzupjkG9GXo5Amxs+ZP1iDDH++AvaWURl5+DmzL8MYokSDsvu8XcI+IMT8jBvbKFeB6tmmBHkctZhjiGhFymidW+eNracdqlNhd9o2kt5IeAxO0k/xTRO7iM//2c6f7wFA+Y472gLsaY8Rg2wRu5/tUZwjcIfpxivqQTmWUWCQ0FQfEj9aUmSDCgfNEIn46279E5BYeZd1niVAfRO5iXNIsES5artwIEVpsEjmB3azcX3eB41n+IXB1hbbZOniXVLNNJv+ktr+U8iP/X2SSSK4v/eeq2Rbg2JvZ5jMIvJLUT8Tab3hCsI3i3xTMzKzFOQUzM2vxpGBmZi2eFMzMrMWTgpmZtXhSMDOzlr9BdqExHBa4IgAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7f35ad084978>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "databasefile = 'databases/MITOS_WSI_CCMCT_ODAEL.sqlite'\n",
    "\n",
    "resfile = 'results/RetinaNet-ODAEL-export.pth-CCMCT_ODAEL-val-inference_results_boxes.p.bz2'\n",
    "\n",
    "thr, F1scores, thrs = optimize_threshold(databasefile=databasefile, resfile=resfile)\n",
    "\n",
    "\n",
    "import matplotlib.pyplot as plt \n",
    "plt.plot(thrs, F1scores)\n",
    "plt.xlabel('cutoff value for mitotic figure detection')\n",
    "plt.ylabel('F1 score')\n",
    "plt.title('Optimization of detection threshold (trainval set)')\n",
    "plt.text(0.7, 0.5, 'max: %.3f @ %.2f' % (np.max(np.array(F1scores)),np.arange(0.5,0.99,0.01)[np.argmax(F1scores)]))\n",
    "plt.plot([0.5, 1.0], np.max(np.array(F1scores))*np.array([1,1]),'r--')\n",
    "\n",
    "plt.savefig('results/figures/Opt_F1_CCMCT_ODAEL_RetinaNet.pdf')\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Evaluation on test set (using optimized threshold from validation set)\n",
    "\n",
    "1. ODAEL dataset variant"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Calculating F1 for test set of 11 files\n",
      "Overall: \n",
      "TP: 15478 FP: 11325 FN:  6998 F1: 0.6281783315408186\n",
      "F1 score:  0.628\n"
     ]
    }
   ],
   "source": [
    "databasefile = 'databases/MITOS_WSI_CCMCT_ODAEL.sqlite'\n",
    "resfile = 'results/RetinaNet-ODAEL-export.pth-CCMCT_ODAEL-inference_results_boxes.p.bz2'\n",
    "\n",
    "F1, individ = calculate_F1(databasefile=databasefile, resfile=resfile, det_thres=0.79)\n",
    "\n",
    "print('F1 score: ', np.round(F1,3))\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "2. HEAEL dataset variant"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Calculating F1 for test set of 11 files\n",
      "Overall: \n",
      "TP: 14986 FP: 11807 FN:  6585 F1: 0.6197171449838723\n",
      "F1 score:  0.62\n"
     ]
    }
   ],
   "source": [
    "databasefile = 'databases/MITOS_WSI_CCMCT_HEAEL.sqlite'\n",
    "resfile = 'results/RetinaNet-ODAEL-export.pth-CCMCT_ODAEL-inference_results_boxes.p.bz2'\n",
    "\n",
    "F1, individ = calculate_F1(databasefile=databasefile, resfile=resfile, det_thres=0.79)\n",
    "\n",
    "print('F1 score: ', np.round(F1,3))\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "3. MEL dataset variant"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Calculating F1 for test set of 11 files\n",
      "Overall: \n",
      "TP: 14975 FP: 11818 FN:  6576 F1: 0.6195184511004468\n",
      "F1 score:  0.62\n"
     ]
    }
   ],
   "source": [
    "databasefile = 'databases/MITOS_WSI_CCMCT_MEL.sqlite'\n",
    "resfile = 'results/RetinaNet-ODAEL-export.pth-CCMCT_ODAEL-inference_results_boxes.p.bz2'\n",
    "\n",
    "F1, individ = calculate_F1(databasefile=databasefile, resfile=resfile, det_thres=0.79)\n",
    "\n",
    "print('F1 score: ', np.round(F1,3))\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Manual Expert Labelling "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Optimize threshold (on train+val set)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Optimizing threshold for validation set of 21 files: \n",
      "Best threshold: F1= 0.6208417258560374 Threshold= 0.8000000000000003\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYUAAAEWCAYAAACJ0YulAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4xLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvAOZPmwAAIABJREFUeJzs3Xd4FWX2wPHvSULvPQFCCYQSuiIEUMSCNAUVUcGGgrh2XftaV13d4lrXn4u6ipUiFkC6WFABIUhNpEkLLYRek5Dk/P6YSbyElEvIzSS55/M8eXLnTjvvzNw5M+/MvCOqijHGGAMQ4nUAxhhjSg5LCsYYY7JZUjDGGJPNkoIxxphslhSMMcZks6RgjDEmmyWFABGRJiJyRERCCzn+ERGJKkkxncF8G4jIfBE5LCL/9mP4kSLyU3HEVkAc54nI2mKaV7GVWURURFoWYrxm7rhhefR/RkQ+zmf8CiKSICLhpzvvfKZ5nYjMKarp5TOf70VkdIDnUUFE1ohI/UDOpyCWFFzuj3KViBwTkV0i8paI1DyN8TeLyMVZ3aq6VVWrqmpGYeJxx91YmHEDFdMZGAPsAaqr6gNFOWERGScizxfRtE7aWarqj6rauiimnWM++e5cy7AxwHxV3QVFs+5U9RNVvaRIoitmORONqqYC7wGPeBeVJQUAROQB4B/AQ0ANIBZoCswVkfJexlZGNAUS1J6ULBLiKI2/3duAj/wdOAiTJsCnwE0iUsGzCFQ1qP+A6sAR4Ooc31cFdgO3uN3PAJOBicBh4Fegk9vvIyATOO5O62GgGaBAmDvM98DzwAJ3mGlAHeAT4BCwBGjmM38FWgIN3eGz/o45q00BWgDfAntxjsQ/AWqeRkwNganAPmADcKvP/J8BJgEfuuWNB7rmsxx7umU46P7v6X4/DjgBpLlxXJzLuHXcOA4Bi4HngJ98+rcB5rpxrs1aVzhHnr7TnuZTrs+BZGATcI/PtEKBvwC/u+VaCkQC891lc9Sd1jVAH2Cbz7ht3fV4wF0eg336jQPeBKa70/0FaJHHstrqzitrnfYARgI/AS8B+924B/iM8z3wN+Bnd522xDmA+R+wE9iOs32FusO3BH5w18ceYGKObetPwHp3Xm8C4vYLAZ4AtuBs/x8CNdx+zTh5+2nuzuOwu37+A3ycR5mbuHGHFbDuNuMcKa8EUoEw4FGf9ZUAXOEz3ZGcvK3kWjaggrve2vsMW8+NqT5QC/gaZ5vZ735unGP5j86jbN2AOJztNwl42adfLM5v/gCwAujjfv83IANIccv/H59x1gPne7ZP9GrGJeUP6A+kZ22sOfp9AIx3Pz/jbsRXAeWAB3F+uOV8NuaLfcbN+QP6HmfH2wLnx5wArAMudjf8D4H3c2zcLXOJ6ROfmFoCfd0Nvh7Oju1Vn2ELiukH4P+AikBn9wdxkU95U4CBODvSF4FFeSzD2u4P6Qa3LMPd7jpu/3HA8/msgwk4CagK0B5nB/eT268KkAjc7E77LJydXLvcpo2zU1sKPAWUB6KAjUA/t/9DwCqgNc7OopNPnCctc3ySgrvON+AklPLAhTg7qdY+cezD2UGEuetpQh7lPWk9uN+NxNm+bnWX9+3ADv7YWX+Pk0zaudMvB3wFjHWXUX2chHqbO/x44HF3eVQEzs2xbX0N1MTZWScD/d1+t7jljMI5MPoC+CiP7Wch8DLO9tfbXR55JYVBQHyO707ZLnC22eU4ibqS+90wnEQfgpOsjwIRPsstZ1LIq2zvAX/zGfZOYJb7uQ4wFKgMVAM+A77yGfZ78k4KC4Eb3M9VgVj3cyOcA7aBbux93e56+U0T5wDpntzmVRx/pfEUtKjVBfaoanou/Xa6/bMsVdXJqnoC58dQEedIwF/vq+rvqnoQmAn8rqrfuPP+DOiS38gi8gjOUfMtAKq6QVXnqmqqqia7MZ3vTyAiEgmcCzyiqimquhx4F2fHnuUnVZ2hzjWIj3B2oLkZBKxX1Y9UNV1VxwNrgMv8iCMU58f4lKoeVdXVOMk4y6XAZlV93532rzhnAVflMclzcH50z6pqmjrXZd4BrnX7jwaeUNW16lihqnsLihNnPVcF/u5O91ucnc9wn2G+UNXF7vr8BCfRno4tqvqOu7w/ACKABj79x6lqvDv92sAA4D53ue0GXvEp5wmcaruG7vrNeRH776p6QFW3At/5xHodzpHuRlU9AjwGXJuzKkdEmuAs6yfd7W8+ztlvXmriJA1/vK6qiap6HEBVP1PVHaqaqaoTcY6ku+Uzfl5l+5ST19cI9ztUda+qfq6qx1T1MM6RvF+/JZxl3VJE6qrqEVVd5H5/PTDD/Q1lqupcnDOKgQVM7zDO8vKEJQXnqLNuHvWXEW7/LIlZH1Q1E9iGcwTjrySfz8dz6a6a14giMgC4F7g868ciIvVFZIKIbBeRQ8DHnJzE8tMQ2Of+ALJswTm6ybLL5/MxoGIey6mhO66vnNPKSz2cI99En+98p9UU6C4iB7L+cHZced3B0hRomGP4v/DHzjUSpyridDUEEt317htnfssrz/WZh+zxVfWY+9F3Gr7LqCnO2cJOn3KOxTljAKe6UIDFIhIvIrfkNa8cseZcl1tw1o9vcsoabr+qHs0xbF724xyB+8O3nIjIjSKy3Kec7cl/O8+rbN8ClUSku4g0xUkWX7rzqCwiY0Vki/tbmg/U9PNOvVFAK2CNiCwRkUvd75sCw3Jsi+fi7FfyUw2nuskTwXghJ6eFOHWXV+JUYQAgIlVwjsT+4jNspE//EKAxzik+OKetASEirXGOHK9UVd8fzIvufDuq6l4RuRynXjdLfjHtAGqLSDWfxNAEp+rmdO3A+QH4agLM8mPcZJzqu0ics4uscbMkAj+oat88xs9ZxkRgk6pG5zF8Ik4V3mo/YvO1A4gUkRCfxNAEpwrwdBV2W/EdLxFnu62b21muOnf43AogIucC34jIfFXdUMA8cq7LJjjrJwlne8+yE6glIlV8EkMT8i7bSiBKRMJ84s1r2Ozv3Z33O8BFwEJVzRCR5TgJ77SoaqaITMI5W0gCvvbZ9h/AqVLsrqq7RKQzsMyf+ajqemC4u0+4EpgsInVw1tFHqnprQeXMoS1Q4K3bgRL0ZwpuVc5fgTdEpL+IlBORZjjVOds4+W6Js0XkSvdo+T6cH2XWqWISTj1skRKR6sAUnCqPnFUA1XAuUh0QkUY49eW+8ozJTS4LgBdFpKKIdMQ54vmkEGHOAFqJyAgRCRORa4AYnOqVfLlVJV8Az7hHazHATT6DfO1O+wZ33ZQTkXNEpG0eZVwMHBKRR0SkkoiEikh7ETnH7f8u8JyIRLt38XR0f8C5TcvXLzh12Q+7MfTBqR6bUFAZc5GMcxNAobcXVd0JzAH+LSLVRSRERFqIyPkAIjJMRLJ24vtxdkD+3Io8HrhfRJqLSFXgBZyL1CclHlXdglMV8lcRKe8mnjyrC1V1G6dW+/jzm6nixp7slutmnDOFwvoU57rEde7nLNVwztYPiEht4Gl/Jygi14tIPfdgIesIPwPnzP0yEennbocVRaSPz3o5pfzu77g2f+xXil3QJwUAVf0nzhnBSzh3EPyCk+UvUufe4SxTcDaorIuqV7rXF8A5an/CPU18sAjDOwvnCOZlcR48OyIiR9x+f3X7H8S56+WLHOMWFNNwnIuHO3BOo5926z1Pi1snfynO0dZenKqLS1V1T74j/uEunFP8XTgXH9/3mfZh4BKcuvId7jD/wLm4Cc7dNzFuGb9yk8xlOFUDm3Cq/97FubgPznWXSTg71EPu+JXcfs8AH7jTujpHGdOAwThnj3twLtDfqKprOE1u1dDfgJ/deZ3OdSlfN+Jc9E7A2SYn80fVxDnAL+62MhW4V1U3+THN93AOhObjLL8U4O48hh0BdMe5wP40zs0S+RnLydesTlp3uY2gqgk4R80LcXaiHXDuwCoUVc1K7g1xrutleRVnO9iDs0P25yw3S38g3l3WrwHXutdxEoEhOPuWZJx9ykP8sd99DbhKRPaLyOvudyOAD3Lsd4pV1p0NpgAi8gzOnSnXex2LMaWROPfeL8M52NrpdTwljbt8VgC93RsHPGHXFIwxxcI9+o3xOo6Syl0+bbyOw6qPjDHGZLPqI2OMMdnsTMEYY0y2UndNoW7dutqsWTOvwzDGmFJl6dKle1S1XkHDlbqk0KxZM+Li4rwOwxhjShURye+J82xWfWSMMSabJQVjjDHZLCkYY4zJZknBGGNMNksKxhhjsllSMMYYk82SgjHGmGyl7jmFM9Knz6nfXX013HEHHDsGA3N5S97Ikc7fnj1wVS5vgLz9drjmGkhMhBtuOLX/Aw/AZZfB2rVw222n9n/iCbj4Yli+HO6779T+L7wAPXvCggXwl7+c2v/VV6FzZ/jmG3j++VP7jx0LrVvDtGnw71ze2/HRRxAZCRMnwltvndp/8mSoWxfGjXP+cpoxAypXhv/7P5g06dT+33/v/H/pJfg6x+sVKlWCmW7rxc89B/Pmndy/Th34/HPn82OPwcKFJ/dv3Bg+/tj5fN99zjL01aoVvP2283nMGFiX4304nTs7yw/g+uth27aT+/foAS++6HweOhT25nhr50UXwZNPOp8HDIDjx0/uf+ml8KDbYnkht720628kedM2qt0wnLT0TDIyFQVUYfXgEazpPYAKu3Zw2UuPUD4shAruX1hoCGLbnvO/LG17WWUKoOBKCsaUYBmZyoHjJ9h/NI3jJzL4Ylo849bOpObRg7y149Apw09Zvp2vU9cRcSiZjnuPntQvRIQpX67i+PFIbqydQicK8aoyE5RKXYN4Xbt2VXui2ZQVh1JOMO+3JGau2sUP65JJTc+kTpXyxDSsTkSNikTUqETDmn/8r1qhHGGhQrnQEMqFCmEhIYSFCIdSTrBt/3G2HzjO9v3H2XHgOIn7j7Fgw14Op6bTvG4Vru4aydCzG1G/WkWvi208ICJLVbVrgcNZUjCmeB1JTWdO/C6mrdjBTxv2cCJDCa9ekf7tw+nfPpxzmtUmNKRojuuPp2UwY9VOJsYlsnjTPkJDhAvb1Gdkz2b0alm3SOZhSgdLCsaUICknMvhuzW6mrtjBt2t2k5qeSaOalRjYIZwBHSLo3LgmIUWUCPLye/IRJsUl8vnS7ew5kkrfmAY8OSiGJnUqB3S+pmQoEUlBRPrjvIc0FHhXVf+eyzBX47wbV4EVqjoiv2laUjClRWamsmjTXiYv3cac+CSOpKZTt2p5BnWIYHDnhpzVpBYixV/Tn5aeyXs/b+L1eetJz1T+dH4Lbj+/BZXKhxZ7LKb4eJ4URCQUWAf0BbYBS4Dh7ou4s4aJxnmJ+oWqul9E6hf0blJLCqak237gOJ8v3cZnSxNJ3HecahXC6N8+nMGdG9Ijqg5hoSXjTvBdB1N4YcZvTF2xg0Y1K/HkpW3p1y7ck0RlAs/fpBDIu4+6ARtUdaMb0ARgCJDgM8ytwJuquh/Ay5dVG3Mm0jMymbl6F5PiEvlpwx5UoVfLOjzQtzX92oWXyKPw8BoVeX14F0Z0b8LTU+L508e/cmGb+rx2bWeqVSzndXjGI4FMCo2ARJ/ubUD3HMO0AhCRn3GqmJ5R1Vk5JyQiY4AxAE2aNAlIsMYUhqoyJyGJf85aw+/JR2lUsxL3XBjNVWc3JrJ26airj42qw/R7zmXcgs28OHMNw99ZxLibu1G3agWvQzMeCGRSyO0cNGddVRgQDfQBGgM/ikh7VT1w0kiqbwNvg1N9VPShGnP6lm7Zxwsz1rB0y36i6lXhv9efzSUxDQJ+wTgQwkJDGH1eFFH1qnDHJ78y7L8L+fCWbqUmsZmiE8jKzW1ApE93Y2BHLsNMUdUTqroJWIuTJIwpsTbsPsKYD+MY+tZCtu47xgtXdGDOfb3p3z68VCYEXxe2acDHo7qz90gqQ99awNpdh70OyRSzQCaFJUC0iDQXkfLAtcDUHMN8BVwAICJ1caqTNgYwJmMKbc+RVB7/chX9Xp3Pzxv28EDfVvzwUB9GdG9SYi4eF4WuzWrz2Z96IgLD/ruAuM37vA7JFKOAbcmqmg7cBcwGfgMmqWq8iDwrIoPdwWYDe0UkAfgOeEhV9+Y+RWO8kXIigze/20Cff33PhCWJXNe9CT88fAF3XxRN5fJls6WY1uHVmPynntSpWoHr//cL365J8jokU0zs4TVj8pCZqUxdsYN/zV7L9gPHubhtAx4b2IYW9ap6HVqx2XMklZHvL2btrsN8cHM3etpT0KWWv7eklp1zXmOK0NIt+7j8/37mvonLqVWlHJ/e2p13b+oaVAkBoG7VCnwyOpbmdatw28dLWZdk1xjKOksKxvhIPpzKA5NWMPSthew+lMq/h3Vi6p3n0rNF8B4h16hUjvdGnkPFcqHc/P4Sdh9O8TokE0CWFIzBefjs/Z83ceFL3zN1xXZu79OCeQ+cz9CzG5f6O4qKQuNalXl/5DnsP5bGqHFxHE1N9zokEyCWFEzQ+2XjXi594yf+Oi2Bzk1qMuu+3jzSvw1VKpTNi8iF1b5RDd4Y3oX4HQe5Z/wy0jMyvQ7JBIAlBRO0Dh47wUOfreCatxdxOCWd/15/Fh/e0i3orhucjovaNuCvQ9ozb81u/jotgdJ2o4opmB0KmaA0NyGJx79cxd6jadzRpwV3XxhdItsnKoluiG3Ktn3HGDt/I01qV+bW3lFeh2SKkCUFE1T2HU3jr9PimbJ8B23Cq/HeyHNo36iG12GVOo/0b8O2/cd5YeZvdIqsSbfmtb0OyRQRqz4yQWP6yp30ffkHZqzayZ/7tmLqXedaQiikkBDhX8M60rhWJR75fCUpJzK8DskUEUsKpsxLOpTCbR/Fceenv9KoViWm3X0u91wUTfkw2/zPROXyYfz9yo5s2nOUV+au8zocU0Ss+siUWZmZyieLt/LPmWtIy8jkkf5tuPW85mWqnSKv9WpZl+HdmvDOjxud14pG1vQ6JHOG7NdhyqT1SYcZNnYhT361mk6RNZlzf29u79PCEkIAPDawDQ2qV+ThyStITbdqpNLOfiGmTEk5kcHLc9cx8PUf2Zh8hH8P68RHo7rRtE4Vr0Mrs6pXLMcLV3RgXdIR/vPtBq/DMWfIqo9MmbFk8z4e/Xwlvycf5coujXh8UFvq2NvDisUFbepz5VmN+L/vf6dfu3C7gF+K2ZmCKfUOp5zgya9WM+y/C0lNz+TDW7rx8jWdLSEUs6cujaFW5fI8PHklJ+xp51LLkoIp1b5JSKLvy/P55JctjDq3OXPu703vVvW8Diso1axcnucvb0/CzkOM/eF3r8MxhWTVR6ZUSj6cyl+nxfP1yp20blCN/95wtt35UgL0bx/OoI4RvDZvPZd2bEizunYtp7SxMwVT6syJ30W/V+czJz6JB/q2Ytrd51pCKEGevjSGEBH+73u76FwaWVIwpcbR1HQe/XwlYz5aSsOaFZl+z7ncbQ+hlTj1q1dkeLcmfPHrdhL3HfM6HHOa7NdkSoXliQcY9PqPTIxL5PY+Lfji9l5EN6jmdVgmD7edH0WICG/ZtYVSx5KCKdHSMzJ5fd56hr61gBMZyvhbY3mkfxs7OyjhImpU4qqujZkct42dB497HY45DfbLMiXW1r3HuObtRbw8dx2Xdoxgxr3nERtVx+uwcjVr1ixat25Ny5Yt+fvf/57ncJMmTSImJoZ27doxYsQIAJYvX06PHj1o164dHTt2ZOLEidnD/+c//6Fly5aICHv27Mk3hmXLlnHNNdfQoUMHzjnnHJ555hmOH897h/zBBx8QHR1NdHQ0H3zwQa7DLF++nNjYWDp37kzXrl1ZvHgxAKrKPffcQ8uWLenYsSO//vrrKePefn4LMlUZ+8PGfOM2JYyqlqq/s88+W03ZlpmZqRMWb9GYJ2dq+6dn6VfLtnkdUr7S09M1KipKf//9d01NTdWOHTtqfHz8KcOtW7dOO3furPv27VNV1aSkJFVVXbt2ra5bt05VVbdv367h4eG6f/9+VVX99ddfddOmTdq0aVNNTk7OM4YpU6Zoz549dcGCBZqZmampqak6duxYveCCCzQlJeWU4ffu3avNmzfXvXv36r59+7R58+bZcfnq27evzpgxQ1VVp0+frueff3725/79+2tmZqYuXLhQu3XrlmtcD05arq0en6FJh47nGbspHkCc+rGPtTMFU6LsOZLKmI+W8sjnq+jYuCaz7+vNkM6NTmsamzdvpk2bNowePZr27dtz3XXX8c0339CrVy+io6Ozj3YXL15Mz5496dKlCz179mTt2rUAvPzyy9xyyy0ArFq1ivbt23PsWN4XTBcvXkzLli2JioqifPnyXHvttUyZMuWU4d555x3uvPNOatWqBUD9+vUBaNWqFdHR0QA0bNiQ+vXrk5ycDECXLl1o1qxZvuU9cOAAzz77LLNnz6ZHjx6ICOXLl2fMmDFcd911vP7666eMM3v2bPr27Uvt2rWpVasWffv2ZdasWacMJyIcOnQIgIMHD9KwYUMApkyZwo033oiIEBsby4EDB9i5c+cp4995QUtOZGTyznw7WygtApoURKS/iKwVkQ0i8mgu/UeKSLKILHf/RgcyHlOyzfstif6vzueHdck8Magtn4zuTsOalQo1rQ0bNnDvvfeycuVK1qxZw6effspPP/3ESy+9xAsvvABAmzZtmD9/PsuWLePZZ5/lL3/5CwD33XcfGzZs4Msvv+Tmm29m7NixVK5cmbi4OEaPPnUT3b59O5GRkdndjRs3Zvv27acMt27dOtatW0evXr2IjY3NdSe8ePFi0tLSaNGihd9lnTRpErfddhtVq1blueee46yzzuKhhx5i1KhR3HTTTcycObPQMb/66qs89NBDREZG8uCDD/Liiy+e1vjN6lZhSOdGfLxoK3uPpPpdJuOdgCUFEQkF3gQGADHAcBGJyWXQiara2f17N1DxmJLrWFo6f/lyFaM+iKNu1QpMu+tcRp8XRUiIFHqazZs3p0OHDoSEhNCuXTsuuugiRIQOHTqwefNmwDnyHTZsGO3bt+f+++8nPj4egJCQEMaNG8cNN9zA+eefT69evQDo2rUr77576iaqubynWOTU2NPT01m/fj3ff/8948ePZ/To0Rw4cCC7/86dO7nhhht4//33CQnx/6e5YsUKYmNjWbFiBcuXLycuLo4ePXrw+++/ExaW+/Op/sb81ltv8corr5CYmMgrr7zCqFGjTmt8cM4WUtIz+N9Pm/wuk/FOIM8UugEbVHWjqqYBE4AhAZyfKYUSdhzisjd+YvzirdzWO4opd/WidfiZ32paocIf7R6FhIRkd4eEhJCeng7Ak08+yQUXXMDq1auZNm0aKSkp2eOsX7+eqlWrsmPHjgLn1bhxYxITE7O7t23bll3NknO4IUOGUK5cOZo3b07r1q1Zv349AIcOHWLQoEE8//zzxMbGnlZZVZXQ0FDWrFlD3759CQkJYcCAAUUS8wcffMCVV14JwLBhw7Kr3vwdH6Bl/aoM7BDBhwu3cOBY2mmVzRS/QCaFRkCiT/c297uchorIShGZLCKRufQ3ZZCq8t5Pm7j8zZ85nJLOx6O689jAtlQICy22GA4ePEijRs4mOW7cuJO+v/fee5k/fz579+5l8uTJ+U7nnHPOYf369WzatIm0tDQmTJjA4MGDTxnu8ssv57vvvgNgz549rFu3jqioKNLS0rjiiiu48cYbGTZs2GmXo0OHDixcuJDWrVszb948MjMzmT17NuDs1LPOdHz169ePOXPmsH//fvbv38+cOXPo16/fKcM1bNiQH374AYBvv/02+9rH4MGD+fDDD1FVFi1aRI0aNYiIiMgzxrsvbMmR1HTe/3nzaZfPFDN/rkYX5g8YBrzr030D8EaOYeoAFdzPfwK+zWNaY4A4IK5JkyZFekXeFL/kwyk68r1ftOkjX+uocYt175HUIp3+pk2btF27dtndN910k3722Wen9FuwYIFGR0drz5499YknntCmTZuqqurNN9+sr732mqqqbt26VVu0aKFJSUm6ZMkSHTVqVK7znD59ukZHR2tUVJQ+//zz2d8/+eSTOmXKFFV17qq6//77tW3bttq+fXsdP368qqp+9NFHGhYWpp06dcr+W7Zsmaqqvvbaa9qoUSMNDQ3ViIiIXOe/Z88ebdu2rR4+fFiffvpp7dKliz744IPau3dvffzxxzU1Nffl+7///U9btGihLVq00Pfeey/7+1GjRumSJUtUVfXHH3/Us846Szt27KjdunXTuLi47LLccccdGhUVpe3bt88ePj+3frBEOzw9Sw8eTytwWFP08PPuI9Fc6gaLgoj0AJ5R1X5u92NuEnoxj+FDgX2qmm9D7F27dtW4uLiiDtcUk/nrkvnzpBUcSjnBE4PackNs0zzroo3/Jk6cyNixY3nzzTdp27YtJ06cYNasWTRt2pSOHTt6HR4AK7cdYPB/fubpy2K4uVdzr8MJOiKyVFW7FjRcIFtJXQJEi0hzYDtwLTDCdwARiVDVrPvYBgO/BTAe46G09Ez+PWctY+dvpFWDqnw8uhttwqt7HVaZcc0119C0aVMee+wxtmzZQtWqVRk0aBCXXHKJ16Fl69i4Jp0a12D84q2M7NnMDgZKqIAlBVVNF5G7gNlAKPCeqsaLyLM4pzFTgXtEZDCQDuwDRgYqHuOdrXuPcfeEZaxIPMCI7k14clAMlcoX37WDYBEbG8tXX33ldRj5GtG9CY98voqlW/bTtVltr8MxuQhY9VGgWPVR6TJl+XYe/3I1IQL/GNqRAR3yvhhpyr6jqel0f2Eel7RrwMtXd/Y6nKDib/WRPdFsAuJYWjoPT17BvROW0zq8GjPuPc8SgqFKhTAu79KQ6St3cvDYCa/DMbmwpGCK3OrtB7nsjZ/4bOk27rqgJRPHxNK4VmWvwzIlxPBuTUhNz+SLZdu8DsXkwpKCKTIZmcpb3//OFf/3M0dSnWcPHuzXmrBQ28zMH9o1rEGnyJp8+svWXJ+MNt6yX6spEtv2H2P4O4v4x6w19I1pwOz7etOrZV2vwzIl1HXdmrB+9xGWbtnvdSgmB0sK5ox9tWw7A179kYQdh/j3sE68OeIsalYu73VYpgS7tFME1SqE8enjVMD9AAAgAElEQVQvW70OxeRgScEU2sFjJ7hn/DLum7icVuHVmHnveQw9u7Hdf24KVLl8GJd3acTXq3Zae0gljCUFUygLNuyh/2vzmb5qJw/0bcXEMbFE1raLycZ/w7s1IS09ky9+PbXJbeMdSwrmtKScyOD5rxMY8e4vVCoXyhe39+Tui6LtYrI5bTENq9M5sibjF9sF55LEfsnGb7/tPMTlb/7Muz9t4obYpky/5zw6Rdb0OixTio1wLzjH2QXnEsOSgilQZqbyzvyNDPnPz+w9msb7N5/Dc5e3t6YqzBnLuuA83i44lxiWFEy+Evc5t5r+bcZvXNCmHrPv680Fret7HZYpI+yCc8ljScHkSlWZFJfIgNd+JH7HIf55VUf+e/3Z1K5it5qaopV1wXnaioLfcmcCL5BNZ5tSas+RVB77YhVzE5Lo3rw2Lw3rZHcWmYCJaVid6PpV+XrlTm7o0czrcIKeJQVzkrkJSTz2xUoOpaTzxKC23NKrOSEh9tyBCaxBHSN4bd56dh9KoX71il6HE9Ss+sgATpPGj36+kls/jKN+tYpMu+tcRp8XZQnBFItBHSJQhVnxu7wOJejZmYJhReIB7pu4nM17j3JHnxbcd3EryofZ8YIpPtENqtGqgVOFdKNVIXnKfvlBLCNTefO7DQx9awGpJzIYf2ssD/dvYwnBeGJQh4Ys2byP3YdSvA4lqNmvP0jtOHCcEe8s4l+z19KvfTgz7+1NbFQdr8MyQWxQx3BUYeZqq0LyklUfBaFpK3bw+JeryMhUXhrWiaFnNbJG7IznWtavRpvwakxfuZObejbzOpygZUkhiBw4lsaTU+KZtmIHnSNr8uo1nWlWt4rXYRmTbWCHCF75Zh27DqYQXsPuQvKCVR8Fifnrkun36nxmuq2aTv5TD0sIpsQZ6N6FNHP1Tq9DCVqWFMq442kZPDVlNTe+t5hqFcvx5R29rFVTU2K1rF+VNuHVmLHKkoJXrPqoDFuReID7Jy5n456j3NKrOQ/3b03FctaInSnZLu0YwUtzrArJK3a4WAZlZCpvzFvPlW8t4PiJDD4Z3Z2nLouxhGBKhYEdIgDsbMEjAU0KItJfRNaKyAYReTSf4a4SERWRroGMJxgk7jvGNWMX8u+56xjYIYJZ9/amV8u6XodljN+i6lWlbUR1SwoeCVhSEJFQ4E1gABADDBeRmFyGqwbcA/wSqFiCgary+dJtDHjtR9buOsyr13TmjeFdqFG5nNehGXPaLu0YQdyW/ew8eNzrUIJOIM8UugEbVHWjqqYBE4AhuQz3HPBPwB5jLKSDx05w1/hlPPDZCmIiqjPj3vO4vEsjr8MyptD+qEKyB9mKWyCTQiMg0ad7m/tdNhHpAkSq6tf5TUhExohInIjEJScnF32kpdiSzfsY8Np8Zq/exUP9WjN+TKw1c21KveZ1qxATUZ3pK+0dC8UtkEkht0dks9/OLSIhwCvAAwVNSFXfVtWuqtq1Xr16RRhi6ZWRqbz2zXquGbuQcmEhfH57T+68oCWh1qqpKSMGdYzg160H2HHAqpCKUyCTwjYg0qe7MeCb9qsB7YHvRWQzEAtMtYvNBdt58DjD31nEK9+sY3Cnhnx997l0iqzpdVjGFKlBdheSJwKZFJYA0SLSXETKA9cCU7N6qupBVa2rqs1UtRmwCBisqnEBjKnUmxO/iwGv/cjq7Qd5+epOvHptF6pVtIvJpuxpVrcKbcKrMTchyetQgkrAkoKqpgN3AbOB34BJqhovIs+KyOBAzbesOpqazhNfrWLMR0tpXKsS0+85jyvPaux1WMYEVN+YBizZvI/9R9O8DiVoBPSJZlWdAczI8d1TeQzbJ5CxlGaLN+3jwc9WkLj/GLee15wH+7WmQpg9iGbKvr4xDXjj2w18u2Y3Q8+2g6DiYM1clGApJzJ4afZa/vfzJhrXqsSEW2Ppbu88MEGkfcMaNKhegbkJSZYUioklhRJq2db9PPDZCjYmH+X62CY8NqAtVSrY6jLBJSREuLhtA75ctp2UExnWVEsxKPCagohUFpEnReQdtztaRC4NfGjBKS09k3/NXsPQtxaQkpbBx6O68/zlHSwhmKDVN6YBx9IyWPD7Hq9DCQr+7GneB5YCPdzubcBnQL4PnJnTt2bXIe6fuILfdh7iqrMb89RlMVS3O4tMkOvRog5VK4QxNyGJC9s08DqcMs+fu49aqOo/gRMAqnqc3B9MM4WUkamM/eF3Br/xM8mHU3jnxq68NKyTJQRjgAphoZzfqh7f/LabzEwteARzRvw5U0gTkUq4TyOLSAsgNaBRBZHEfcd4YNIKFm/eR792DXjhig7UqVrB67CMKVH6xjRg+qqdLN92gLOa1PI6nDLNn6TwNDALiBSRT4BewMhABhUMVJWJSxJ57usEQkT497BOXHlWI0TsJMyYnC5oXZ/QEGFuQpIlhQDLNymIs4daA1yJ0wyFAPeqql3xOQM7Dx7n0c9X8cO6ZHpE1eGlqzvRqGYlr8MypsSqUbkc3ZvXZm5CEo/0b+N1OGVavklBVVVEvlLVs4HpxRRTmaWqTF66jWe/TiA9Q3l2SDuu796UEGvEzpgC9Y1pwF+nJbBpz1Ga163idThllj8XmheJyDkBj6SMSzqUwqgP4nho8krahldn1n3ncWOPZpYQjPFT3xjnzqO5CfaOhUDy55rCBcBtIrIFOIpThaSq2jGgkZURqsqXy7bzzNR40jIyefqyGG6yZGDMaWtcqzJtI6ozNyGJMb1beB1OmeVPUhgQ8CjKqN2HU/jLF6v55rckujatxb+GdbLTXmPOQN+YBvzn2/XsPZJqd+kFSIHVR6q6BagJXOb+1XS/M3lQVaYs384lr8znx/XJPDGoLRNv62EJwZgzdElMAzIV5q3Z7XUoZZY/zVzcC3wC1Hf/PhaRuwMdWGm150gqt3/8K/dOWE7zulWYce95jD4vyt6IZkwRaNewOg1rVLR3LASQP9VHo4DuqnoUQET+ASwE3ghkYKXR9JU7eXLKao6kpvPYgDaWDIwpYiLCxTENmBSXyPG0DCqVtwbyipo/dx8JkOHTnYE1c3GSfUfTuPPTX7nz01+JrFWJ6Xefy23nt7CEYEwA9I1pQMqJTH7aYI9LBYK/DeL9IiJfut2XA/8LXEily+z4XTz+5SoOHj/BQ/1ac1vvKMJCA/mWU2OCW/fmdahWIYy5Cbuyb1M1RafApKCqL4vI98C5OGcIN6vqskAHVtIdOJbGM1Pj+Wr5Dto1rM7Ho7vTJry612EZU+aVDwuhT5v6zPttNxmZamfkRazApCAisUC8qv7qdlcTke6q+kvAoyuhvl2TxKOfr2Lf0TTuvSiauy5sSTk7OzCm2PSNacC0FTtYtnU/XZvV9jqcMsWf6qO3gLN8uo/m8l1Q2HMklee/TuCr5Tto3aAa7408h/aNangdljFBp0/repQLdRrIs6RQtPxJCqKq2Y2Yq2qmiATVa8Cy2iz624zfOJqazj0XRXPnBS2oEGZ3PhjjheoVyxEbVYe5CUk8NrCt1+GUKf7UeWwUkXtEpJz7dy+wMdCBlRSb9hxlxDu/8NDklUTXr8qMe87jz31bWUIwxmOXxDRg456jbNh9xOtQyhR/ksKfgJ7AdpxXcXYHxgQyqJIgLT2TN7/bQL9X57N6x0FeuKIDE8f0ILpBNa9DM8YAF7t3Hs2xBvKKlD93H+0Gri3MxEWkP/AaEAq8q6p/z9H/T8CdOM8+HAHGqGpCYeZVlH7dup/HPl/F2qTDDOoQwdOXxVC/ekWvwzLG+IioUYkOjWowNyGJO/q09DqcMsOfZi7+KSLV3aqjeSKyR0Su92O8UOBNnAb1YoDhIhKTY7BPVbWDqnYG/gm8XIgyFJnDKSd4aspqhr61gEMpJ3j3xq68ed1ZlhCMKaH6xjRgeeIBdh9O8TqUMsOf6qNLVPUQcClO9VEr4CE/xusGbFDVjaqaBkwAhvgO4E43SxXc90B7YU78Lvq+PJ+PFm3hph7NmPvn87NPT40xJVPfmAaowrzfrIG8ouLPXUTl3P8DgfGqus/P9wg3AhJ9urOuR5xERO4E/gyUBy70Z8JFadfBFJ6ZGs+s+F20Ca/Gf284m86RNYs7DGNMIbQJr0bjWpWYm5DE8G5NvA6nTPAnKUwTkTXAceAOEakH+HOullvmOOVMQFXfBN4UkRHAE8BNp0xIZAzuxe0mTYpmxaelZ/Lez5t4fd56MjKVh/u35tbzouwhNGNKERGhb0wDPvllK0dT06lSIajulg8If96n8CjQA+iqqieAY+SoBsrDNiDSp7sxsCOf4SfgtKuUWwxvq2pXVe1ar149P2adv5/W72HAa/P5+8w19GxRl2/+fD539LGnko0pjfrGNCAtPZMf1yd7HUqZ4FdaVdX9Pp+P4jzVXJAlQLSINMe5nfVaYITvACISrarr3c5BwHoCaMeB4/xt+m9MX7WTpnUq8/7Ic7igTf1AztIYE2DdmtWmRqVyzElIon/7CK/DKfUCdq6lqukichcwG+eW1PdUNV5EngXiVHUqcJeIXAycAPaTS9VRUZm8dBtPfrWaTFUe6NuKW3tHUbGcPYBmTGkXFhrCRW3q8+2a3aRnZForxWcooBVwqjoDmJHju6d8Pt8byPn7alyrEudF1+XJS2OIrF25uGZrjCkGfWMa8MWy7cRt2U9sVB2vwynVCpUURKSNqq4p6mACKTaqjm0sxpRRvVvVo3xYCHMTkux3foYKe541p0ijMMaYM1ClQhi9WtRhTsIufNrvNIWQ55mCiLyeVy/AbuQ3xpQofWPC+e5Lp3kae+FV4eV3pnAzsBpYmuMvDkgLfGjGGOO/i9s6dxLOjU/yOJLSLb9rCkuA1aq6IGcPEXkmYBEZY0wh1K9ekS5NajInIYm7L4r2OpxSK78zhauA5bn1UNXmgQnHGGMKr3+7cFZtP8i2/ce8DqXUyi8pVFVVW7LGmFKjX7twAGZbFVKh5ZcUvsr6ICKfF0MsxhhzRprVrUKb8GrMjrcX7xRWfknBt0G7qEAHYowxRaFfu3CWbN5H8uFUr0MplfJLCprHZ2OMKbH6tw9HFb75zaqQCiO/pNBJRA6JyGGgo/v5kIgcFpFD+YxnjDGeaRNejaZ1KjNrtVUhFUaet6SqqrUWZ4wpdUSEfu3Cef/nTRxKOUH1iuUKHslks+YEjTFlTr924ZzIUL5bY6/pPF2WFIwxZU6XyJrUr1bBqpAKwZKCMabMCQlxqpC+X5tMyokMr8MpVSwpGGPKpH7twjl+IoP56+w1nafDkoIxpkzqHuW8pnOWPch2WiwpGGPKpHKhIVzctgHfJCRxIiPT63BKDUsKxpgyq3/7cA6lpLNo416vQyk1LCkYY8qs86LrUrl8qLWFdBosKRhjyqyK5ULp07oes+OTyMy01nr8YUnBGFOm9WsXTvLhVJYl7vc6lFLBkoIxpky7oE19yoWKPcjmJ0sKxpgyrXrFcpzbsi4zVu1C1aqQChLQpCAi/UVkrYhsEJFHc+n/ZxFJEJGVIjJPRJoGMh5jTHAa0CGC7QeOs2r7Qa9DKfEClhREJBR4ExgAxADDRSQmx2DLgK6q2hGYDPwzUPEYY4LXJTENCAsRpq/a6XUoJV4gzxS6ARtUdaOqpgETgCG+A6jqdz7vgV4ENA5gPMaYIFWzcnl6tazLjFU7rQqpAIFMCo2ARJ/ube53eRkFzMyth4iMEZE4EYlLTrZ2TIwxp29QhwgS9x1n9XZ7R1h+ApkUJJfvck3RInI90BX4V279VfVtVe2qql3r1atXhCEaY4JF35gGhFoVUoECmRS2AZE+3Y2BHTkHEpGLgceBwapqb9o2xgRErSrl6dmiDjNXWxVSfgKZFJYA0SLSXETKA9cCU30HEJEuwFichGCvSDLGBNSgDhFs2XuM+B1WhZSXgCUFVU0H7gJmA78Bk1Q1XkSeFZHB7mD/AqoCn4nIchGZmsfkjDHmjF3SLpzQEGGGVSHlKSyQE1fVGcCMHN895fP54kDO3xhjfNWuUp4eUXWYsWonD/VrjUhulz6Dmz3RbIwJKgM7RLB57zESdloVUm4sKRhjgkq/ds5dSDNXWVtIubGkYIwJKnWqViA2qrY9yJYHSwrGmKAzsEMEG/ccZc2uw16HUuJYUjDGBJ1+7cIJEewupFxYUjDGBJ26VSvQvXkdplsV0iksKRhjgtLAjhFsTD7KuqQjXodSolhSMMYEpf5uFZK1hXQySwrGmKBUr5pThTR1+XYyM60KKYslBWNM0Lq2WySb9x7jxw17vA6lxLCkYIwJWgPaR1C3agU+XLDZ61BKDEsKxpigVT4shOHdIvl27W4S9x0reIQgYEnBGBPURnRvQogIHy/a4nUoJYIlBWNMUIuoUYlLYhowMS6RlBMZXofjOUsKxpigd2OPZhw4doKpK055OWTQsaRgjAl6sVG1adWgKh8u3Bz0TzhbUjDGBD0R4YYezVi9/RDLEg94HY6nLCkYYwxwZZdGVKsQFvS3p1pSMMYYoEqFMIae3ZgZq3aRfDjV63A8Y0nBGGNc18c2JS0jk4lLtnodimcsKRhjjKtl/aqc27Iun/yylfSMTK/D8YQlBWOM8XFjj6bsPJjCN78leR2KJywpGGOMj4vaNqBRzUqMC9ILzgFNCiLSX0TWisgGEXk0l/69ReRXEUkXkasCGYsxxvgjNES4uVczFm3cx89B2HpqwJKCiIQCbwIDgBhguIjE5BhsKzAS+DRQcRhjzOm6PrYpjWpW4u8z1wTduxYCeabQDdigqhtVNQ2YAAzxHUBVN6vqSiA4r+gYY0qkiuVC+XPfVqzafjDo3swWyKTQCEj06d7mfmeMMSXe5V0a0Sa8Gi/NWUtaevActwYyKUgu3xXqPExExohInIjEJScnn2FYxhhTsNAQ4ZEBbdiy9xgTgui5hUAmhW1ApE93Y6BQTRCq6tuq2lVVu9arV69IgjPGmIL0aVWP2KjavD5vPUdS070Op1gEMiksAaJFpLmIlAeuBaYGcH7GGFOkRIRHB7Rlz5E03v1xo9fhFIuAJQVVTQfuAmYDvwGTVDVeRJ4VkcEAInKOiGwDhgFjRSQ+UPEYY0xhdI6sycAO4bw9f2NQtIkU0OcUVHWGqrZS1Raq+jf3u6dUdar7eYmqNlbVKqpaR1XbBTIeY4wpjAcvaU1qeiZvfLve61ACzp5oNsaYAkTVq8rwbpF8+stWNu856nU4AWVJwRhj/HDPRdGUCw3hX3PWeh1KQFlSMMYYP9SvVpFbe0cxfeVOflpfdpu/sKRgjDF+uqNPC5rXrcJfvlzF8bQMr8MJCEsKxhjjp4rlQnnxyg5s3XeMV79Z53U4AWFJwRhjTkNsVB2Gd4vknR83snr7Qa/DKXKWFIwx5jQ9OqAtdapW4OHJKzlRxt7QZknBGGNOU41K5XhuSDsSdh7ifz9t8jqcImVJwRhjCqF/+wguiWnAK3PXlalnFywpGGNMIT07pD3lQ0P4y5erUC0bL+OxpGCMMYUUXqMijw5sw4Lf9/LZ0m1eh1MkLCkYY8wZGH5OE7o1q83zXyfw285DXodzxiwpGGPMGQgJEf59dScqlw9jxDuLSn1isKRgjDFnKLJ2ZSaMiaViudBSnxgsKRhjTBFoVrcK42/9IzEk7CidicGSgjHGFJFmdatknzFc927pTAyWFIwxpgg1reMkhkrlQhlRChODJQVjjCliTetUYfyYWCq7iWHxpn1eh+Q3SwrGGBMAzhlDD2pVLs917y5iUlyi1yH5xZKCMcYESJM6lfnqjl50b16Hhyev5G/TE8jILNlPPltSMMaYAKpRuRzv33wON/Zoyjs/buLWD+M4nHLC67DyZEnBGGMCrFxoCM8Oac9zQ9rxw7pkhr61gMR9x7wOK1eWFIwxppjc0KMZH9zcjV0HUxjy5s+sSDzgdUinsKRgjDHF6Nzounx1Zy9iIqoTUbOi1+GcIqBJQUT6i8haEdkgIo/m0r+CiEx0+/8iIs0CGY8xxpQEUfWq8vHo7tSvFkRJQURCgTeBAUAMMFxEYnIMNgrYr6otgVeAfwQqHmOMMQUL5JlCN2CDqm5U1TRgAjAkxzBDgA/cz5OBi0REAhiTMcaYfAQyKTQCfJ/W2OZ+l+swqpoOHATq5JyQiIwRkTgRiUtOTg5QuMYYYwKZFHI74s/51IY/w6Cqb6tqV1XtWq9evSIJzhhjzKkCmRS2AZE+3Y2BHXkNIyJhQA2g9DQSYowxZUwgk8ISIFpEmotIeeBaYGqOYaYCN7mfrwK+1bLy9mtjjCmFwgI1YVVNF5G7gNlAKPCeqsaLyLNAnKpOBf4HfCQiG3DOEK4NVDzGGGMKFrCkAKCqM4AZOb57yudzCjAskDEYY4zxn5S22hoRSQa2FHL0usCeIgynNLAyBwcrc3A4kzI3VdUC79QpdUnhTIhInKp29TqO4mRlDg5W5uBQHGW2to+MMcZks6RgjDEmW7Alhbe9DsADVubgYGUODgEvc1BdUzDGGJO/YDtTMMYYkw9LCsYYY7KVyaTgx8t9RopIsogsd/9GexFnUSqozO4wV4tIgojEi8inxR1jUfNjPb/is47XiUjJe/fhafKjzE1E5DsRWSYiK0VkoBdxFhU/yttUROa5Zf1eRBp7EWdREpH3RGS3iKzOo7+IyOvuMlkpImcVaQCqWqb+cJrU+B2IAsoDK4CYHMOMBP7jdazFXOZoYBlQy+2u73XcgS5zjuHvxmlqxfPYA7ye3wZudz/HAJu9jjvA5f0MuMn9fCHwkddxF0G5ewNnAavz6D8QmInTynQs8EtRzr8snin483KfssafMt8KvKmq+wFUdXcxx1jUTnc9DwfGF0tkgeNPmRWo7n6uwaktE5cm/pQ3Bpjnfv4ul/6ljqrOJ//WoocAH6pjEVBTRCKKav5lMSn483IfgKHuqddkEYnMpX9p4k+ZWwGtRORnEVkkIv2LLbrA8Hc9IyJNgebAt8UQVyD5U+ZngOtFZBtOu2N3F09oAeFPeVcAQ93PVwDVROSUF3WVMX5v+4VRFpOCPy/umQY0U9WOwDf88UrQ0sqfMofhVCH1wTlqfldEagY4rkDy6wVNrmuByaqaEcB4ioM/ZR4OjFPVxjjVDB+JSGn9nftT3geB80VkGXA+sB1ID3RgHjudbf+0ldaNJT8FvtxHVfeqaqrb+Q5wdjHFFij+vtBoiqqeUNVNwFqcJFFa+VPmLNdS+quOwL8yjwImAajqQqAiTiNqpZE/v+UdqnqlqnYBHne/O1h8IXridLb901YWk0KBL/fJUf82GPitGOMLBH9eaPQVcAGAiNTFqU7aWKxRFi1/yoyItAZqAQuLOb5A8KfMW4GLAESkLU5SKK0vNvfnt1zX50zoMeC9Yo7RC1OBG927kGKBg6q6s6gmHtD3KXhB/Xu5zz0iMhjnNHMfzt1IpZafZZ4NXCIiCUAG8JCq7vUu6jPjZ5nBqU6ZoO5tG6WZn2V+AHhHRO7HqVIYWVrL7md5+wAviogC84E7PQu4iIjIeJxy1XWvDT0NlANQ1f/iXCsaCGwAjgE3F+n8S+n2YowxJgDKYvWRMcaYQrKkYIwxJpslBWOMMdksKRhjjMlmScEYY0w2SwpBSESaicgIP4cd7zYHcr+ItHFbHF0mIi3OMIbN7vMSZ0REKojIN25c15zp9AqY1wwRqen+3eHH8CctZxHpKiKvn8b8znNbtF0uIo1EZHJhYy8KIjJORK4qYJiRItKwkNM/o+VlioYlheDUDCgwKYhIONBTVTuq6ivA5ThPRXdR1d8DHKO/ugDlVLWzqk70ZwQRCS3MjFR1oKoeAGoCBSYFcixnVY1T1XtOY5bXAS+5ZduuqvnukP0lIoF8PmkkUKikwJkvL1MUvG4m1v6K5g+4EViJ00DYR+5344CrfIY54v5fBBwElgP34zz1+j6wCqd57Qvc4VYCx93hngZ24bQt812Oed8O/NOneyTwhvv5K2ApEA+M8RlmM07zC83waSIYpy2bZ9zPLYBZ7vg/Am1yzLc+zgM8WWVpgfM07zK3LO8BFXzm9xTwE3BtjumMA97CaWVzI04bOu/hPOk+LpeYJ/gsl3/htEXzL2C1O99r8ljOfYCv3X5VfZb5SmBojphG4zxYuQn4xHc5AZVxmrJYCUwEfgG6+q5j9/NVWfG7ZXzZLeO/gSpuGZe4y2tILtuUAP8BEoDpOA9NXeX2Oxv4wV03s4EId35HcJpQWQ5Uym04d/yWOO2OrQB+ddddfsurNs62tNIdrqP7/TNuOb531909Xv8WS/uf5wHYXxGsRGjn/hDrut213f/jyD0pZP/Y3O4HgPfdz21wmkqoyKk77GeAB3OZfz2cJo6zumcC5+aIpRLOTrOO272ZgpPCPCDa/dwd+DaXefvuOCritB7Zyu3+ELjPZ34P57H8xuHs6AWnWeJDQAecM+mlQOcCYh4KzMV56raBu/wiclnOvrH+A3jVp1+tPOLK2glnz9NdRmPdz+1xnsz3Jyl8DYS63S8A17ufawLrgCo55n+lT7kaAgfcaZYDFgD13OGuwX1XBc7OOSuW/Ib7BbjCZ71VLmB5vQE87X6+EFjus00uACq462Yvzpmj57/L0vpX5pq5CFIX4rQCugdAVfNriz035+L86FDVNSKyBadtpEP+jKyqySKy0W2HZT3QGvjZ7X2PiFzhfo7EaYSvwOY1RKQq0BP4TCS7UcgKBYzWGtikquvc7g9wmj141e3Or3ppmqqqiKwCklR1lRtHPM4OeXk+454LjFenFdYkEfkBOIf8l9/FOG35AKDuey78dC7wmjveahFZ6ed4n+kfLcVeAgwWkQfd7opAE05uB6w3f5Rrh4hkNT3eGicZzXXXTSiQW9s7uQ4nItWARqr6pVuGFACf9ZxXmYe6w38rInVEpIbbb7o6DVymishunMS8raCFYXJnSaFsEHJvOhmzXSAAAALBSURBVDcd97qROL+48vmMf6YmAlcDa4Av3R1sH5ydXw9VPSYi3+PsfHKN0ZXVPwQ4oKqdTyOGgspxNJ9+Wa3mZvp8zuou6HdSmOWX1zrzd9y8+E4z57L2Lb/gVFmtLWBeucUoQLyq9ihg3FyHE5HqeQxf0LTyis13fWVg+7UzYheay4Z5wNVZLxcRkdru95v5o1nwIbiNagGHgWo+48/HuaiJiLTCOWIsaGeR0xc4F6KH88cReQ1gv5sQ2uC8OjCnJKC+e+RXAbgUQFUPAZtEZJgbl4hIpwJiWAM0E5GWbvcNOPXZRS235XeNiISKSD2cI+zFuQznaw5wV1aHiNQ6jfn/hJOAEZEYnKquLEki0tZtOfSK3EZ2zQbudg8WEJEuuQwzH7jWLVcEbiu7ONtGPRHp4Y5bTkTauf18y5zrcO663SYil7vfVxCRyuS/vHy30T7AHnc6pohZUvj/9u4YJ4EgCuP4/0Vp7OzsvIS9BzHGGCtPoKURTTT2hCtY0VDYiKGwoWIX8QweQdFkLN7bwSgiRBMD+X4lWWBnipnhe8vMEkgpDYFzoGtmBV5QBD8rYtvMengmX60US+DNzIrYTbMBrER0co3vrPnCHCL+eAQ2U0q9ePkGWI14o44XCD+/7xU4xTPmNj6wV3aAg2jTkB+OWowYYh+PnAb4Kr85TztmkXx32XszezCzK6DFuMjfwWsXT3zt54/OgPX4jILxgDuLBj7YlsBRfE91hsAx3o8dJkc6lTq+SCjND4ivT7imhceBA7wQ3432j/DawmXcex+P+sBrF00z6+Nx0XfX7eLRYonXBDaY3l8nwFZcfwHsTWmb/IJ2SRVZMPFIbS2l9Bz/F7nFi+ujf741WQLK3kQWzxpwZ2Y1PGs/1IQgf0W/FEREJFNNQUREMk0KIiKSaVIQEZFMk4KIiGSaFEREJHsHTmc80VgYMN8AAAAASUVORK5CYII=\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7f35e2631940>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "databasefile = 'databases/MITOS_WSI_CCMCT_MEL.sqlite'\n",
    "\n",
    "resfile = 'results/RetinaNet-MEL-export.pth-CCMCT_MEL-val-inference_results_boxes.p.bz2'\n",
    "\n",
    "thr, F1scores, thrs = optimize_threshold(databasefile=databasefile, resfile=resfile)\n",
    "\n",
    "\n",
    "import matplotlib.pyplot as plt \n",
    "plt.plot(thrs, F1scores)\n",
    "plt.xlabel('cutoff value for mitotic figure detection')\n",
    "plt.ylabel('F1 score')\n",
    "plt.title('Optimization of detection threshold (trainval set)')\n",
    "plt.text(0.7, 0.5, 'max: %.3f @ %.2f' % (np.max(np.array(F1scores)),np.arange(0.5,0.99,0.01)[np.argmax(F1scores)]))\n",
    "plt.plot([0.5, 1.0], np.max(np.array(F1scores))*np.array([1,1]),'r--')\n",
    "\n",
    "plt.savefig('results/figures/Opt_F1_CCMCT_MEL_RetinaNet.pdf')\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "1. MEL on MEL data set"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Calculating F1 for test set of 11 files\n",
      "Overall: \n",
      "TP: 14158 FP: 10931 FN:  7393 F1: 0.6071183533447685\n",
      "F1 score:  0.607\n"
     ]
    }
   ],
   "source": [
    "databasefile = 'databases/MITOS_WSI_CCMCT_MEL.sqlite'\n",
    "resfile = 'results/RetinaNet-MEL-export.pth-CCMCT_MEL-inference_results_boxes.p.bz2'\n",
    "\n",
    "F1, individ = calculate_F1(databasefile=databasefile, resfile=resfile, det_thres=0.80)\n",
    "\n",
    "print('F1 score: ', np.round(F1,3))\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "2. MEL on ODAEL data set variant"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Calculating F1 for test set of 11 files\n",
      "Overall: \n",
      "TP: 14653 FP: 10445 FN:  7823 F1: 0.6160087442720814\n",
      "F1 score:  0.616\n"
     ]
    }
   ],
   "source": [
    "databasefile = 'databases/MITOS_WSI_CCMCT_ODAEL.sqlite'\n",
    "resfile = 'results/RetinaNet-MEL-export.pth-CCMCT_MEL-inference_results_boxes.p.bz2'\n",
    "F1, individ = calculate_F1(databasefile=databasefile, resfile=resfile, det_thres=0.80)\n",
    "\n",
    "print('F1 score: ', np.round(F1,3))\n",
    "\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Calculating F1 for test set of 11 files\n",
      "Overall: \n",
      "TP: 14171 FP: 10918 FN:  7400 F1: 0.6074153450492927\n",
      "F1 score:  0.607\n"
     ]
    }
   ],
   "source": [
    "databasefile = 'databases/MITOS_WSI_CCMCT_HEAEL.sqlite'\n",
    "resfile = 'results/RetinaNet-MEL-export.pth-CCMCT_MEL-inference_results_boxes.p.bz2'\n",
    "F1, individ = calculate_F1(databasefile=databasefile, resfile=resfile, det_thres=0.80)\n",
    "\n",
    "print('F1 score: ', np.round(F1,3))\n",
    "\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Hard Example Augmented Expert Labelling (HEAEL) -  dataset variant"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Optimizing threshold for validation set of 21 files: \n",
      "Best threshold: F1= 0.6062053455927717 Threshold= 0.8300000000000003\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYUAAAEWCAYAAACJ0YulAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4xLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvAOZPmwAAIABJREFUeJzs3Xd4FWX2wPHvSQgt9BR6SSP0DtIUFJWiggUEe2d1Lair/uyyiuvau6uuInZELAiioCKySO8QhBAgQAAhoYSahCTn98dM4iWkXCA3N+V8nidP7tx5Z+ZMuXNm3pl5R1QVY4wxBiDA3wEYY4wpPSwpGGOMyWVJwRhjTC5LCsYYY3JZUjDGGJPLkoIxxphclhR8RESaicghEQk8xeEPiUhkaYrpNKZbX0TmiMhBEXnRi/LXi8jckoitiDjOFJH1JTStEptnEVERiT6F4Vq4w1YqoP9YEfmkkOGriMhaEWlwstMuZJxXicjM4hpfIdOZLSI3+3gaVURknYiE+3I6RbGk4HJ/lKtF5IiI/Cki/xGROicxfKKInJvTrapbVbWGqmadSjzusJtOZVhfxXQaRgMpQC1V/UdxjlhEJojIuGIa13E7S1X9n6rGFse480yn0J1rOTYamKOqf0LxrDtV/VRVzy+W6EpY3kSjqunAeOD//BeVJQUAROQfwLPA/UBtoCfQHPhJRCr7M7ZyojmwVu1JyWIhjrL42/0b8LG3hStg0gT4DLhORKr4LQJVrdB/QC3gEHB5nu9rALuBG93uscBk4AvgILAM6Oj2+xjIBo6643oAaAEoUMktMxsYB8xzy0wFQoBPgQPAYqCFx/QViAYaueVz/o44q00BooBZwB6cI/FPgTonEVMj4DtgL5AA3OIx/bHAJOAjd37jgG6FLMfe7jykuv97u99PAI4BGW4c5+YzbIgbxwFgEfAUMNejfyvgJzfO9TnrCufI03PcUz3m6ysgGdgM3OUxrkDgYWCjO19LgabAHHfZHHbHNRLoDyR5DNvaXY/73eUx1KPfBOBN4Ht3vAuBqAKW1VZ3WjnrtBdwPTAXeAHY58Y92GOY2cDTwO/uOo3GOYB5H9gJbMfZvgLd8tHAb+76SAG+yLNt3QpscKf1JiBuvwDgUWALzvb/EVDb7deC47efCHcaB9318wbwSQHz3MyNu1IR6y4R50h5FZAOVAIe9Fhfa4FLPMZ7PcdvK/nOG1DFXW/tPMqGuTGFA3WBaTjbzD73c5M8y//mAuatB7AEZ/vdBbzk0a8nzm9+P7AS6O9+/zSQBaS58/+GxzAbgH5+2yf6a8Kl5Q8YBGTmbKx5+n0IfO5+HutuxMOBIOA+nB9ukMfGfK7HsHl/QLNxdrxROD/mtUA8cK674X8EfJBn447OJ6ZPPWKKBs5zN/gwnB3bKx5li4rpN+AtoCrQyf1BDPCY3zRgCM6O9BlgQQHLsJ77Q7rGnZcr3O4Qt/8EYFwh62AiTgIKBtrh7ODmuv2CgW3ADe64u+Ds5NrmN26cndpS4HGgMhAJbAIGuv3vB1YDsTg7i44ecR63zPFICu46T8BJKJWBc3B2UrEecezF2UFUctfTxALm97j14H53Pc72dYu7vG8DdvDXzno2TjJp644/CPgWeMddRuE4CfVvbvnPgUfc5VEV6Jtn25oG1MHZWScDg9x+N7rzGYlzYPQ18HEB28984CWc7e8sd3kUlBQuAOLyfHfCdoGzza7ASdTV3O9G4CT6AJxkfRho6LHc8iaFguZtPPC0R9nbgR/dzyHAZUB1oCbwJfCtR9nZFJwU5gPXuJ9rAD3dz41xDtiGuLGf53aHFTZOnAOku/KbVkn8lcVT0OIWCqSoamY+/Xa6/XMsVdXJqnoM58dQFedIwFsfqOpGVU0FfgA2qurP7rS/BDoXNrCI/B/OUfONAKqaoKo/qWq6qia7MfXzJhARaQr0Bf5PVdNUdQXwHs6OPcdcVZ2uzjWIj3F2oPm5ANigqh+raqaqfg6sAy7yIo5AnB/j46p6WFXX4CTjHBcCiar6gTvuZThnAcMLGGV3nB/dk6qaoc51mf8Co9z+NwOPqup6daxU1T1FxYmznmsA/3bHOwtn53OFR5mvVXWRuz4/xUm0J2OLqv7XXd4fAg2B+h79J6hqnDv+esBg4G53ue0GXvaYz2M41XaN3PWb9yL2v1V1v6puBX71iPUqnCPdTap6CHgIGJW3KkdEmuEs68fc7W8OztlvQergJA1vvKaq21T1KICqfqmqO1Q1W1W/wDmS7lHI8AXN22ccv76udL9DVfeo6leqekRVD+IcyXv1W8JZ1tEiEqqqh1R1gfv91cB09zeUrao/4ZxRDClifAdxlpdfWFJwjjpDC6i/bOj2z7Et54OqZgNJOEcw3trl8floPt01ChpQRAYDY4CLc34sIhIuIhNFZLuIHAA+4fgkVphGwF73B5BjC87RTY4/PT4fAaoWsJwaucN6yjuugoThHPlu8/jOc1zNgTNEZH/OH86Oq6A7WJoDjfKUf5i/dq5NcaoiTlYjYJu73j3jLGx5Fbg+C5A7vKoecT96jsNzGTXHOVvY6TGf7+CcMYBTXSjAIhGJE5EbC5pWnljzrsstOOvHMznllNunqofzlC3IPpwjcG94zicicq2IrPCYz3YUvp0XNG+zgGoicoaINMdJFt+406guIu+IyBb3tzQHqOPlnXo3AS2BdSKyWEQudL9vDozIsy32xdmvFKYmTnWTX1TECzl5zcepu7wUpwoDABEJxjkSe9ijbFOP/gFAE5xTfHBOW31CRGJxjhwvVVXPH8wz7nQ7qOoeEbkYp143R2Ex7QDqiUhNj8TQDKfq5mTtwPkBeGoG/OjFsMk41XdNcc4ucobNsQ34TVXPK2D4vPO4DdisqjEFlN+GU4W3xovYPO0AmopIgEdiaIZTBXiyTnVb8RxuG852G5rfWa46d/jcAiAifYGfRWSOqiYUMY2867IZzvrZhbO959gJ1BWRYI/E0IyC520VECkilTziLahs7vfuzvu/wABgvqpmicgKnIR3UlQ1W0Qm4Zwt7AKmeWz7/8CpUjxDVf8UkU7Acm+mo6obgCvcfcKlwGQRCcFZRx+r6i1FzWcerYEib932lQp/puBW5fwTeF1EBolIkIi0wKnOSeL4uyW6isil7tHy3Tg/ypxTxV049bDFSkRqAVNwqjzyVgHUxLlItV9EGuPUl3sqMCY3ucwDnhGRqiLSAeeI59NTCHM60FJErhSRSiIyEmiDU71SKLeq5GtgrHu01ga4zqPINHfc17jrJkhEuotI6wLmcRFwQET+T0SqiUigiLQTke5u//eAp0Qkxr2Lp4P7A85vXJ4W4tRlP+DG0B+nemxiUfOYj2ScmwBOeXtR1Z3ATOBFEaklIgEiEiUi/QBEZISI5OzE9+HsgLy5Fflz4B4RiRCRGsC/cC5SH5d4VHULTlXIP0Wkspt4CqwuVNUkTqz28eY3E+zGnuzO1w04Zwqn6jOc6xJXuZ9z1MQ5W98vIvWAJ7wdoYhcLSJh7sFCzhF+Fs6Z+0UiMtDdDquKSH+P9XLC/Lu/43r8tV8pcRU+KQCo6nM4ZwQv4NxBsBAnyw9Q597hHFNwNqici6qXutcXwDlqf9Q9TbyvGMPrgnME85I4D54dEpFDbr9/uv1Tce56+TrPsEXFdAXOxcMdOKfRT7j1nifFrZO/EOdoaw9O1cWFqppS6IB/uQPnFP9PnIuPH3iM+yBwPk5d+Q63zLM4FzfBufumjTuP37pJ5iKcqoHNONV/7+Fc3AfnussknB3qAXf4am6/scCH7rguzzOPGcBQnLPHFJwL9Neq6jpOkls19DTwuzutk7ku5elanIvea3G2ycn8VTXRHVjobivfAWNUdbMX4xyPcyA0B2f5pQF3FlD2SuAMnAvsT+DcLFGYdzj+mtVx6y6/AVR1Lc5R83ycnWh7nDuwTomq5iT3RjjX9XK8grMdpODskL05y80xCIhzl/WrwCj3Os42YBjOviUZZ59yP3/td18FhovIPhF5zf3uSuDDPPudEpVzZ4MpgoiMxbkz5Wp/x2JMWSTOvffLcQ62dvo7ntLGXT4rgbPcGwf8wq4pGGNKhHv028bfcZRW7vJp5e84rPrIGGNMLqs+MsYYk8vOFIwxxuQqc9cUQkNDtUWLFv4OwxhjypSlS5emqGpYUeXKXFJo0aIFS5Ys8XcYxhhTpohIYU+c57LqI2OMMbksKRhjjMllScEYY0wuSwrGGGNyWVIwxhiTy5KCMcaYXJYUjDHG5PLpcwoiMginedhA4D1V/Xc+ZS7HabJYgZWqeqXPAurf/8TvLr8c/v53OHIEhuTzlrzrr3f+UlJgeD5vgLztNhg5ErZtg2uuObH/P/4BF10E69fD3/52Yv9HH4Vzz4UVK+Duu0/s/69/Qe/eMG8ePPzwif1feQU6dYKff4Zx407s/847EBsLU6fCi/m8t+Pjj6FpU/jiC/jPf07sP3kyhIbChAnOX17Tp0P16vDWWzBp0on9Z892/r/wAkzL83qFatXgB7f14qeegl9+Ob5/SAh89ZXz+aGHYP784/s3aQKffOJ8vvtuZxl6atkS3n3X+Tx6NMTneR9Op07O8gO4+mpISjq+f69e8MwzzufLLoM9ed7aOWAAPPaY83nwYDh69Pj+F14I97ktlpfibS972XLS77yLg2mZpGdmEyAQIMLK2+7nQOceNIhbRof/PEfNqpUIFI93zti253wuyW0vZ558yGdJwX2N3Zs4L6tOAhaLyHdu++g5ZWJw3gHbR1X3iUh4/mMzxhSXLFUOpmUya1kSU7Yu4uiipdy77cS3P747ZzPLNlWlS9IGHth5ABGhVrUg6lYLok71IKr6IXbjez5rEE9EegFjVXWg2/0QgKo+41HmOSBeVd/zdrzdunVTe6LZmJOTmZXN/xJSmLJ8OzPidnH0WBYiEBNegy7N6jp/zesQEVqDY1nZpGdmk5GZTXpmFhmZ2STtO8rs9cnMXr+bTSnO2zcjQ4MZ0DqcK89oTkRosJ/n0BRFRJaqarciy/kwKQwHBqnqzW73NTjvP73Do8y3OO+47YNTxTRWVU9445GIjAZGAzRr1qzrli1ePa1tTIWmqqxMSuXb5duZtmoHKYcyqFW1Ehd0aMSgdg3o3KwOtaoGnfR4t+w5zOz1yfy6fjfzEvZwLDubc2LDubFvBL2jQhA56dcnmxLgbVLw5TWF/LaMvBmoEhAD9Md5Kfj/RKSdqh53Lquq7wLvgnOmUPyhGlN+ZGcrM+L+5NVfNrDuz4NUrhTAgFbhXNy5Mf1jw6hSKfC0xt88JJjregdzXe8W7D6YxqcLtvLJgi1c9d5CYuvX5Ma+LRjWqTFVg05vOsY/fJkUkoCmHt1NcN6xm7fMAvc9x5tFZD1Okljsw7iMKZeys5Uf4/7k1Z83sH7XQSJDg3nm0vYMad+Q2tVO/ozAG+E1q3LPeS25rX8UU1fu4P25m/m/r1bz/Iz1PHtZBwa0ru+T6Rrf8WX1USWcqqEBwHacHf2VqhrnUWYQcIWqXicioTjvb+3kvgg+X3ZNwZjjnZAMwoIZMyCGCzs0IjCgZKtyVJX5m/YwbtofrN15gFv7RXHf+S2pFGh3v/ub36uPVDVTRO4AZuBcLxivqnEi8iSwRFW/c/udLyJrgSzg/sISgjHmL6rKzLW7ePmneNb9eZCosGBeHdXJL8kgh4jQOyqUr//em39OXcvbv21k2dZ9vH5FZ+rXsvuVyoIy9zpOO1MwFZ2q8lt8Mi/OjGf19lQiQ4MZc65/zgyK8s3yJB7+eg3BVQJ5bVRnekeH+jukCsvvZwrGmOI3f+MeXpy5niVb9tGkbjWeH96BSzo3LrXVM5d0bkLbRrX5+6fLuPr9hdx9bkvuODuagFKWvMxfLCkYUwas3XGAf03/g7kJKdSvVYVxF7fj8m5NqVypdCYDTy3r12TK7X14+JvVvPRTPIfSM3l4SGt/h2UKYEnBmFJs98E0XpwRz6Sl26hdLYhHL2jN1T2bl7nbPYOrVOKVkZ2oVTWId+dsonGdalzXu4W/wzL5sKRgTCmUdiyL9+du5q1fE8jIyuamPhHceU4Mtav75tbSkiAijB3alp2paYydGkeD2lUZ2LaBv8MyeZT+c09jKhBVZcqK7Zzzwmyen7GevjGhzLynH49e2KZMJ4QcgQHC61d0pkOTOtz1+XKWbd3n75BMHpYUjCkl1mxPZcTb8xkzcQV1gyvz+S09eeeabuWuXaFqlQN5/7pu1K9VlZs/XEKi25aSKR0sKRjjZ3sPZ/DwN6u56I25bE45zLOXtWfqHX3pFRXi79B8JrRGFT68sQeqynUfLGLPoXR/h2RclhSM8ZPMrGw+nJdI/+d/5YvF27ixTwSz7uvPyO7NKsQtmxGhwbx3XXf+TE3jpg+XcDQjy98hGSwpGOMXixP3cuHrc3niuzg6NKnDj2PO5LEL2/isjaLSqmvzurw6qhMrtu3n9Vkb/B2Owe4+MqZE7TmUzr9/WMeXS5NoXKcab1/dlYFt61fo5qYHtWvIsE6NeH/uZq7q2ZzGdar5O6QKzc4UjCkB2dnK54u2MuCl3/hm+XZu6x/FT/eexaB2DSp0Qshx/8BYFHj+x3X+DqXCszMFY3wsbkcqj367huVb93NGRD3GXdyOmPo1/R1WqdKkbnVu6hvBf2Zv5IY+EXRsWsffIVVYdqZgjI8cy8rm5Z/iGfrG72zdc4SXLu/IxNE9LSEU4O/9owgJrszT3/9BWWuoszyxpGCMD2xKPsTw/8zj1V82MKxjI2b9oz+XdmliVUWFqFk1iLvPa8mixL3MiNvl73AqLEsKxhQjVeWTBVu44LW5bNl7hLeu6sJLIzuVi6eRS8IV3ZsSHV6Df//wBxmZ2f4Op0KypGBMMUk+mM5NHy7h0W/X0K1FXWbcfRZD2jf0d1hlSqXAAB4e0orEPUf4ZMEWf4dTIdmFZmOKwax1u7j/y1UcSs9k7EVtuLZXiwrxAJovnB0bTp/oEF6btYHLujSxs6wSZmcKxpyGjMxsxk1by40TlhBeqyrT7uzL9X0iLCGcBhHhkSFtSD16zB5o8wNLCsacosSUw1z2n3m8N3cz1/Vqzjd/7213FhWTNo1qMbxLEz6cn8iWPdZgXkmypGDMKZiyYjsXvj6XrXuP8M41XfnnsHZl7sU3pd19A2MREd6fu9nfoVQolhSMOQlHMjJ5YPJKxkxcQasGNZk+5kx7UYyP1K9VlcHtGvDt8u2kHbPG8kqKJQVjvLRh10GGvfE7Xy5N4s5zopk4umduOz0//vgjsbGxREdH8+9//7vAcUyaNIk2bdrQtm1brrzyytzvP/zwQ2JiYoiJieHDDz/M/T4jI4PRo0fTsmVLWrVqxVdffVXguGfNmsVFF11E+/bt6dWrF6+88gpZWQXvTJ955hmio6OJjY1lxowZ+Zb55Zdf6NKlC506daJv374kJCQA8Pbbb9O+ffvc79euXVvgdE7HyO5NOZCWyY9r/vTJ+E0+VLVM/XXt2lWNKWlfLd2mrR79Qbs+NVP/F598XL/MzEyNjIzUjRs3anp6unbo0EHj4uJOGEd8fLx26tRJ9+7dq6qqu3btUlXVPXv2aEREhO7Zs0f37t2rERERuWUef/xxfeSRR1RVNSsrS5OTk08Yr6rqW2+9pQMHDtTVq1erquqhQ4f06aef1uHDh2t2dvYJ5ePi4rRDhw6alpammzZt0sjISM3MzDyhXExMjK5du1ZVVd9880297rrrVFU1NTU1t8yUKVN04MCBBS+805CVla1nPTdLR74zzyfjr0iAJerFPtbOFEy5k5iYSKtWrbj55ptp164dV111FT///DN9+vQhJiaGRYsWAbBo0SJ69+5N586d6d27N+vXrwfgpZde4sYbbwRg8bIVhDWL5u5PF9KhSW2+v+tM+saEHje9RYsWER0dTWRkJJUrV2bUqFFMmTLlhLj++9//cvvtt1O3bl0AwsPDAZgxYwbnnXce9erVo27dupx33nn8+OOPAIwfP56HHnoIgICAAEJDQ08Y74YNG5g0aRLTpk2jXbt2AAQHB/Pwww/TqlUrJk+efMIwU6ZMYdSoUVSpUoWIiAiio6Nzl4snEeHAgQMApKam0qhRIwBq1aqVW+bw4cM+e1I7IEC4vFtTFmzay2Z7Q1uJ8GlSEJFBIrJeRBJE5MF8+l8vIskissL9u9mX8ZiKIyEhgTFjxrBq1SrWrVvHZ599xty5c3nhhRf417/+BUCrVq2YM2cOy5cv58knn+Thhx8G4O677yYhIYG3JnzGucNGUanf37jr/Hbc0ymQR+6944Rpbd++naZNm+Z2N2nShO3bt59QLj4+nvj4ePr06UPPnj1zd/wFDb9//34AHnvsMbp06cKIESPYtevE5h8++OADHn74YQICArj99tvp2rUrY8eOZcyYMdx777188sknpxzze++9x5AhQ2jSpAkff/wxDz7418/4zTffJCoqigceeIDXXnvthGGLy/CuTQgMECYt2eazaZi/+CwpiEgg8CYwGGgDXCEibfIp+oWqdnL/3vNVPKZiiYiIoH379gQEBNC2bVsGDBiAiNC+fXsSExMB58h3xIgRtGvXjnvuuYe4uDjAOSK/6oF/c8fom6jSrB0TH7ue+wbG0vOMHrz33ombqObTeFt+R86ZmZls2LCB2bNn8/nnn3PzzTezf//+AofPzMwkKSmJPn36sGzZMnr16sV99913QtmVK1fSs2dPpk6dSlBQEEuXLqVWrVqkpqZSt25dDh48eMoxv/zyy0yfPp2kpCRuuOEG7r333tx+t99+Oxs3buTZZ59l3LhxJwxbXOrXqsrZseFMXprEsSxr+sLXfHmm0ANIUNVNqpoBTASG+XB6xuSqUqVK7ueAgIDc7oCAADIzMwHnCPzss89mzZo1TJ06lbS0NNKOZfHQ16t5/KOfqVytOn0aBtCvZVih02rSpAnbtv11FJuUlJRbzZK33LBhwwgKCiIiIoLY2Fg2bNhQ4PAhISFUr16dSy65BIARI0awbNmyE8arqgQGBrJu3ToGDRoEwODBgwFIT08/blmcTMzJycmsXLmSM844A4CRI0cyb968E8Y1atQovv3224IXUDEY1b0pyQfT+XXdbp9Ox/g2KTQGPM/3ktzv8rpMRFaJyGQRaZpPf0RktIgsEZElycnJvojVVECpqak0buxskhMmTCAzW7nkrXl8+r8/0PkTWLbgdw4f2J9vnbyn7t27s2HDBjZv3kxGRgYTJ05k6NChJ5S7+OKL+fXXXwFISUkhPj6eyMhIBg4cyMyZM9m3bx/79u1j5syZDBw4EBHhoosuYvbs2YBzJ1CbNieebLdv35758+cTGxvLzJkzAec6hary7LPPMnz48BOGGTp0KBMnTiQ9PZ3NmzezYcMGevTocVyZunXrkpqaSnx8PAA//fQTrVu3BpzrGDm+//57YmJiCl1Gp6t/bBjhNavwxWKrQvI5b65Gn8ofMAJ4z6P7GuD1PGVCgCru51uBWUWN1+4+MkXZvHmztm3bNrf7uuuu0y+//PKEfvPmzdOYmBjt3bu3jrxljAbVDteO/5yhgy4dpa+++qqqqm7dulWjoqJ0165dunjxYr3pppvyneb333+vMTExGhkZqePGjcv9/rHHHtMpU6aoqmp2drbec8892rp1a23Xrp1+/vnnueXef/99jYqK0qioKB0/fnzu94mJiXrmmWdq+/bt9ZxzztEtW7acMO24uDjt0aOHZmRk6K233qpdunTRJ554Qrt27aovvvhivncfqaqOGzdOIyMjtWXLljp9+vTc7wcPHqzbt29XVdWvv/5a27Vrpx06dNB+/frpxo0bVVX1rrvu0jZt2mjHjh21f//+umbNmoJWR7F57sc/NOLBabpz/1GfT6s8wsu7j0R99DILEekFjFXVgW73Q24SeqaA8oHAXlWtXdh4u3XrpkuWLCnucE0FlZ6Zxbhpf/Dxgi10aVaH16/sUibfEfzCCy8wf/58Xn75ZZo1a8bRo0f5+uuvOeuss467oFyWbdlzmH7Pz+a+81tyxzm+PTMpj0Rkqap2K6qcL6uPFgMxIhIhIpWBUcB3ngVExLNd4aHAHz6Mx5jjxO1IZdgbv/Pxgi3ccmYEX/ytV5lMCAD33XcfN910E7fccgudO3dm8ODBHDp0KLd6rDxoHhJM76gQvliyjexsezObr/is6WxVzRSRO4AZQCAwXlXjRORJnNOY74C7RGQokAnsBa73VTzG5MjMyuadOZt45ed46lSvzPjru3FOq/r+Duu0DRkyhCFDhvg7DJ8a2b0pYyauYP6mPfSJPvGZDXP6fPo+BVWdDkzP893jHp8fAh7yZQzGeNqccph7J61g+db9XNC+IeMubkfd4Mr+Dst4aWDbBtSuFsTExdssKfiIvWTHVAjqvibzX9PXERQovDqqE0M7NrJ3JpcxVYMCuaRzYz5buJV9hzMsofuANXNhyr0d+49y7fhFPDYlju4R9Zh5Tz+GdWpsCaGMGtm9KRlZ2Xyz/MQnsM3pszMFU26pKl8v287YqXFkZStPXdyOq89oZsmgjGvdsBZtGtZi2qod3Ng3wt/hlDuWFEy5lHIonYe/Xs3Mtbvo3qIuL4zoSPOQYH+HZYrJBR0a8vyM9ezYf5RGZfSOsdLKqo9MufPjmp2c//IcZscn88iQ1kwc3csSQjkzpL1zN/sP9p6FYmdJwZQbB9OOce8XK7j1k2U0rlON7+/syy1nRRIYYNVF5U1EaDCtG9Zi+uqd/g6l3LHqI1MuLN+6jzETV5C07whjBsRwxznRBAXaMU95dkH7BrwwM56dqUdpWNuqkIqL/WpMmZaVrbz5awLD355PVrby5a29uOe8lpYQKoDcKqTVVoVUnOyXY8qsnalHueq9BTw/Yz1D2jdk+pgz6dq8nr/DMiUkMqwGrRrUtCqkYmbVR6ZMmhH3Jw9MXsWxrGxeGNGRy7rYcwcV0QXtG/LiT/H8mZpGg9pV/R1OuWBnCqZMycjM5p9T4/jbx0tpVq863991JsO7NrGEUEENzr0Lyc4WioslBVNmbNt7hBHvzOeD3xO5oU8LvrqtNxGhdqvx/+YWAAAgAElEQVRpRRYdXoPY+laFVJys+siUCTPj/uS+L1eiwNtXd2FQu4ZFDmMqhiHtG/LKL/HsOpBG/VpWhXS67EzBlGrHsrIZN20toz9eSvOQYL6/80xLCOY4F3RogCr8YGcLxcKSgim1ElMOM+Lt+bw3dzPX9mrO5Nt60Sykur/DMqVMdHhNWtavwXS7NbVYWPWRKXVUlYmLt/HUtLVUChDevLILF3SwswNTsCHtG/LqLxvYfSCNcKtCOi12pmBKlZRD6dzy0VIe+no1nZrWYcY9Z1lCMEW6oH1DpwrJ2kI6bXamYEqNWet28cDkVRxIy+TRC1pzY58IAqzdIuOFmPo1iQmvwferd3Jd7xb+DqdMs6Rg/C7tWBZPTVvLpwu30qpBTT69uSexDWr6OyxTxgxp35DXZlkV0umy6iPjVxt2HWTYG7/z6cKtjD4rkil39LGEYE7JELcK6cc4q0I6HZYUjF+oKl8u2cbQN34n5VA6H97Yg4eHtKZKpUB/h2bKqJb1axAZFsxPa3f5O5QyzaqPTIk7lJ7JY9+u4Zvl2+kVGcIrozrZQ0fmtIkIZ8eG8/H8LRzJyKR6Zdu9nQo7UzAlKm5HKkNfn8uUFdu559yWfHLzGZYQTLHp1zKMjKxsFmza4+9QyixLCqZEZGcr78/dzCVvzeNwRiaf3dKTMefG2FvRTLHqEVGPqkEB/LY+2d+hlFk+TQoiMkhE1otIgog8WEi54SKiItLNl/EY//gzNY1rxy/iqWlrOSsmjOl3nUnPyBB/h2XKoapBgfSKDGF2vCWFU+WzSjcRCQTeBM4DkoDFIvKdqq7NU64mcBew0FexGP+ZvnonD329mozMbJ65tD2juje1Zq6NT/WPDefX9XEkphymhbWie9J8eabQA0hQ1U2qmgFMBIblU+4p4DkgzYexmBJ2MO0Y/5i0kr9/uowWIdWZPuZMrujRzBKC8bl+LcMAmL1+t58jKZt8mRQaA9s8upPc73KJSGegqapO82EcpoQt2ryXwa/+j2+WJ3HXOdFMtvcemBLUIjSYFiHV+c2qkE6JL+/Zyu+QUHN7igQALwPXFzkikdHAaIBmzZoVU3imuKVnZvHSzHje/d8mmtatzpe39rJ3Jhu/6NcyjC+WbCPtWBZVg+zZl5PhyzOFJKCpR3cTYIdHd02gHTBbRBKBnsB3+V1sVtV3VbWbqnYLCwvzYcjmVK3dcYChr//OO3M2Map7M34Yc6YlBOM3/WPDSTuWzaLNe/0dSpnjyzOFxUCMiEQA24FRwJU5PVU1FQjN6RaR2cB9qrrEhzGZYpaVrbwzZyMv/xRPneqV+eD67pzdKtzfYZkKrmdkCJUrBTB7fTJntbQDyZPhs6SgqpkicgcwAwgExqtqnIg8CSxR1e98NW1TMrbuOcK9k1awZMs+hrRvwLiL21MvuLK/wzKGapUDOSOiHr/F7wba+DucMsWnz4Gr6nRgep7vHi+gbH9fxmKKj6oyack2npy6loAA4ZWRnRjWqZHdWWRKlX4twxj3/R9s23uEpvXsjX3esieazUlJOZTO6I+X8n9fraZDkzrMuPssLu7c2BKCKXX6xzrVmHYX0smxFqOM135eu4sHv7aX4JiyISosmMZ1qjF7fTJX92zu73DKDEsKpkiH0jN5+vs/+HyRvQTHlB0iQv/YML5Zvp2MzGwqV7KKEW/YUjKFmrVuF+e/9BsTF2/lb/YSHFPG9GsZxpGMLJYk2q2p3rIzBZOvlEPp/HPqWqau3EFMeA0m39qbrs3r+jssY05K7+hQggKF3+KT6R0dWvQAxpKCOZ6qMnlpEk9P/4Mj6Vncc25Lbu0faW9EM2VSjSqV6N6iHrPXJ/PQkNb+DqdMKLL6SESqi8hjIvJftztGRC70fWimpG3dc4Sr31/I/ZNXER1Wg+lj+jLm3BhLCKZM69cyjPW7DrIz9ai/QykTvLmm8AGQDvRyu5OAcT6LyJS4zKxs3p2zkfNf+Y2V21J56uJ2TPpbL6LD7dqBKftyb021F+94xZvqoyhVHSkiVwCo6lGxm9LLjbgdqfzfV6tYs/0A57auz1MXt6Vh7Wr+DsuYYtOyfg0a1KrKb/HJjOphDWoWxZukkCEi1XBbOBWRKJwzB1OGpR3L4tVfNvDunE3UrR7Em1d2YUj7BvYQmil3RIS+MaH8tHYX2dlqz9YUwZuk8ATwI9BURD4F+uBFc9em9JqXkMIj365hc8phLu/WhIeHtKZOdWuzyJRffaJDmLw0ibU7D9CucW1/h1OqFZoU3GqidcClOE1bCzBGVVNKIDZTzJIPpvP092v5dsUOmodU59Obz6CP3aZnKoDeUc52/ntCiiWFIhSaFFRVReRbVe0KfF9CMZlilpWtfLZoK8/9uI60Y1ncdU40fz872l4+YiqM+rWqEhUWzO8b9/C3flH+DqdU86b6aIGIdFfVxT6PxhS7NdtTeeSb1axMSqV3VAhPXdyOqLAa/g7LmBLXJzqUL5ckWZMXRfAmKZwN/E1EtgCHcaqQVFU7+DQyc1oOph3jxZnxfDQ/kXrBla15a1Ph9Y4K5aP5W1ixbT89IuytgAXxJikM9nkUptioKlNX7WTctLUkH0rnqjOacf/5rahdPcjfoRnjV70iQwgQ57qCJYWCFZkUVHWLiHQEznS/+p+qrvRtWOZUbEo+xONT4pibkEL7xrX577Xd6Ni0jr/DMqZUqF09iHaNazNvYwr3nNfS3+GUWkUmBREZA9wCfO1+9YmIvKuqr/s0MuO1tGNZvPVrAm//tokqlQJ4clhbrjqjOYF2P7Yxx+kVFcL7/9vMkYxMqle2pt/y481SuQk4Q1UPA4jIs8B8wJKCn6kqP63dxVPfr2Xb3qMM69SIRy5oTXjNqv4OzZhSqU9UKO/8tolFm/fmNn9hjudNUhAgy6M7y/3O+NHG5EP8c+pa5sQnExNeg89uPsOaBjamCN1b1KNyYADzNu6xpFAAb5LCB8BCEfnG7b4YeN93IZnCHErP5PVZGxg/dzNVKwXy2IVtuLZXc4IC7RY7Y4pSrXIgnZvV4fcEe/62IN5caH5JRGYDfXHOEG5Q1eW+DswcT1WZsmIH/5r+B7sPpjOiaxMeGNSKsJpV/B2aMWVK76hQXvklnn2HM6gbbM275OXNheaeQJyqLnO7a4rIGaq60OfRGQBWJe1n7HdxLNu6n/aNa/P2NV3p0szegmbMqegTHcLLP8OCTXsY3L6hv8MpdbypPvoP0MWj+3A+3xkfSD6YzvMz1vHl0iRCgivz3GUdGN61ibXyaMxp6Ni0DsGVA/l9Y4olhXx4daFZVTWnQ1WzRcTu5fKhjMxsPpyXyGu/bODosSxu7hvBnQNiqFXVHkAz5nQFBQbQI6Ie8xL2+DuUUsmbq5ObROQuEQly/8YAm7wZuYgMEpH1IpIgIg/m0/9WEVktIitEZK6ItDnZGShPcm4xHfTKHJ6e/gddW9Rlxj1n8cgFbSwhGFOMekeFsinlsL2iMx/eJIVbgd7AdpxXcZ4BjC5qIBEJBN7EaSajDXBFPjv9z1S1vap2Ap4DXjqJ2MuVtTsOcNV7C7nloyUgMP76bky4oYc1XmeMD/SODgGws4V8eHP30W5g1CmMuweQoKqbAERkIjAMWOsx7gMe5YNx3+5Wkew+mMaLM+KZtHQbtasFMfaiNlzV024xNcaXWjeoRb3gyvy+MYXLujbxdzilijd3Hz0HjAOO4ryBrSNwt6p+UsSgjYFtHt05Zxl5x387cC9QGTingBhG456dNGtWPt6xejQji/G/b+atXxPIyMrmxj4R3HVOjDVcZ0wJCAgQekWGMC9hD6pqrQd78OZw9Hz3iP5CnB17S+B+L4bLbymfcCagqm+qahTwf8Cj+Y1IVd9V1W6q2i0sLMyLSZdemVnZfL5oK/2e/5XnZ6ynd3QoM+/px2MXtrGEYEwJ6h0dwp8H0ticctjfoZQq3txFlLOnGgJ8rqp7vcyqSUBTj+4mwI5Cyk/EudW1XFJVZsTt4rkZ69iUfJiuzevy5lVd6N7CmvA1xh9yX9G5cQ+Rdu0ulzdJYaqIrMOpPvq7iIQBaV4MtxiIEZEInIvUo4ArPQuISIyqbnA7LwA2UA4t2ryXZ374g+Vb9xMVFsy713TlvDb17ZTVGD9qEVKdRrWrMi8hhWt6Nvd3OKWGNxeaH3RbRj2gqlkicgTngnFRw2WKyB3ADCAQGK+qcSLyJLBEVb8D7hCRc4FjwD7gutOZmdImftdBnv1hHb+s2039WlV49rL2XNalCZXsIrIxfici9IwK4bf1yXZdwYNXD6Gp6j6Pz4dxnmr2ZrjpwPQ83z3u8XmMd2GWLTtTj/LSzHi+WpZEcJVKPDAolht6R1CtcqC/QzPGeOjRoh5fL9vOxuTDRIdbFRJ4mRSMd1KPHuOt2QlM+D0RVbipbwR/7x9tjW4ZU0p1d1/LuThxryUFlyWFYnAkI5MJ8xJ557dNHEg7xiWdGnPv+S1pUre6v0MzxhQiMjSY0BqVWbx5L1f0KB+3u5+uU0oKItJKVdcVdzBlTdqxLD5buJW3ZieQciiDc1qFc9/5sbRpVMvfoRljvCAidGtej0WJe/0dSqlxqmcKM4EKm1aPZWXz5ZIkXp+1gZ2pafSOCuGda2Lp2tyaszamrOkeUY8f4/5kZ+pRGtau5u9w/K7ApCAirxXUC6jjm3BKt8ysbL5Zvp3XZyWwde8RujSrw4sjOtprMI0pw3q0yLmusI+hHS0pFHamcAPwDyA9n35X+Cac0ikzK5spK3bw+qwNJO45QrvGtRh/fTfOjg2329iMKeNaN6xJcOVAFm/ey9COjfwdjt8VlhQWA2tUdV7eHiIy1mcRlSJZ2cp3K7fz+i8JbEo5TJuGtezBM2PKmUqBAXRpXpfFdl0BKDwpDKeAJ5dVNcI34ZQOmVnZTFu1k9dnbWBj8mFaNajJ21d35fw29e2tZ8aUQ91b1OPln+NJPXKswrdBVlhSqKGqFSp15lQTvfFrAptTDhNbvyb/uaoLA9s2sGRgTDnWvUU9VGHJlr0MaF3f3+H4VWFJ4Vvc9zCLyFeqelnJhFTyjmVl882y7bzxq3MBuXXDWrx9dRfOb2PJwJiKoHOzOgQFCosSLSkUlhQ894aRvg7EH9KOZTF5aRLvzNnItr1HadfYrhkYUxFVDQqkfePaLN5coSpH8lVYUtACPpd5B9OO8enCrbw/dzPJB9Pp2LQO/xza1u4mMqYC6x5Rj/FzN5N2LIuqQRW3nbLCkkJHETmAc8ZQzf2M262qWuYe291zKJ0Pfk/ko/mJHEjLpG90KK+O7ESvqBBLBsZUcD1a1OOd3zaxfOt+ekWF+DscvykwKahquUqVny7cwlPT1pKemc3ANg24rX8UHZtWyGfwjDH56Na8HiKwJHGvJYWKICqsBhe0b8Rt/SOJDq/p73CMMaVM7epBxNavWeHbQaowSaFnZAg9Iytu9jfGFK1bi7p8s2w7mVnZFfZlWBVzro0xJh/dW9TjcEYWf+w86O9Q/MaSgjHGuHq4L92pyFVIlhSMMcbVsHY1mtStVqGfV7CkYIwxHnq0qMfixL2olqvHs7xmScEYYzx0j6jHnsMZbEo57O9Q/MKSgjHGeOie89KdClqFZEnBGGM8RIUFExJcucJebLakYIwxHkSEbi0q7kt3fJoURGSQiKwXkQQReTCf/veKyFoRWSUiv4hIc1/GY4wx3ugZGcK2vUdJ2nfE36GUOJ8lBREJBN4EBgNtgCtEpE2eYsuBbqraAZgMPOereIwxxls5bR/N37jHz5GUPF+eKfQAElR1k6pmABOBYZ4FVPVXVc1JxQuAJj6MxxhjvNIyvCYhwZUtKRSzxsA2j+4k97uC3AT8kF8PERktIktEZElycnIxhmiMMScKCBB6RoYwf9OeCve8gi+TQn4vKMh36YrI1UA34Pn8+qvqu6raTVW7hYWFFWOIxhiTv15RIexMTSNxT8W6ruDLpJAENPXobgLsyFtIRM4FHgGGqmq6D+MxxhivVdTrCr5MCouBGBGJEJHKwCjgO88CItIZeAcnIez2YSzGGHNSIkODqV+rCvM2pvg7lBLls6SgqpnAHcAM4A9gkqrGiciTIjLULfY8UAP4UkRWiMh3BYzOGGNKlIjQKzKEBRXsuoJPX7KjqtOB6Xm+e9zj87m+nL4xxpyO3lGhfLtiBxt2H6Jl/YrxxkZ7otkYYwpQEa8rWFIwxpgCNK1XnSZ1q1Wo6wqWFIwxphDOdYW9ZGdXjOsKlhSMMaYQvaNDSD16jLU7D/g7lBJhScEYYwrRKzIUgAWbKsZ1BUsKxhhTiAa1qxIZGsy8CnKx2ZKCMcYUoWdUCIs27yUzK9vfoficJQVjjClC76gQDqVnsnp7qr9D8TlLCsYYU4Seke7zChXguoIlBWOMKUJojSrE1q9ZIR5is6RgjDFe6BUVwuLEvWRklu/rCpYUjDHGC72iQkg7ls2Kbfv9HYpPWVIwxhgv9IwIQaT8t4NkScEYY7xQu3oQbRvVKvftIFlSMMYYL/WOCmX51v0cTs/0dyg+Y0nBGGO81D82jIysbP63ofyeLVhSMMYYL3VvUY9aVSvx8x+7/B2Kz1hSMMYYLwUFBtA/Npxf1+0mq5w2pW1JwRhjTsKA1uHsOZxRbm9NtaRgjDEnoX/LcAIDhF/KaRWSJQVjjDkJtasH0b1FXX75Y7e/Q/EJSwrGGHOSzm1dn/W7DrJt7xF/h1LsLCkYY8xJGtC6PkC5rEKypGCMMScpIjSYyLBgfi6HVUg+TQoiMkhE1otIgog8mE//s0RkmYhkishwX8ZijDHF6bzW9Vm4eQ8H0475O5Ri5bOkICKBwJvAYKANcIWItMlTbCtwPfCZr+IwxhhfGNC6PseylDnx5evpZl+eKfQAElR1k6pmABOBYZ4FVDVRVVcB5buBcmNMudOlWR3qVA8qd9cVfJkUGgPbPLqT3O9OmoiMFpElIrIkOTm5WIIzxpjTUSkwgLNjw/l1ffl6utmXSUHy+e6Ulpyqvquq3VS1W1hY2GmGZYwxxWNA63D2HTnGsq37/B1KsfFlUkgCmnp0NwF2+HB6xhhTos5qGUalAClXDeT5MiksBmJEJEJEKgOjgO98OD1jjClRtaoGcUZkPX5ea0mhSKqaCdwBzAD+ACapapyIPCkiQwFEpLuIJAEjgHdEJM5X8RhjjC8MaFWfjcmHSUw57O9QioVPn1NQ1emq2lJVo1T1afe7x1X1O/fzYlVtoqrBqhqiqm19GY8xxhS3c92nm8tLFZI90WyMMaehWUh1WtavUW4ayLOkYIwxp2lA6/osStxLyqF0f4dy2iwpGGPMabqsS2OyspUvFm8runApZ0nBGGNOU3R4TXpHhfDpgi1kZpXtBhosKRhjTDG4tlcLdqSm8cu6sn1twZKCMcYUg3Nbh9OodlU+mp/o71BOiyUFY4wpBpUCA7iqZ3N+T9hDwu5D/g7nlFlSMMaYYjKye1MqBwbwyYIt/g7llFlSMMaYYhJaowoXdGjI5KVJHErP9Hc4p8SSgjHGFKNrejXnUHom3yzf7u9QToklBWOMKUadm9ahfePafDQvEdWy954FSwrGGFOMRIRrejVnw+5DLNi019/hnDRLCsYYU8yGdmxEnepBZfL2VEsKxhhTzKoGBTKyW1Nmrt3FztSj/g7npFhSMMYYH7i6Z3OyVfls4VZ/h3JSLCkYY4wPNK1XnXNiw/l80VbSjmX5OxyvWVIwxhgfueWsSFIOZfD4lDVl5k4kSwrGGOMjPSNDuPOcaCYtSeLzRWWjWW1LCsYY40N3n9uSs1qGMfa7OFZs2+/vcIpkScEYY3woMEB4bVQnwmtV4bZPlpb6t7NZUjDGGB+rU70yb1/dlb2HM7jzs+Wl+kU8lhSMMaYEtGtcm6cvac/8TXt4fsZ6f4dTIEsKxhhTQoZ3bcLVPZvxzpxNTF+909/h5MuSgjHGlKDHL2xL52Z1uP/LlSTsPujvcE7g06QgIoNEZL2IJIjIg/n0ryIiX7j9F4pIC1/GY4wx/la5UgD/uaorXZrXJSiw9B2X+ywiEQkE3gQGA22AK0SkTZ5iNwH7VDUaeBl41lfxGGNMadGgdlU+vukMmocE+zuUE/gyTfUAElR1k6pmABOBYXnKDAM+dD9PBgaIiPgwJmOMMYXwZVJoDHg+wpfkfpdvGVXNBFKBkLwjEpHRIrJERJYkJyf7KFxjjDG+TAr5HfHnbfzDmzKo6ruq2k1Vu4WFhRVLcMYYY07ky6SQBDT16G4C7CiojIhUAmoDZe9VRcYYU074MiksBmJEJEJEKgOjgO/ylPkOuM79PByYpWWlKUFjjCmHKvlqxKqaKSJ3ADOAQGC8qsaJyJPAElX9Dngf+FhEEnDOEEb5Kh5jjDFF81lSAFDV6cD0PN897vE5DRjhyxiMMcZ4r/Q9OWGMMcZvpKxV4YtIMrDlFAcPBVKKMZyywOa5YrB5rhhOZ56bq2qRt2+WuaRwOkRkiap283ccJcnmuWKwea4YSmKerfrIGGNMLksKxhhjclW0pPCuvwPwA5vnisHmuWLw+TxXqGsKxhhjClfRzhSMMcYUwpKCMcaYXOUyKXjxxrfrRSRZRFa4fzf7I87iVNQ8u2UuF5G1IhInIp+VdIzFzYv1/LLHOo4Xkf3+iLM4eTHPzUTkVxFZLiKrRGSIP+IsLl7Mb3MR+cWd19ki0sQfcRYnERkvIrtFZE0B/UVEXnOXySoR6VKsAahqufrDaWdpIxAJVAZWAm3ylLkeeMPfsZbwPMcAy4G6bne4v+P29TznKX8nTvtbfo/dx+v5XeA293MbINHfcft4fr8ErnM/nwN87O+4i2G+zwK6AGsK6D8E+AHn1QM9gYXFOf3yeKbgzRvfyhtv5vkW4E1V3QegqrtLOMbidrLr+Qrg8xKJzHe8mWcFarmfa3Nic/VliTfz2wb4xf38az79yxxVnUPhrxAYBnykjgVAHRFpWFzTL49JwZs3vgFc5p56TRaRpvn0L0u8meeWQEsR+V1EFojIoBKLzje8Xc+ISHMgAphVAnH5kjfzPBa4WkSScBqjvLNkQvMJb+Z3JXCZ+/kSoKaInPD2xnLG623/VJTHpODN29ymAi1UtQPwM3+9J7qs8maeK+FUIfXHOWp+T0Tq+DguX/LqrX2uUcBkVc3yYTwlwZt5vgKYoKpNcKoZPhaRsvo792Z+7wP6ichyoB+wHcj0dWB+djLb/kkrqxtLYYp845uq7lHVdLfzv0DXEorNV7x9y90UVT2mqpuB9ThJoqzyZp5zjKLsVx2Bd/N8EzAJQFXnA1VxGlEri7z5Le9Q1UtVtTPwiPtdasmF6Bcns+2ftPKYFIp841ue+rehwB8lGJ8vePOWu2+BswFEJBSnOmlTiUZZvLyZZ0QkFqgLzC/h+HzBm3neCgwAEJHWOEkhuUSjLD7e/JZDPc6EHgLGl3CM/vAdcK17F1JPIFVVdxbXyH36kh1/UO/e+HaXiAzFOc3ci3M3Upnl5TzPAM4XkbVAFnC/qu7xX9Snx8t5Bqc6ZaK6t22UZV7O8z+A/4rIPThVCteX1Xn3cn77A8+IiAJzgNv9FnAxEZHPceYr1L029AQQBKCqb+NcKxoCJABHgBuKdfpldHsxxhjjA+Wx+sgYY8wpsqRgjDEmlyUFY4wxuSwpGGOMyWVJwRhjTC5LChWQiLQQkSu9LPu52xzIPSLSym1xdLmIRJ1mDInu8xKnRUSqiMjPblwjT3d8RUxruojUcf/+7kX545aziHQTkddOYnpnui3arhCRxiIy+VRjLw4iMkFEhhdR5noRaXSK4z+t5WWKhyWFiqkFUGRSEJEGQG9V7aCqLwMX4zwV3VlVN/o4Rm91BoJUtZOqfuHNACISeCoTUtUhqrofqAMUmRTIs5xVdYmq3nUSk7wKeMGdt+2qWugO2Vsi4svnk64HTikpcPrLyxQHfzcTa3/F8wdcC6zCaSDsY/e7CcBwjzKH3P8LgFRgBXAPzlOvHwCrcZrXPtsttwo46pZ7AvgTp22ZX/NM+zbgOY/u64HX3c/fAkuBOGC0R5lEnOYXWuDRRDBOWzZj3c9RwI/u8P8DWuWZbjjOAzw58xKF8zTvcndexgNVPKb3ODAXGJVnPBOA/+C0srkJpw2d8ThPuk/IJ+aJHsvleZy2aJ4H1rjTHVnAcu4PTHP71fBY5quAy/LEdDPOg5WbgU89lxNQHacpi1XAF8BCoJvnOnY/D8+J353Hl9x5fBEIdudxsbu8huWzTQnwBrAW+B7noanhbr+uwG/uupkBNHSndwinCZUVQLX8yrnDR+O0O7YSWOauu8KWVz2cbWmVW66D+/1Ydz5mu+vuLn//Fsv6n98DsL9iWInQ1v0hhrrd9dz/E8g/KeT+2NzufwAfuJ9b4TSVUJUTd9hjgfvymX4YThPHOd0/AH3zxFINZ6cZ4nYnUnRS+AWIcT+fAczKZ9qeO46qOK1HtnS7PwLu9pjeAwUsvwk4O3rBaZb4ANAe50x6KdCpiJgvA37Ceeq2vrv8GuaznD1jfRZ4xaNf3QLiytkJ507TXUbvuJ/b4TyZ701SmAYEut3/Aq52P9cB4oHgPNO/1GO+GgH73XEGAfOAMLfcSNx3VeDsnHNiKazcQuASj/VWvYjl9TrwhPv5HGCFxzY5D6jirps9OGeOfv9dltW/ctfMRQV1Dk4roCkAqlpYW+z56Yvzo0NV14nIFpy2kQ54M7CqJovIJrcdlg1ALPC72/suEbnE/dwUpxG+IpvXEJEaQG/gS5HcRiGrFDFYLLBZVePd7g9xmj14xe0urHppqqqqiKwGdqnqajeOOJwd8opChu0LfK5OK6y7ROQ3oDuFL79zcdryAUDd91x4qS/wqjvcGhFZ5eVwX+pfLcWeDwwVkfvc7qpAM45vB+ws/pqvHSKS0/R4LE4y+sldNz83QsQAAAMASURBVIFAfm3v5FtORGoCjVX1G3ce0gA81nNB83yZW36WiISISG233/fqNHCZLiK7cRJzUlELw+TPkkL5IOTfdG4m7nUjcX5xlQsZ/nR9AVwOrAO+cXew/XF2fr1U9YiIzMbZ+eQboyunfwCwX1U7nUQMRc3H4UL65bSam+3xOae7qN/JqSy/gtaZt8MWxHOceZe15/wLTpXV+iKmlV+MAsSpaq8ihs23nIjUKqB8UeMqKDbP9ZWF7ddOi11oLh9+AS7PebmIiNRzv0/kr2bBh+E2qgUcBGp6DD8H56ImItIS54ixqJ1FXl/jXIi+gr+OyGsD+9yE0Arn1YF57QLC3SO/KsCFAKp6ANgsIiPcuEREOhYRwzqghYhEu93X4NRnF7f8lt9IEQkUkTCcI+xF+ZTzNBO4I6dDROqexPTn4iRgRKQNTlVXjl0i0tptOfSS/AZ2zQDudA8WEJHO+ZSZA4xy56shbiu7ONtGmIj0cocNEpG2bj/Pec63nLtu/7+9O8ZJIIjCOP5/UWJCZ2fnJez1CPbGGGPlCbQ0oonGnnAFKxoLGzUUNlSwiCewsPAAiiZj8d4uRBEhkhjI9ys3y7IzITPL92Dmycw24/iSmZUZ3V+Dn9EN4CWuI1OmSWEOpJS6wCnQMLM2XlAE3yti3cyaeCafPylmwIeZtWM1zSqwENHJJb6y5hsTiPjjEVhNKTXj8DWwGPFGBS8Qfn3dO3CMZ8xX+MCe2wL2ok1dftlqMWKIXTxy6uBP+bVJ2jGO5KvL3pvZg5ldAHX6Rf5bvHbxzPd+HnQCLMc12vQH3HFU8cE2Aw7iffI9BA7xfrxleKSTq+APCZn5BvGVIefU8TiwgxfiG9H+Hl5bOI97b+FRH3jtomZmLTwu+um8bTxazPCawAqj++sIWIvzz4CdEW2TP9AqqSIzJn5SW0opvcb/RW7w4nrvn29N5oCyN5HZUwbuzKyEZ+37mhBkWvRNQURECqopiIhIQZOCiIgUNCmIiEhBk4KIiBQ0KYiISOET6EZogWZJmN4AAAAASUVORK5CYII=\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7f358ad7c2e8>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "databasefile = 'databases/MITOS_WSI_CCMCT_HEAEL.sqlite'\n",
    "\n",
    "resfile = 'results/RetinaNet-HEAEL-export.pth-CCMCT_HEAEL-val-inference_results_boxes.p.bz2'\n",
    "thr, F1scores, thrs = optimize_threshold(databasefile=databasefile, resfile=resfile)\n",
    "\n",
    "\n",
    "import matplotlib.pyplot as plt \n",
    "plt.plot(thrs, F1scores)\n",
    "plt.xlabel('cutoff value for mitotic figure detection')\n",
    "plt.ylabel('F1 score')\n",
    "plt.title('Optimization of detection threshold (trainval set)')\n",
    "plt.text(0.7, 0.5, 'max: %.3f @ %.2f' % (np.max(np.array(F1scores)),np.arange(0.5,0.99,0.01)[np.argmax(F1scores)]))\n",
    "plt.plot([0.5, 1.0], np.max(np.array(F1scores))*np.array([1,1]),'r--')\n",
    "\n",
    "plt.savefig('results/figures/Opt_F1_CCMCT_HEAEL_RetinaNet.pdf')\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "1. HEAEL on HEAEL"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Calculating F1 for test set of 11 files\n",
      "Overall: \n",
      "TP: 15991 FP: 14452 FN:  5580 F1: 0.6148729188295459\n",
      "F1 score:  0.615\n"
     ]
    }
   ],
   "source": [
    "databasefile = 'databases/MITOS_WSI_CCMCT_HEAEL.sqlite'\n",
    "resfile = 'results/RetinaNet-HEAEL-export.pth-CCMCT_HEAEL-inference_results_boxes.p.bz2'\n",
    "\n",
    "F1, individ = calculate_F1(databasefile=databasefile, resfile=resfile, det_thres=0.83)\n",
    "\n",
    "print('F1 score: ', np.round(F1,3))\n",
    "\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "2. HEAEL on ODAEL"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Calculating F1 for test set of 11 files\n",
      "Overall: \n",
      "TP: 16551 FP: 13902 FN:  5925 F1: 0.6254038428838633\n",
      "F1 score:  0.625\n"
     ]
    }
   ],
   "source": [
    "databasefile = 'databases/MITOS_WSI_CCMCT_ODAEL.sqlite'\n",
    "resfile = 'results/RetinaNet-HEAEL-export.pth-CCMCT_HEAEL-inference_results_boxes.p.bz2'\n",
    "\n",
    "F1, individ = calculate_F1(databasefile=databasefile, resfile=resfile, det_thres=0.83)\n",
    "\n",
    "print('F1 score: ', np.round(F1,3))\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "3. HEAEL on MEL"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Calculating F1 for test set of 11 files\n",
      "Overall: \n",
      "TP: 15978 FP: 14465 FN:  5573 F1: 0.6146093780051545\n",
      "F1 score:  0.615\n"
     ]
    }
   ],
   "source": [
    "databasefile = 'databases/MITOS_WSI_CCMCT_MEL.sqlite'\n",
    "resfile = 'results/RetinaNet-HEAEL-export.pth-CCMCT_HEAEL-inference_results_boxes.p.bz2'\n",
    "\n",
    "F1, individ = calculate_F1(databasefile=databasefile, resfile=resfile, det_thres=0.83)\n",
    "\n",
    "print('F1 score: ', np.round(F1,3))\n",
    "\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# RetinaNet + 2nd stage (ResNet-18) classifier"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## ODAEL dataset variant"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Find threshold (on train+val set)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Optimizing threshold for validation set of 21 files: \n",
      "Best threshold: F1= 0.8080314505919197 Threshold= 0.5400000000000003\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYUAAAEWCAYAAACJ0YulAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4xLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvAOZPmwAAIABJREFUeJzt3Xd8FWX2+PHPSSEJJCRAEloSAoTekSLFwtpYVrEhghUsuK666tr97bp8rbuubVdx7WIFFRsgawELilKl99BDKAklQEICSc7vj5nES+oFcnNvkvN+vfLK3DvPzJwpd87M80wRVcUYY4wBCPJ3AMYYYwKHJQVjjDHFLCkYY4wpZknBGGNMMUsKxhhjillSMMYYU8ySgo+ISJKIHBKR4BMc/pCItAmkmE5iuk1FZLaIHBSRp70oP0ZEfqqO2CqJ4zQRWVtN06q2eRYRFZGUExgu2R02pJz+40Xk3QqGDxORVSLS7HinXcE4rxSRr6tqfBVM53sRucHH0wgTkTUiEu/L6VTGkoLL/VEuF5EcEdkpIv8VkZjjGH6ziJxd9FlVt6pqpKoWnEg87rAbT2RYX8V0EsYBmUBDVb2rKkcsIhNF5NEqGtcxO0tV/VFVO1TFuEtMp8Kday02Dpitqjuhatadqr6nqudWSXTVrGSiUdU84A3gPv9FZUkBABG5C/gncA8QDZwKtAK+EZF6/oytlmgFrFK7U7JKiKMm/nZvAt7xtnAdTJoA7wPXikiY3yJQ1Tr9BzQEDgEjS3wfCewGrnM/jwemAB8AB4FfgR5uv3eAQuCwO657gWRAgRC3zPfAo8DPbplpQBPgPeAAsABI9pi+AilAC7d80V+Os9oUoC3wLbAH50j8PSDmOGJqAUwF9gKpwI0e0x8PfAi87c7vSqBPBctxoDsPWe7/ge73E4GjwBE3jrPLGLaJG8cBYD7wCPCTR/+OwDdunGuL1hXOkafnuKd5zNfHQAawCfizx7iCgQeBDe58LQISgdnussl2x3U5cCaQ5jFsJ3c97neXx3CPfhOBCcAX7njnAW3LWVZb3WkVrdMBwBjgJ+ApYJ8b9+89hvkeeAyY467TFJwDmNeBHcB2nO0r2C2fAvzgro9M4IMS29YfgfXutCYA4vYLAv4KbMHZ/t8Got1+yRy7/bR2p3HQXT8vAO+WM89Jbtwhlay7zThHysuAPCAEuN9jfa0CLvYY7xiO3VbKnDcgzF1vXT3KxrkxxQONgOk428w+tzuhxPK/oZx56wcsxNl+dwHPePQ7Fec3vx9YCpzpfv8YUADkuvP/gscw64Ez/LZP9NeEA+UPGArkF22sJfq9BUxyu8e7G/EIIBS4G+eHG+qxMZ/tMWzJH9D3ODvetjg/5lXAOuBsd8N/G3izxMadUkZM73nElAKc427wcTg7tuc8ylYW0w/Ai0A40NP9QZzlMb+5wDCcHekTwNxylmFj94d0tTsvo93PTdz+E4FHK1gHk3ESUAOgK84O7ie3XwNgGzDWHXdvnJ1cl7LGjbNTWwQ8BNQD2gAbgfPc/vcAy4EOODuLHh5xHrPM8UgK7jpPxUko9YDf4eykOnjEsRdnBxHirqfJ5czvMevB/W4MzvZ1o7u8bwbS+W1n/T1OMunijj8U+Ax42V1G8TgJ9Sa3/CTg/7nLIxwYXGLbmg7E4OysM4Chbr/r3Plsg3Ng9AnwTjnbzy/AMzjb3+nu8igvKfwBWFniu1LbBc42uwQnUUe4312Gk+iDcJJ1NtDcY7mVTArlzdsbwGMeZW8BvnS7mwCXAvWBKOAj4DOPst9TflL4Bbja7Y4ETnW7W+IcsA1zYz/H/RxX0ThxDpD+XNa0quOvJp6CVrVYIFNV88vot8PtX2SRqk5R1aM4P4ZwnCMBb72pqhtUNQv4H7BBVWe60/4I6FXRwCJyH85R83UAqpqqqt+oap6qZrgxneFNICKSCAwG7lPVXFVdAryGs2Mv8pOqzlCnDeIdnB1oWf4ArFfVd1Q1X1UnAWuAC7yIIxjnx/iQqmar6gqcZFzkfGCzqr7pjvtXnLOAEeWMsi/Oj+5hVT2iTrvMq8Aot/8NwF9Vda06lqrqnsrixFnPkcA/3PF+i7PzGe1R5hNVne+uz/dwEu3x2KKqr7rL+y2gOdDUo/9EVV3pjr8x8HvgDne57Qae9ZjPozjVdi3c9VuyEfsfqrpfVbcC33nEeiXOke5GVT0EPACMKlmVIyJJOMv6b+72Nxvn7Lc8MThJwxv/UdVtqnoYQFU/UtV0VS1U1Q9wjqT7VTB8efP2Pseuryvc71DVPar6sarmqOpBnCN5r35LOMs6RURiVfWQqs51v78KmOH+hgpV9RucM4phlYzvIM7y8gtLCs5RZ2w59ZfN3f5FthV1qGohkIZzBOOtXR7dh8v4HFnegCLye+B24KKiH4uIxIvIZBHZLiIHgHc5NolVpAWw1/0BFNmCc3RTZKdHdw4QXs5yauEO66nkuMoTh3Pku83jO89xtQL6i8j+oj+cHVd5V7C0AlqUKP8gv+1cE3GqIo5XC2Cbu94946xoeZW7PstRPLyq5ridnuPwXEatcM4WdnjM58s4ZwzgVBcKMF9EVorIdeVNq0SsJdflFpz145mcisrtU9XsEmXLsw/nCNwbnvOJiFwjIks85rMrFW/n5c3bt0CEiPQXkVY4yeJTdxr1ReRlEdni/pZmAzFeXql3PdAeWCMiC0TkfPf7VsBlJbbFwTj7lYpE4VQ3+UVdbMgp6RecustLcKowABCRBjhHYg96lE306B8EJOCc4oNz2uoTItIB58jxElX1/ME84U63u6ruEZGLcOp1i1QUUzrQWESiPBJDEk7VzfFKx/kBeEoCvvRi2Ayc6rtEnLOLomGLbAN+UNVzyhm+5DxuAzapartyym/DqcJb4UVsntKBRBEJ8kgMSThVgMfrRLcVz+G24Wy3sWWd5apzhc+NACIyGJgpIrNVNbWSaZRcl0k462cXzvZeZAfQSEQaeCSGJMqft2VAGxEJ8Yi3vLLF37s771eBs4BfVLVARJbgJLzjoqqFIvIhztnCLmC6x7Z/F06VYn9V3SkiPYHF3kxHVdcDo919wiXAFBFpgrOO3lHVGyubzxI6AZVeuu0rdf5Mwa3K+T/geREZKiKhIpKMU52TxrFXS5wiIpe4R8t34Pwoi04Vd+HUw1YpEWkIfI5T5VGyCiAKp5Fqv4i0xKkv91RuTG5y+Rl4QkTCRaQ7zhHPeycQ5gygvYhcISIhInI50BmneqVCblXJJ8B492itM3CtR5Hp7rivdtdNqIj0FZFO5czjfOCAiNwnIhEiEiwiXUWkr9v/NeAREWnnXsXT3f0BlzUuT/Nw6rLvdWM4E6d6bHJl81iGDJyLAE54e1HVHcDXwNMi0lBEgkSkrYicASAil4lI0U58H84OyJtLkScBd4pIaxGJBB7HaaQ+JvGo6hacqpD/E5F6buIpt7pQVdMoXe3jzW+mgRt7hjtfY3HOFE7U+zjtEle63UWicM7W94tIY+Dv3o5QRK4SkTj3YKHoCL8A58z9AhE5z90Ow0XkTI/1Umr+3d9xY37br1S7Op8UAFT1SZwzgqdwriCYh5Plz1Ln2uEin+NsUEWNqpe47QvgHLX/1T1NvLsKw+uNcwTzjDg3nh0SkUNuv/9z+2fhXPXySYlhK4tpNE7jYTrOafTf3XrP4+LWyZ+Pc7S1B6fq4nxVzaxwwN/cinOKvxOn8fFNj3EfBM7FqStPd8v8E6dxE5yrbzq78/iZm2QuwKka2IRT/fcaTuM+OO0uH+LsUA+4w0e4/cYDb7njGlliHo8Aw3HOHjNxGuivUdU1HCe3augxYI47reNpl/J0DU6j9yqcbXIKv1VN9AXmudvKVOB2Vd3kxTjfwDkQmo2z/HKB28opewXQH6eB/e84F0tU5GWObbM6Zt2VNYCqrsI5av4FZyfaDecKrBOiqkXJvQVOu16R53C2g0ycHbI3Z7lFhgIr3WX9b2CU246zDbgQZ9+SgbNPuYff9rv/BkaIyD4R+Y/73RXAWyX2O9Wq6MoGUwkRGY9zZcpV/o7FmJpInGvvF+McbO3wdzyBxl0+S4HT3QsH/MLaFIwx1cI9+u3s7zgClbt8Ovo7Dqs+MsYYU8yqj4wxxhSzMwVjjDHFalybQmxsrCYnJ/s7DGOMqVEWLVqUqapxlZWrcUkhOTmZhQsX+jsMY4ypUUSkojvOi1n1kTHGmGKWFIwxxhSzpGCMMaaYJQVjjDHFLCkYY4wpZknBGGNMMUsKxhhjitW4+xROyplnlv5u5Ej4058gJweGlfGWvDFjnL/MTBhRxhsgb74ZLr8ctm2Dq68u3f+uu+CCC2DtWrjpptL9//pXOPtsWLIE7rijdP/HH4eBA+Hnn+HBB0v3f+456NkTZs6ERx8t3f/ll6FDB5g2DZ4u470d77wDiYnwwQfw3/+W7j9lCsTGwsSJzl9JM2ZA/frw4ovw4Yel+3//vfP/qaconDadQo/Hqmh4BDmfTQOBsCceI+T77wAoVKWwUDkS04i1EyaSd7SQxH89TNTihShQWKgUqnIgthn/u+9fhAQLZ730BHEbVhMaHERwkCACBSntyHn+RQAibr2ZoNRUCgsVEYioF0xwr17O8gO46ipISzs29gED4IknnO5LL4U9Jd7aedZZ8Le/Od2//z0cPnxs//PPh7vdJ5bbtle6fzVue0wv8WqPiAj4n/vk7EcegVmzju3fpAl8/LHT/cAD8Msvx/ZPSIB333W677jDWYae2reHV15xuseNg3Ul3sXUs+eJbXtF8+RDdSspGJ8pKFRy8vI5lJtPXn4BR/ILOZJfyE3/+JaDuUe54sfVnLnp2J1qbkgYYx7+GoDb5qQyaMveY/rvyyzk5pecH+O9y3fQe9exr/jdkZfFM984P7bw1Ew67z62/8bsdB583PmxP75sB232HvuGw20F2/jqrYW0bxrJqL05NMrLJyI0mJCg436plzG1hk8fiCciQ3FeJBEMvKaq/yjRPwnnNZMxbpn7VXVGRePs06eP2h3N/qOq7M0+Qtq+w6TtO8ymzEPM37yPhZv3knPEebFXVFgITaPDadYwnPiGYTQMDyWiXjD1Q4OJqBdMkPy201WP8RYJCw2mQb1g6tcLoUFYMOHujjo0OIjQ4CDCQ4OIqBdMRKjzd7RA2ZtzhH3ZR9iTfYTsvHwnKRU4iQmgfr1g6tcLJqJeCIeP5LNu1yHW7jrIup0H2ZSZTX7hb9Nv3KAecZFhNImsR5PIMJo1DKNzi4Z0bRFNm7hIgi1pmBpIRBapap9Ky/kqKbgvvF4HnIPzWssFwGj3TUpFZV4BFqvqf93XMM5Q1eSKxmtJoXqoKpsys1m+PYsNGdlszDjEhoxsNmdmc/josW91bN80klPbNOHUNk3o17oxsZFh5Yw1MB0tKGTr3hw2ZmSzIeMQW/fmsOdQHpmHjrDnUB47snLJc5NLRGgwnVs0pIubJDq3aEj7plHUC7HmORPYvE0Kvqw+6gekqupGN6DJOK+mW+VRRoGGbnc0zusWjR/kHi1g8db9LNy8l1+37mPxtv3sz3HeNBokkNCoPm3iGnBqm8YkNqpPQqMIEhrVJ6FxBA3DQ/0c/ckJDQ6ibVwkbeMiOYempfrnFxSyISObFduzWJGexYrtWXy8KI23f9niDi+0i4+iU/OGxQmjR0IMEfWCq3tWjDlpvjxTGAEMVdUb3M9XA/1V9VaPMs1x3pXbCOcF3Wer6qIyxjUOGAeQlJR0ypYtXj3XyVSgsFBZvG0/367ZxfxNe1m6LYsjBc7RcLv4SHonNaJXUgw9EmNoHduA8FDbwXkqLFQ278lmRfoBVqZnsXrHQValHyDzkPNq3dBgoXtCDP1aN3bOoJIbW5IwfhUI1UeXAeeVSAr9VPU2jzJ/cWN4WkQG4LzIu6uqFpY3Xqs+OnGqyrK0LKYvS+eLZTtIz8olOEjo1jKa/q0b079NY05p1ZjoiJp95O9PGQfzWLE9i3mb9jJv0x6Wp2WRX6iEhQRxapsmDOkQx5CO8bRq0sDfoZo6JhCqj9KARI/PCZSuHroeGAqgqr+ISDgQC/jtpdW1iaqyISObuRv38MvGPczbuIfMQ0cIDRZObxfH3ed14OzOTWt89U8giYsKY0jHeIZ0jAcg50g+Czbv44e1GXy/djfjp61i/LRVpMRHcnanppzTuSm9EmMIssZrEyB8eaYQgtPQfBawHaeh+QpVXelR5n/AB6o6UUQ6AbOAllpBUHamULGjBYXM27iXmat38c2qXWzf71w736xhOAPaNmFQSizndGpKdH1LBP6wOTOb79buZubqXczbuJf8QiU2MoxzOsdzXpdmDGwba43Wxif8Xn3kBjEMeA7nctM3VPUxEXkYWKiqU90rjl4FInEane9V1a8rGqclhdLyCwr5ecMePlu8nW9W7+Jgbj5hIUGc1i6W33VsysC2TWjVpD4idjQaSLIOH+X7tbv5ZtUuvluzm+wjBUSFhTCkYzwX92rJ6e3j7PJXU2UCIin4giWF36zdeZAPF25j6tJ0Mg7mERUewnldmnFu56ac1i7OGjZrkNyjBfy8IZOvVuzi61U72ZdzlBbR4VzWJ5GRfRNpGRPh7xBNDWdJoZZSVWavz+S1Hzfy4/pMQoOFIR2cI8shHePtKqFa4Eh+ITNX72LS/K38lJoJwIA2Tfh912ac16UZ8Q3D/RyhqYksKdQyefkFfL4kndd+3Mi6XYeIjwrj2oHJXNEviUYN6vk7POMj2/bm8NGiNL5Yls6GjGxE4JSkRlzSO4GLe7W0s0HjNUsKtcT+nCO8N28rE3/eTMbBPDo2i+LG09pwQY8W1iBZx6zfdZAZy3fyxfJ01u06REz9UEb3S+KaAa1oHm3VS6ZilhRquKyco7zw3Xrem7eVnCMFnN4+jnGntWFQShNrMK7jVJX5m/byxpxNfLNqFyLC0C7NuHpAK/q3bmzbhylTINynYE7A0YJC3p+3lWdnruPA4aNc1LMlN57ehk7NG1Y+sKkTRIT+bZrQv00Ttu3N4a2fN/Phwm18sXwH7ZtGcvWprbi4dwKRYfbzNsfPzhQChKry7ZrdPD5jNRsyshmU0oS//qGzJQPjlcNHCpi2NJ23525mxfYDREeEMnZQMmMHtrZ7Ugxg1Uc1RlEy+Pes9SxLy6J1bAMeHNaJszvFWzWAOW6qyq9b9/PSDxv4ZtUuIsNCuGZAK64f3JomNezptaZqWVIIcKrKrNVOMli+PYuERhHc9rsULumdQGiwNSCbk7d6xwEmfJfKF8t3EBYSxOh+Sdx4Whta2D0PdZIlhQBVdGbw3EwnGSQ1rs+tQ1K4uHdLSwbGJ1J3H+KlHzbw2eLtiMDFvVpy85kptI61h/LVJZYUAtD3a3fzzDfrWJaWRWLjCG4b0s6Sgak2aftyeHX2RiYv2EZ+oTKyTwJ/PqudXc5aR1hSCCDb9x9m/NSVfLNql1UTGb/LOJjHi9+n8u7cLYgI1w5oxZ/OTLGbIGs5SwoBIL+gkDfnbObZmesoVOWOs9tz3aDWdtOZCQjb9ubw3Mz1fLI4jQb1Qrh2YCtuGNzGkkMtZUnBz1alH+Cuj5ayescBftcxnv8b3oXExvX9HZYxpazbdZB/z1zPjBU7qB8azDUDk7nxtDY0tuRQq1hS8JP8gkJenr2R52auIzqiHo9e1IXzujSzy0tNwFu36yD/mbWeL5bvICI0mKsHtGLcaW3sUtZawpKCH2zKzOauD5fw69b9DOvWjEcv6mZHW6bGWb/rIM9/m8q0ZemEhzjJ4cbT2hAXZcmhJrOkUI1UlUnzt/HI9FWEBguPXNSV4T1a2NmBqdFSdx9iwnepfL5kO+GhwfzlnPaMGZhMiF0gUSNZUqgmew7lcd/Hy5m5eheDU2J56rIeNIu2592b2mNjxiEemb6K79Zm0LVlQ564uDvdEqL9HZY5TpYUqsF3a3Zzz5RlHMg9yn1DOzJ2YLK9gN3USqrKjOU7GT9tJXsO5TF2UGvuOa+DvdSpBrGnpPpQYaHy7Mx1PP9tKh2bRfHuDf3o2MweXGdqLxHhD92bM7hdLP/6ag2v/7SJXzbs4cUre5Nsd0bXKlY5eJwO5h5l3DuLeP7bVEb2SeCzWwZZQjB1RnREKI9e1I3Xr+3D9v2HOf/5n/hi2Q5/h2WqkCWF47BlTzaXvPgz363dzfgLOvPPS7vb6bOpk87q1JQZt59Gu6aR3PL+rzz0+Qpyjxb4OyxTBSwpeOm7tbsZ/sIcMg7l8c51/RgzqLVdXWTqtJYxEXwwbgA3DG7N279sYfgLP7Eq/YC/wzInyZJCJQoKlae/XsvYNxfQIiaCqbcMZmBKrL/DMiYg1AsJ4q/nd2bi2L7syznKRRPm8MrsDRQW1qwLWMxvLClUIPNQHte8MY/nv03lslMS+PRPA0lqYo+qMKakMzvE89UdpzOkYxyPz1jDFa/NZffBXH+HZU6AJYVyLN22n/P/8xMLN+/jyUu786/Lelj7gTEVaNygHi9ddQpPjujO0m1ZDH9+Dku37fd3WOY4WVIow7Sl6Yx8+RdCgoVP/jSQkX0T/R2SMTWCiDCyTyIf3zyQ4CDhspd/4eNFaf4OyxwHSwoeVJXnZq7jtkmL6dYyms9vGUSXFnbnpjHHq3OLhky7bTCnJDXiro+W8sj0VeQXFPo7LOMFSwqu3KMF3DZpMc/NXM8lvVvy3o397emQxpyExg3q8fb1/RgzMJnXf9rEuHcWkZ2X7++wTCUsKQDZefmMfXMBXyzfwX1DO/L0ZT0IC7H2A2NOVmhwEOOHd+HRi7ryw7oMRr78C7sOWAN0IKvzSSHr8FGufn0e8zfv5ZmRPbj5zLZ2/4ExVeyqU1vx2rV92JyZzUUT5tj9DAGsTieFPYfyGP3KXJZvz2LCFb25uFeCv0MyptYa0iGej/44EFW47KWfmb9pr79DMmWos0lhZ1Yul78yl42Zh3jt2r4M7drM3yEZU+t1btGQz24ZRLPocMa+OZ/FW/f5OyRTQp1MCpsysxnx0s/szMrlrbH9OKN9nL9DMqbOaBYdzvs3nkpsVBjXvjGfFduz/B2S8VDnksLK9Cwue+lnco4UMOnGU+nfpom/QzKmzmnaMJz3buhPVHgo17wxn3W7Dvo7JOOqU0lh/qa9jHplLvWCg/jwpgH29ihj/CihUX3eu6E/IUHCla/NI3X3IX+HZKhDSeG7Nbu5+vV5xEWF8dHNA0mJj/R3SMbUecmxDXj/xv7Fjc9L7LEYfufTpCAiQ0VkrYikisj9ZfR/VkSWuH/rRMRnW0R+odKpeUM+umkALWMifDUZY8xxSomPYsofBxAZHsIVr85l9roMf4dUp/nsHc0iEgysA84B0oAFwGhVXVVO+duAXqp6XUXjPZl3NBcWqr1D2ZgAtftALte8MZ8NGYd4emRPhvdo4e+QahVv39HsyzOFfkCqqm5U1SPAZODCCsqPBib5MB5LCMYEsPiG4Xxw0wB6JTbi9smL7UF6fuLLpNAS2ObxOc39rhQRaQW0Br4tp/84EVkoIgszMuzU0pjaKjoilLev78fAtk247+NlzEnN9HdIdY4vk0JZh+Xl1VWNAqaoapkveVXVV1S1j6r2iYuzewqMqc3CQ4P571Wn0DYukj++s4g1O+2RGNXJl0khDfB8EUECkF5O2VH4uOrIGFNzNAwP5c2xfakfFszYNxewM8seolddfJkUFgDtRKS1iNTD2fFPLVlIRDoAjYBffBiLMaaGaRETwRtj+nLg8FHGTlzAwdyj/g6pTvBZUlDVfOBW4CtgNfChqq4UkYdFZLhH0dHAZPXVZVDGmBqrS4toXrzqFNbtOsif3vuVo/aiHp/z2SWpvnIyl6QaY2qmDxdu494py7ikd0uevqyHPd7+BHh7SWpIdQRjjDEnY2SfRHbsz+XZmetoER3B3ed18HdItZYlBWNMjfDns1LYkXWYF75LpXlMOFf2b+XvkGolSwrGmBpBRHj0oq7sOpDL3z5bQVxkGOd2sfegVLU680A8Y0zNFxIcxIQre9MtIYZbJy3mZ7u5rcpZUjDG1Cj164UwcUxfkpvU54a3F9rb26qYJQVjTI3TqEE93r2+P7GRYYx5cwFrd9pLeqqKJQVjTI0U7769LTw0iKten8fmzGx/h1QrWFIwxtRYiY3r8+71/ckvKOS6iQvIyrG7nk+WJQVjTI3WrmkUL1/dh237crh10q/k213PJ8WSgjGmxuvXujGPXNiVH9dn8viMNf4Op0az+xSMMbXCqH5JrNl5kDfmbKJjsyhG9k2sfCBTip0pGGNqjb/+oROntYvl/322nIWb9/o7nBrJkoIxptYICQ7ihdG9aRkTwe2Tl5Cdl+/vkGocSwrGmFolun4oT4/sQXrWYZ780toXjpclBWNMrXNKq8ZcOyCZt37ZwgKrRjoulhSMMbXSPed1IKFRBPdNWUbu0TJf/27KYEnBGFMrNQgL4R+XdGdjZjbPzVzv73BqDEsKxphaa3C7WEb1TeSV2RtYlrbf3+HUCJYUjDG12oN/6ERcVBgPfLKcgsKa9fphf7CkYIyp1RqGh/LgsE6sTD/Ax4vS/B1OwLOkYIyp9Yb3aEHvpBie/Goth+zehQpZUjDG1HoiwkMXdCHzUB4Tvkv1dzgBzZKCMaZO6JkYwyW9WvL6j5vYtjfH3+EELEsKxpg6496hHQkOEh6fsdrfoQQsSwrGmDqjWXQ4N5/Zlv+t2MncjXv8HU5AqjQpiEh9EfmbiLzqfm4nIuf7PjRjjKl6N57WhhbR4Tw8bZVdoloGb84U3gTygAHu5zTgUZ9FZIwxPhRRL5j7ft+RVTvsEtWyeJMU2qrqk8BRAFU9DIhPozLGGB8a3qMFvZJi+NfXdolqSd4khSMiEgEogIi0xTlzMMaYGklE+Nv5nck4mMdL32/wdzgBxZuk8HfgSyBRRN4DZgH3+jQqY4zxsd5JjbiwZwte/XEjafvsEtUiFSYFERFgDXAJMAaYBPRR1e99HpkxxvjYvUM7AvDPL9f6OZLAUWFSUFUFPlPVPar6hapOV9XMaorNGGN8qmVMBDed3oZpS9NZtGWfv8MJCN5UH80Vkb4+j8QYY/yGZFrPAAAemElEQVTgpjPaEh8VxmNfrMI5Dq7bvEkKQ4BfRGSDiCwTkeUisszXgRljTHVoEBbC7We349et+/kp1SpCQrwo83ufR2GMMX404pQEXvg2lX/PXM/glFic5tS6qdIzBVXdAsQAF7h/Me53xhhTK4SFBHPzmW1ZuGUfv9Txx19485iL24H3gHj3710Ruc2bkYvIUBFZKyKpInJ/OWVGisgqEVkpIu8fT/DGGFNVRvZJJD4qjP/Mqtvvc/amTeF6oL+qPqSqDwGnAjdWNpCIBAMTcKqfOgOjRaRziTLtgAeAQaraBbjjOOM3xpgqER4azB/PaMvcjXuZV4fPFrxJCgIUeHwuwLvHXPQDUlV1o6oeASYDF5YocyMwQVX3Aajqbi/Ga4wxPnFF/yRiI8N4/tu6+yIebx+IN09ExovIeGAu8LoXw7UEtnl8TnO/89QeaC8ic0RkrogMLWtEIjJORBaKyMKMjAwvJm2MMcfPOVtow0+pmSzastff4fiFNw3NzwBjgb3APmCsqj7nxbjLOpsoeRFwCNAOOBMYDbwmIjFlxPCKqvZR1T5xcXFeTNoYY07MFf2TaNKgHv+ZVTfPFrxpaD4VWK+q/1HVfwOpItLfi3GnAYkenxOA9DLKfK6qR1V1E7AWJ0kYY4xf1K8XwnWDW/PDugxWpR/wdzjVzpvqo/8Chzw+Z7vfVWYB0E5EWotIPWAUMLVEmc9wbo5DRGJxqpM2ejFuY4zxmav6t6J+vWBe+7Hu7Y68amhWj3u/VbUQL256U9V84FbgK2A18KGqrhSRh0VkuFvsK2CPiKwCvgPuUdW62+xvjAkI0fVDubxvIlOXprMj67C/w6lW3iSFjSLyZxEJdf9ux8ujeVWdoartVbWtqj7mfveQqk51u1VV/6KqnVW1m6pOPvFZMcaYqnPdoNYoMHHOZn+HUq28SQp/BAYC23HaAPoD43wZlDHG+Fti4/oM69ac9+dt5WDuUX+HU228ufpot6qOUtV4VW2qqlfY/QTGmLrgxtNaczAvn8nzt1VeuJbw5uqjJ0WkoVt1NEtEMkXkquoIzhhj/Kl7QgyntmnMG3M2cbSg0N/hVAtvqo/OVdUDwPk41UftgXt8GpUxxgSIcae3YUdWLl8s2+HvUKqFN0kh1P0/DJikqnXzNj9jTJ10Zvt4UuIjeWX2xjrxEh5vksI0EVkD9AFmiUgckOvbsIwxJjAEBQljByWzascBlmzb7+9wfM6bhub7gQFAH1U9CuRQ+sF2xhhTaw3v0YKI0OA60eDszZkCqrpPVQvc7mxV3enbsIwxJnBEhYdyQY/mTFuWzqG8fH+H41NeJQVjjKnrRvVLIudIAVOXlHyEW+1iScEYY7zQKzGGDk2jmLxgq79D8akTSgoi0rGqAzHGmEAmIozql8iytCxWpmf5OxyfOdEzha+rNApjjKkBLu7VknohQbW6wbncp52KyH/K6wWUehGOMcbUdjH16zGsazM+W7KdB4d1IqJesL9DqnIVnSmMBVYAi0r8LQSO+D40Y4wJPKP6JXEwN58Zy2vnHc4VvRdhAbBCVX8u2cN9V7MxxtQ5/Vs3pk1sAyYv2MqlpyT4O5wqV9GZwghgSVk9VLW1b8IxxpjAJiKM7JvIgs372JSZ7e9wqlxFSSFSVXOqLRJjjKkhLurZEhH4dPF2f4dS5SpKCp8VdYjIx9UQizHG1AjNosMZ1DaWzxZvr3UPyasoKYhHdxtfB2KMMTXJRb1asnVvDou27PN3KFWqoqSg5XQbY0ydN7RrM8JDg/ikllUhVZQUeojIARE5CHR3uw+IyEEROVBdARpjTCCKDAvhvC7N+GLZDvLyC/wdTpUpNymoarCqNlTVKFUNcbuLPjesziCNMSYQXdyrJVmHj/Ldmtrz2np7IJ4xxpygwSmxxEaG8cmvtacKyZKCMcacoJDgIC7s2YLv1u5mX3bteNCDJQVjjDkJF/dqydECZXoteeyFJQVjjDkJXVo0pH3TSD79Nc3foVQJSwrGGHMSRISLerXk16372b7/sL/DOWmWFIwx5iSd27kZAN+u3uXnSE6eJQVjjDlJbeMakNykPjNX1/xLUy0pGGPMSRIRzurUlF827CE7L9/f4ZwUSwrGGFMFzuoYz5GCQn5KzfR3KCfFkoIxxlSBvq0bExUewqwa3q5gScEYY6pAaHAQZ7SP49s1GRQW1txniFpSMMackC+//JIOHTqQkpLCP/7xjzLLbN26lSFDhtCrVy+6d+/OjBkzivs98cQTpKSk0KFDB7766qvi75999lm6dOlC165dGT16NLm5uWWOu7CwkNdee43BgwfTo0cPzjnnHKZPn15uvHl5eVx++eWkpKTQv39/Nm/eXGa55ORkunXrRs+ePenTp0+p/k899RQiQmZm6WqiszrFk3koj2Xbs8qNI+Cpao36O+WUU9QY41/5+fnapk0b3bBhg+bl5Wn37t115cqVpcrdeOON+uKLL6qq6sqVK7VVq1bF3d27d9fc3FzduHGjtmnTRvPz8zUtLU2Tk5M1JydHVVUvu+wyffPNN0uNt7CwUEeNGqW333677ty5U1VV09LS9PLLL9fnnnuuzJgnTJigN910k6qqTpo0SUeOHFlmuVatWmlGRkaZ/bZu3arnnnuuJiUllVlm76E8bX3/dH3qqzVlDu9PwEL1Yh9rZwrG1AKbN2+mY8eO3HDDDXTt2pUrr7ySmTNnMmjQINq1a8f8+fMBmD9/PgMHDqRXr14MHDiQtWvXAvDMM89w3XXXAbB8+XK6du1KTk75b+OdP38+KSkptGnThnr16jFq1Cg+//zzUuVEhAMHnCftZ2Vl0aJFCwA+//xzRo0aRVhYGK1btyYlJaU4xvz8fA4fPkx+fj45OTnFw3h66623aNWqFc899xxNmzYFoGXLlrz//vtMnz6d7dtLP6Du888/59prrwVgxIgRzJo167jfmnbnnXfy5JNPIiJl9m/UoB59WjVmVg2+NNWnSUFEhorIWhFJFZH7y+g/RkQyRGSJ+3eDL+MxpjZLTU3l9ttvZ9myZaxZs4b333+fn376iaeeeorHH38cgI4dOzJ79mwWL17Mww8/zIMPPgjAHXfcQWpqKp9++iljx47l5Zdfpn79+ixcuJAbbij9s9y+fTuJiYnFnxMSEsrcEY8fP553332XhIQEhg0bxvPPP1/h8C1btuTuu+8mKSmJ5s2bEx0dzbnnnltqvG+//TYPPvggGRkZDBs2jIEDB3LPPffw0Ucfccstt/DBBx9UGHNISAjR0dHs2bOnVDkR4dxzz+WUU07hlVdeKf5+6tSptGzZkh49epS9AlxndYpn1Y4DpNfQu5t9lhREJBiYAPwe6AyMFpHOZRT9QFV7un+v+SoeY2q71q1b061bN4KCgujSpQtnnXUWIkK3bt2K68+zsrK47LLL6Nq1K3feeScrV64EICgoiIkTJ3L11VdzxhlnMGjQIAD69OnDa6+V/lmWdYRd1tHzpEmTGDNmDGlpacyYMYOrr76awsLCcofft28fn3/+OZs2bSI9PZ3s7GzefffdUmXz8/Np2LAhjz/+OOPGjePHH38kNTWVw4cP06FDBzZs2HDCMc+ZM4dff/2V//3vf0yYMIHZs2eTk5PDY489xsMPP1yqfElndYoHYFYNfceCL88U+gGpqrpRVY8Ak4ELfTg9Y+q0sLCw4u6goKDiz0FBQeTnOzdU/e1vf2PIkCGsWLGCadOmHdOIu379eiIjI0lPT690WgkJCWzbtq34c1paWpnVPK+//jojR44EYMCAAeTm5pKZmVnu8DNnzqR169bExcURGhrKJZdcws8//1xqvMHBwQCsWbOGoUOHEhwcXHxGsXv3buLj4yuMOT8/n6ysLBo3blyqXNF8xMfHc/HFFzN//nw2bNjApk2b6NGjB8nJyaSlpdG7d2927txZavi2cZG0alK/xj7ywpdJoSWwzeNzmvtdSZeKyDIRmSIiiWX0R0TGichCEVmYkZHhi1iNqROysrJo2dL5GU6cOPGY72+//XZmz57Nnj17mDJlSoXj6du3L+vXr2fTpk0cOXKEyZMnM3z48FLlkpKSmDVrFgCrV68mNzeXuLg4hg8fzuTJk8nLy2PTpk2sX7+efv36kZSUxNy5c8nJyUFVmTVrFp06dSozhoMHD9KhQwe+/vprCgsL+eabb8jNzeXpp5/m8ssvL1V++PDhvPXWWwBMmTKF3/3ud6XOFLKzszl48GBx99dff03Xrl3p1q0bu3fvZvPmzWzevJmEhAR+/fVXmjVrVmo6IsJZHZsyZ8Meco7UvLubfZkUymqJKXn+Ng1IVtXuwEzgrbJGpKqvqGofVe0TFxdXxWEaU3fce++9PPDAAwwaNIiCgt/eK3znnXfypz/9ifbt2/P6669z//33s3v37nLbFEJCQnjhhRc477zz6NSpEyNHjqRLly4APPTQQ0ydOhWAp59+mldffZUePXowevRoJk6ciIjQpUsXRo4cSefOnRk6dCgTJkwgODiY/v37M2LECHr37k23bt0oLCxk3LhxpaY/evRoHnroIR544AFefPFFBg8eTLt27Zg8eTK33HILHTt2LDXM9ddfz549e0hJSeGZZ54pvow2PT2dYcOGAbBr167iS1z79evHH/7wB4YOHXrcy3lIxziO5Bcyb+Pe4x7W3+R4W9+9HrHIAGC8qp7nfn4AQFWfKKd8MLBXVaMrGm+fPn104cKFVR2uMaYGKSws5NJLL6Vnz5785S9/ISoqioyMDD755BOuv/56QkJC/Bpf7tECej78NaP6JjF+eBe/xlJERBapaukbL0rw5ZnCAqCdiLQWkXrAKGCqZwERae7xcTiw2ofxGGNqiaCgIKZMmULjxo0577zz6N27N2PHjqVdu3Z+TwgA4aHB9G/dhNnra151t8+Wnqrmi8itwFdAMPCGqq4UkYdxbqKYCvxZRIYD+cBeYIyv4jHG1C7BwcHcdttt3Hbbbf4OpUynt4/jkemrSNuXQ0Kj+v4Ox2s+vU9BVWeoantVbauqj7nfPeQmBFT1AVXtoqo9VHWIqq7xZTzGGFNdzmgfC8DsdTXrqal2R7MxxvhA27hImkeHM3tdzapCsqRgjDE+ICKc3i6OORsyyS8o9Hc4XrOkYIwxPnJ6+zgO5uazNG2/v0PxmiUFY4zxkcEpsQQJ/FCD2hUsKRhjjI9E1w+lR2JMjWpXsKRgjDE+dHq7OJal7Wd/zhF/h+IVSwrGGONDp7ePpVBhTmrpx3QHIksKxhjjQz0SYogKD6kxVUiWFIwxxodCgoMYnBLL7PUZx/2mN3+wpGCMMT52evs4dmTlsiEj29+hVMqSgjHG+Fjf5EYALNkW+PcrWFIwxhgfaxMbSVRYCEu27fN3KJWypGCMMT4WFCR0T4xm6bYsf4dSKUsKxhhTDXokxLB6xwFyjxZUXtiPLCkYY0w16JEYQ36hsjL9gL9DqZAlBWOMqQa9EmMAWBrgjc2WFIwxphrENwyneXR4wF+BZEnBGGOqSc/EmIB/jLYlBWOMqSY9EmPYsieHvdmB+3A8SwrGGFNNeiS47QoBfLZgScEYY6pJ94RoggSWbLWkYIwxdV6DsBDaxUfZmYIxxhhHz8QYlm7bH7BPTLWkYIwx1ahHYgz7co6ydW+Ov0MpkyUFY4ypRj0So4HAfWKqJQVjjKlGHZpGER4aZEnBGGOM8ya2bi2jA/ZxF5YUjDGmmvVMjGFF+gGOFhT6O5RSLCkYY0w165EYw5H8QtbuPOjvUEqxpGCMMdWsU/OGAKyxpGCMMaZV4/rUCw5i/S5LCsYYU+eFBAfRJq4B6ywpGGOMAWjfNIp1uw75O4xSLCkYY4wftG8ayfb9h8nOy/d3KMewpGCMMX7QrmkUAOt3B9bZgiUFY4zxg/ZuUgi0dgWfJgURGSoia0UkVUTur6DcCBFREenjy3iMMSZQJDWuT1hI4F2B5LOkICLBwATg90BnYLSIdC6jXBTwZ2Cer2IxxphAExwktI2LDLjGZl+eKfQDUlV1o6oeASYDF5ZR7hHgSSDXh7EYY0zAad80su6cKQAtgW0en9Pc74qJSC8gUVWn+zAOY4wJSO2aRpGelcvB3KP+DqWYL5OClPFd8auGRCQIeBa4q9IRiYwTkYUisjAjI6MKQzTGGP8pamxODaArkHyZFNKARI/PCUC6x+cooCvwvYhsBk4FppbV2Kyqr6hqH1XtExcX58OQjTGm+rSLjwRgfQC1K/gyKSwA2olIaxGpB4wCphb1VNUsVY1V1WRVTQbmAsNVdaEPYzLGmICR6F6BFEiXpfosKahqPnAr8BWwGvhQVVeKyMMiMtxX0zXGmJoiOEhIiY9kXQBVH4X4cuSqOgOYUeK7h8ope6YvYzHGmEDUvmkUczfu8XcYxeyOZmOM8aN2TSPZkZXLgQC5AsmSgjHG+FH7ePcZSAHS2GxJwRhj/KjostRAuYnNkoIxxvhRQqMIIkKDA+ZxF5YUjDHGj4LcK5DW77YzBWOMMTiNzYFyr4IlBWOM8bP2TaPYdSCPrMP+vwLJkoIxxvhZ+6ZFj7vw/9mCJQVjjPGzTs0bcm7npoQG+3+X7NM7mo0xxlSueXQEr1wTGC+e9H9aMsYYEzAsKRhjjClmScEYY0wxSwrGGGOKWVIwxhhTzJKCMcaYYpYUjDHGFLOkYIwxppioqr9jOC4ikgFs8XccHmKBTH8HUYFAjw8CP8ZAjw8CP8ZAjw9qf4ytVDWuskI1LikEGhFZqKqBcStiGQI9Pgj8GAM9Pgj8GAM9PrAYi1j1kTHGmGKWFIwxxhSzpHDyXvF3AJUI9Pgg8GMM9Pgg8GMM9PjAYgSsTcEYY4wHO1MwxhhTzJKCMcaYYpYUvCAiQ0VkrYikisj9ZfT/o4gsF5ElIvKTiHQOtBg9yo0QERWRar30zotlOEZEMtxluEREbqjO+LyJ0S0zUkRWichKEXk/0GIUkWc9luE6EdkfYPElich3IrJYRJaJyLDqjM/LGFuJyCw3vu9FJKGa43tDRHaLyIpy+ouI/MeNf5mI9K7SAFTV/ir4A4KBDUAboB6wFOhcokxDj+7hwJeBFqNbLgqYDcwF+gRSfMAY4IUAX8/tgMVAI/dzfKDFWKL8bcAbgRQfTkPpzW53Z2BzoC1D4CPgWrf7d8A71Rzj6UBvYEU5/YcB/wMEOBWYV5XTtzOFyvUDUlV1o6oeASYDF3oWUNUDHh8bANXdel9pjK5HgCeB3OoMDu/j8ydvYrwRmKCq+wBUdXcAxuhpNDCpWiJzeBOfAg3d7mggvRrjA+9i7AzMcru/K6O/T6nqbGBvBUUuBN5Wx1wgRkSaV9X0LSlUriWwzeNzmvvdMUTkFhHZgLPT/XM1xVak0hhFpBeQqKrTqzMwl1fLELjUPR2eIiKJ1RNaMW9ibA+0F5E5IjJXRIZWW3QOb5cjItIKaA18Ww1xFfEmvvHAVSKSBszAOZupTt7EuBS41O2+GIgSkSbVEJu3vN4OToQlhcpJGd+VOhNQ1Qmq2ha4D/irz6M6VoUxikgQ8CxwV7VFdCxvluE0IFlVuwMzgbd8HtWxvIkxBKcK6Uyco/DXRCTGx3F58mpbdI0CpqhqgQ/jKcmb+EYDE1U1Aaca5B13+6wu3sR4N3CGiCwGzgC2A/m+Duw4HM92cNwsKVQuDfA8ak2g4lPeycBFPo2otMpijAK6At+LyGacesip1djYXOkyVNU9qprnfnwVOKWaYivizXpOAz5X1aOquglYi5MkqsvxbIujqN6qI/AuvuuBDwFU9RcgHOchb9XFm20xXVUvUdVewP9zv8uqvhArdbz7pONTnQ0oNfEP5+hwI86peFHDVJcSZdp5dF8ALAy0GEuU/57qbWj2Zhk29+i+GJgbaMsQGAq85XbH4pzCNwmkGN1yHYDNuDenBlJ8OA2kY9zuTjg7s2qL08sYY4Egt/sx4OHqXI7udJMpv6H5Dxzb0Dy/Sqdd3TNbE/9wTnPX4Vy18P/c7x4Ghrvd/wZWAktwGqbK3SH7K8YSZas1KXi5DJ9wl+FSdxl2DLRl6P4InwFWAcuBUYEWo/t5PPCP6o7Ny2XYGZjjruclwLkBGOMIYL1b5jUgrJrjmwTsAI7inBVcD/wR+KPHdjjBjX95Vf+W7TEXxhhjilmbgjHGmGKWFIwxxhSzpGCMMaaYJQVjjDHFLCkYY4wpZkmhDhKRZBG5wsuyk9xHT9wpIh3dp28uFpG2JxnDZhE56ZuWRCRMRGa6cV1+suOrZFozRCTG/fuTF+WPWc4i0kdE/nMc0zvNfRrrEhFpKSJTTjT2qiAiE0VkRCVlxohIixMc/0ktL1M1LCnUTclApUlBRJoBA1W1u6o+i3On9ueq2ktVN/g4Rm/1AkJVtaeqfuDNACISfCITUtVhqrofiAEqTQqUWM6qulBVj+e5WFcCT7nztl1VK9whe0tEQqpiPOUYA5xQUuDkl5epCv64wcX+qv4PuAZYhnNT0DvudxOBER5lDrn/5wJZODcP3YnzqIE3cW6EWQwMccstAw675f4O7MR5Dsx3JaZ9M/Ckx+cxwPNu92fAIpwb08Z5lNmMc+doMh53buI8d2a8290W+NId/kdK3NAGxAOpHvPSFjjLnYflwBu4Nx6503sI+IkSN525y+m/ODfNbcR53s0bwGqc5/SUjHmyx3L5F87NRP8CVrjTvbyc5XwmMN3tF+mxzJcBl5aI6QacJ2VuAt7zXE5AfZxHRSwDPgDm4d7AVLSO3e4RRfG78/iMO49P4zzN9w1ggbu8LixjmxLgBZyb9b7AeYDdCLffKcAP7rr5CmjuTu8QzuM/lgARZZVzh0/BecbVUuBXd91VtLwa42xLy9xy3d3vx7vz8b277v7s799iTf/zewD2VwUrEbq4P8RY93Nj9/9Eyk4KxT829/NdwJtud0dgK06iKN4Ruf3GA3eXMf04nMcRF33+HzC4RCwRODvNJu7nzVSeFGbhPkIE6A98W8a0PXcc4TiPnmjvfn4buMNjeveWs/wm4uzoBeexxAeAbjhn0ouAnpXEfCnwDc6z+pu6y695GcvZM9Z/As959GtUTlxFO+HiabrL6GW3uyvOw9q8SQrTgWD38+PAVW53DM7duw1KTP8Sj/lqAex3xxkK/AzEueUux31vAx53y1dSbh5wscd6q1/J8noe+Lvb/Ttgicc2+TMQ5q6bPThnjn7/XdbUP1+eRprq8zucJ2JmAqhqRc9iL8tgnB8dqrpGRLbgPCb6QIVDuVQ1Q0Q2isipOI8H6IDzKAOAP4vIxW53Is4D5PZUNk4RiQQGAh+JFD8UMqySwToAm1R1nfv5LeAW4Dn3c0XVS9NUVUVkObBLVZe7cazE2SEvqWDYwcAkdZ5IuktEfgD6UvHyOxvnoXUAqPuOBi8Nxnm0Cqq6QkSWeTncR/rbU1PPBYaLyN3u53AgCefsqMjp/DZf6SJS9BjuDjjJ6Bt33QTjPJahpDLLiUgU0FJVP3XnIRfAYz2XN8+XuuW/FZEmIhLt9vtCnYcp5onIbpzEnFbZwjBls6RQOwhlPzo3H7fdSJxfXL0Khj9ZHwAjgTXAp+4O9kycnd8AVc0Rke9xdj5lxugq6h8E7FfVnscRQ2XzkV1Bv6IntBZ6dBd9rux3ciLLr7x15u2w5fEcZ8ll7Tn/glNltbaSaZUVowArVXVAJcOWWU5EGpZTvrJxlReb5/oqwPZrJ8UammuHWcDIoheBiEhj9/vN/PYI6gtxTucBDuI8TrvIbJxGTUSkPc4RY2U7i5I+wWmIHs1vR+TRwD43IXTEeaJjSbuAePfILww4H4rfZrdJRC5z4xIR6VFJDGuAZBFJcT9fjVOfXdXKWn6Xi0iwiMThHGHPL6Ocp6+BW4s+iEij45j+TzgJGHHeB97No98uEenkvqPg4rIGdn0F3OYeLBS9hKmk2cAod76aA0Pc79cCcSIywB02VES6uP0857nMcu66TRORi9zvw0SkPhUvL89t9EwgU49946GpIpYUagFVXYnziN8fRGQpToMiOO8lOENE5uPUyRcdKS4D8kVkqYjcCbwIBLtVJx/gPNo4j+PgVn+sAlqp6nz36y+BELd64xGcBsKSwx3FeULlPJw67zUeva8ErnfnaSWVvBbRrYYYi1PltBznKP+l45kPb6jqHmCOiKwQkX8Bn/JbI/+3OG0XOym9nD09CjRyx7GU33a43ngRZ2e7DOelTstwGmgB7sdZjt9SdpVOkUdwDhKWifOC+EfKKPMpTnXgcpyG+B/c+T+C07bwTzf2JThVfeC0XbwkIktwqovKK3c1TtXiMpw2gWZUvLzGA33c8v8Arq1g3sxJsKekGlPDuJfUhqpqrnu/yCycxvUjfg7N1AJW92ZMzVMf+E5EQnHq2m+2hGCqip0pGGOMKWZtCsYYY4pZUjDGGFPMkoIxxphilhSMMcYUs6RgjDGm2P8HLU3x3Ou/nbAAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7f359a1f2780>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "resfile = 'results/2ndstage_RetinaNet-ODAEL-export.pth-CCMCT_ODAEL-val-inference_results_boxes.p.bz2'\n",
    "databasefile = 'databases/MITOS_WSI_CCMCT_ODAEL.sqlite'\n",
    "thr, F1scores, thrs = optimize_threshold(databasefile=databasefile, resfile=resfile, minthres=0.25)\n",
    "\n",
    "import matplotlib.pyplot as plt \n",
    "plt.plot(thrs, F1scores)\n",
    "plt.xlabel('cutoff value for mitotic figure detection')\n",
    "plt.ylabel('F1 score')\n",
    "plt.title('Optimization of detection threshold (trainval set)')\n",
    "plt.text(0.7, 0.5, 'max: %.3f @ %.2f' % (np.max(np.array(F1scores)),np.arange(0.25,0.99,0.01)[np.argmax(F1scores)]))\n",
    "plt.plot([0.25, 1.0], np.max(np.array(F1scores))*np.array([1,1]),'r--')\n",
    "\n",
    "plt.savefig('results/figures/Opt_F1_CCMCT_ODAEL_RetinaNet_2ndstage.pdf')\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Inference (on test set)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "1. ODAEL on ODAEL "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Calculating F1 for test set of 11 files\n",
      "Overall: \n",
      "TP: 18248 FP: 3783 FN:  4228 F1: 0.8200058417776979\n",
      "F1 score:  0.82\n"
     ]
    }
   ],
   "source": [
    "resfile = 'results/2ndstage_RetinaNet-ODAEL-export.pth-CCMCT_ODAEL-inference_results_boxes.p.bz2'\n",
    "databasefile = 'databases/MITOS_WSI_CCMCT_ODAEL.sqlite'\n",
    "F1, individ = calculate_F1(databasefile=databasefile, resfile=resfile, det_thres=0.54)\n",
    "\n",
    "print('F1 score: ', np.round(F1,3))\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "2. ODAEL on HEAEL"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Calculating F1 for test set of 11 files\n",
      "Overall: \n",
      "TP: 17656 FP: 4372 FN:  3915 F1: 0.8099268331842473\n",
      "F1 score:  0.81\n"
     ]
    }
   ],
   "source": [
    "resfile = 'results/2ndstage_RetinaNet-ODAEL-export.pth-CCMCT_ODAEL-inference_results_boxes.p.bz2'\n",
    "databasefile = 'databases/MITOS_WSI_CCMCT_HEAEL.sqlite'\n",
    "F1, individ = calculate_F1(databasefile=databasefile, resfile=resfile, det_thres=0.54)\n",
    "\n",
    "print('F1 score: ', np.round(F1,3))\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "3. ODAEL on MEL"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Calculating F1 for test set of 11 files\n",
      "Overall: \n",
      "TP: 17642 FP: 4387 FN:  3909 F1: 0.8096374483708123\n",
      "F1 score:  0.81\n"
     ]
    }
   ],
   "source": [
    "resfile = 'results/2ndstage_RetinaNet-ODAEL-export.pth-CCMCT_ODAEL-inference_results_boxes.p.bz2'\n",
    "databasefile = 'databases/MITOS_WSI_CCMCT_MEL.sqlite'\n",
    "F1, individ = calculate_F1(databasefile=databasefile, resfile=resfile, det_thres=0.54)\n",
    "\n",
    "print('F1 score: ', np.round(F1,3))\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## MEL data set variant"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Find thresholds"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Optimizing threshold for validation set of 21 files: \n",
      "Best threshold: F1= 0.746689526043899 Threshold= 0.8300000000000003\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYsAAAFmCAYAAACcOrbXAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4xLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvAOZPmwAAIABJREFUeJzs3Xl4FeX1wPHvyUbCGpYECJCAkLDKGhFccS1at6oVUFTcUKt1qbVVf1at2trW1rW0FTcUEFCr4ELFfQdlXwKGHRLWECCsIdv5/TETvYQk9wK5mbucz/PkyZ2Zd2bOO/feOXfemXlHVBVjjDGmNjFeB2CMMSb0WbIwxhjjlyULY4wxflmyMMYY45clC2OMMX5ZsjDGGOOXJYt6JiLpIrJHRGKPcP49InJMKMV0FOttLSJfishuEflHAOVHicjX9RGbnzhOFpHcelpXvdVZRFREuhzBfB3deeNqmP6QiEyoZf4GIrJURNoc7rprWeYVIvJhXS2vlvV8LiLXB3kdDUTkBxFJDeZ6/LFk4Yf7ZV0sIvtEZLOI/FtEkg9j/rUicmblsKquV9XGqlp+JPG4864+knmDFdNRGA1sA5qq6l11uWARGScij9bRsg7aiarqV6ratS6WXWU9te50I9ho4EtV3Qx1896p6kRVPbtOoqtnVROQqh4AXgJ+711UlixqJSJ3AX8F7gaaAYOADOAjEUnwMrYIkQEsVbsztE6IIxy/0zcC4wMtHIXJFOA14GoRaeBZBKpqf9X8AU2BPcBlVcY3BrYC17rDDwFvAlOA3cA8oI87bTxQAex3l/U7oCOgQJxb5nPgUeBbt8y7QEtgIrALmA109Fm/Al2ANLd85d8+5+1UgM7Ap0Ahzi/3iUDyYcSUBrwDbAdWAjf4rP8h4HXgVbe+OUB2LdvxBLcORe7/E9zx44BSoMSN48xq5m3pxrEL+B54BPjaZ3o34CM3ztzK9wrnl6rvst/1qdd/gQJgDXCbz7JigfuAVW695gIdgC/dbbPXXdYwYAiQ7zNvd/d93Olujwt8po0DxgDvu8v9Duhcw7Za766r8j0dDIwCvgb+Duxw4z7HZ57PgT8B37jvaRecHzYvApuADTifr1i3fBfgC/f92AZMqfLZuglY4a5rDCDutBjgfmAdzuf/VaCZO60jB39+Ornr2O2+P/8EJtRQ53Q37jg/791anF/Wi4ADQBxwj8/7tRT4hc9yR3HwZ6XaugEN3Petl0/ZFDemVKA58B7OZ2aH+7p9le1/fQ11GwjMwfn8bgGe8Jk2COc7vxNYCAxxx/8JKAeK3fr/02eeFcCpnu0TvVpxqP8BQ4Gyyg9xlWmvAJPc1w+5H+5LgXjgtzhf6HifD/mZPvNW/WJ9jrND7ozzJV8KLAfOdL8QrwIvV/nQd6kmpok+MXUBznK/CCk4O7ynfMr6i+kL4F9AItDX/aKc4VPfYuBcnB3sY8CsGrZhC/cLdqVblxHucEt3+jjg0Vreg8k4iakR0Atnx/e1O60RkAdc4y67P87Or2d1y8bZ2c0FHgASgGOA1cDP3Ol3A4uBrjg7kT4+cR60zfFJFu57vhIn0SQAp+PsvLr6xLEdZ8cR575Pk2uo70HvgztuFM7n6wZ3e98MbOSnnfjnOEmmp7v8eGAq8Jy7jVJxEu2NbvlJwP+52yMROKnKZ+s9IBlnJ14ADHWnXevW8xicH0xvAeNr+PzMBJ7A+fyd4m6PmpLFz4GcKuMO+VzgfGYX4CTwJHfcL3F+AMTgJPG9QFuf7VY1WdRUt5eAP/mUvQX4wH3dErgEaAg0Ad4ApvqU/Zyak8VM4Er3dWNgkPu6Hc4PuXPd2M9yh1NqWybOD6fbqltXffyF4yFrfWkFbFPVsmqmbXKnV5qrqm+qainOlyQR55dDoF5W1VWqWgT8D1ilqh+7634D6FfbzCLye5xf2dcCqOpKVf1IVQ+oaoEb06mBBCIiHYCTgN+rarGqLgBewNnhV/paVaerc45jPM6OtTo/B1ao6nhVLVPVScAPwPkBxBGL8yV9QFX3quoSnCRd6Txgraq+7C57Hs5Rw6U1LPI4nC/jw6paos55n+eB4e7064H7VTVXHQtVtdBfnDjvc2PgL+5yP8XZKY3wKfOWqn7vvp8TcRLw4Vinqs+72/sVoC3Q2mf6OFXNcZffAjgHuMPdbluBJ33qWYrT/Jfmvr9VT57/RVV3qup64DOfWK/A+WW8WlX3APcCw6s2CYlIOs62/oP7+fsS52i5Jsk4ySQQz6hqnqruB1DVN1R1o6pWqOoUnF/eA2uZv6a6vcbB79fl7jhUtVBV/6uq+1R1N84v/4C+SzjbuouItFLVPao6yx0/EpjufocqVPUjnCOQc/0sbzfO9vKEJYuabQNa1dA+2tadXimv8oWqVgD5OL94ArXF5/X+aoYb1zSjiJwD3A5cVPklEpFUEZksIhtEZBcwgYOTW23SgO3uF6PSOpxfQ5U2+7zeByTWsJ3S3Hl9VV1WTVJwfinn+YzzXVYGcLyI7Kz8w9mh1XRFTQaQVqX8ffy00+2A06RxuNKAPPd9942ztu1V4/tZgx/nV9V97kvfZfhuowyco4tNPvV8DucIA5xmRwG+F5EcEbm2pnVVibXqe7kO5/3xTVqV5Xao6t4qZWuyA+cXeyB864mIXCUiC3zq2YvaP+c11e1TIElEjheRDJwk8ra7joYi8pyIrHO/S18CyQFeOXgdkAX8ICKzReQ8d3wG8Msqn8WTcPYrtWmC02zliWg8URSomThtoxfjNIUAICKNcH653edTtoPP9BigPU5TATiHv0EhIl1xfmlerKq+X6TH3PX2VtVCEbkIp924Um0xbQRaiEgTn4SRjtMEdLg24nwxfKUDHwQwbwFOM2AHnKORynkr5QFfqOpZNcxftY55wBpVzayhfB5OU+CSAGLztRHoICIxPgkjHacp8XAd6WfFd748nM9tq+qOitW54ugGABE5CfhYRL5U1ZV+1lH1vUzHeX+24HzeK20CmotII5+EkU7NdVsEHCMicT7x1lT2x/HuTv154AxgpqqWi8gCnER4WFS1QkRexzm62AK85/PZvwunafJ4Vd0sIn2B+YGsR1VXACPcfcLFwJsi0hLnPRqvqjf4q2cV3QG/l5gHix1Z1MBtEvoj8KyIDBWReBHpiNMslM/BV28MEJGL3V/Xd+B8WSsPObfgtPPWKRFpCkzDaTqp2pTQBOfk2E4RaYfTHu+rxpjcpPMt8JiIJIpIb5xfSBOPIMzpQJaIXC4icSIyDOiB00xTK7fJ5S3gIffXXQ/gap8i77nLvtJ9b+JF5DgR6V5DHb8HdonI70UkSURiRaSXiBznTn8BeEREMt2rinq7X+zqluXrO5y28t+5MQzBaWab7K+O1SjAufjgiD8vqroJ+BD4h4g0FZEYEeksIqcCiMgvRaRy574DZ8cUyCXTk4A7RaSTiDQG/oxzcvyghKSq63CaVP4oIgluQqqx2VFV8zm0+SiQ70wjN/YCt17X4BxZHKnXcM57XOG+rtQE5+h+p4i0AB4MdIEiMlJEUtwfEZVHBOU4R/rni8jP3M9hoogM8XlfDqm/+z1uwU/7lXpnyaIWqvo3nCOIv+Nc0fAdzq+CM9S59rnSNJwPWuXJ3Ivd8xfg/Mq/3z3c/G0dhtcf5xfPE+LcULdHRPa40/7oTi/CuQrnrSrz+otpBM5Jy404h+MPuu2qh8Vt8z8P59dZIU4TyHmquq3WGX9yK05TwWack54v+yx7N3A2Tlv8RrfMX3FOqoJzNVAPt45T3eRzPk4TwxqcZsQXcC4qAOe8zus4O9pd7vxJ7rSHgFfcZV1WpY4lwAU4R5vbcC4MuEpVf+AwuU1MfwK+cdd1OOe9fF2Fc7J9Kc5n8k1+auI4DvjO/ay8A9yuqmsCWOZLOD+QvsTZfsXAr2soezlwPM6J/QdxLtKozXMcfE7soPeuuhlUdSnOr+yZODvXY3GuCDsiqlqZ9NNwzhtWegrnc7ANZ0cdyFFxpaFAjrutnwaGu+eJ8oALcfYtBTj7lLv5aX/8NHCpiOwQkWfccZcDr1TZ79SryisqzBESkYdwrpQZ6XUsxoQjce4dmI/zI2yT1/GEGnf7LAROcS9Y8ISdszDGeMr9tdzD6zhClbt9unkdhzVDGWOM8cuaoYwxxvhlRxbGGGP8iphzFq1atdKOHTt6HYYxxoSVuXPnblPVFH/lIiZZdOzYkTlz5ngdhjHGhBURqe0O+x9ZM5Qxxhi/LFkYY4zxy5KFMcYYvyxZGGOM8cuShTHGGL8sWRhjjPHLkoUxxhi/IuY+i6M2ZMih4y67DH71K9i3D86t5omHo0Y5f9u2waXVPM3z5pth2DDIy4Mrrzx0+l13wfnnQ24u3HjjodPvvx/OPBMWLIA77jh0+p//DCecAN9+C/fdd+j0p56Cvn3h44/h0UcPnf7cc9C1K7z7LvyjmmeqjB8PHTrAlCnw738fOv3NN6FVKxg3zvmravp0aNgQ/vUveP31Q6d//rnz/+9/h/eqPOIiKQn+5/YU/cgj8MknB09v2RL++1/n9b33wsyZB09v3x4mTHBe33GHsw19ZWXB2LHO69GjYXmVZxX17etsP4CRIyE//+DpgwfDY485ry+5BAqrPIH1jDPgD39wXp9zDuzff/D0886D37q9w4fwZ+/A7LmU3nY7+0vLiYuJISEuhoTYGOL+8mfkxBPtsxcqn73K+gSRJQtjDAAVquw5UM7eA2VM+3wVU5ckEL94If+3oeiQsk89P4tN35ZwcsEKrtyym9gYIVaEuBghNkaYv2gjsXFt6LJ1Nx3KKoiPjSHmsJ9hZ0JJxHQkmJ2drXYHtzGBKymrYGH+TmauKmTmqkLmrt9BSZnzZNhWjRvQq11TeqY1pVdaMzJbN2Z3cRmbi4rZVFTM5l3O/y1FxewqLmV3cRl7DpSxu7iUihp2KS0bJZDaNJHjOjbnon7t6NchGRHLIF4Tkbmqmu2vnB1ZGBNFSsoqeH/xRt6at4E5a3ewv7QcEejepilXDspg0DEt6dO+GalNE49o+arK/tJydu0vY9ueA2zdXczWXQfYsusAW3YXs3HnfqbMzuPVmevo1KoRF/Vtx0X90sho2aiOa2rqmh1ZGBMFtu05wMRZ65nw3ToKdh+gY8uGDOmayqBjWjLomBYkN0yot1h2F5fyvyWbeXveBmatKUQVBmQ0Z1h2By7om0ZifGy9xWICP7KwZGFMBMvZWMTL36zlnQUbKSmvYEjXFK45sRMnd2lFTAicRNi4cz/vLNzIf+fms2LrHlo2SuCKQRlcOSiDlCYN/C/AHDVLFsZEscX5Rfzjo1w+zy0gKT6WSwe05+oTOtIltbHXoVVLVZm5qpCXvlnDJz9sJT4mhvP7pHHdSZ3okdbU6/AiWkicsxCRocDTQCzwgqr+pcr0J4HT3MGGQKqqJvtMbwosA95W1VuDGasxkWD5lt088eFyPsjZTHLDeH43tCtXDMygWcN4r0OrlYhwQpdWnNClFWu27eXlb9bwxpx8/jsvn9O6pvDHC3qR3rKh12FGtaAdWYhILLAcOAvIB2YDI1R1aQ3lfw30U9VrfcY9DaQA2/0lCzuyMNFs7ba9PPXxcqYt3EijhDiuP7kT157UiaaJoZ0kalO0r5SJ369jzKcrKatQbjsjkxtOPoaEOLuXuC6FwpHFQGClqq52A5oMXAhUmyyAEcCDlQMiMgBoDXwA+K2IMdGocM8Bnvx4OZO+zyM+Vhh9yjHcdEpnmjeqvxPWwdKsYTy/GtKFi/u154/v5vD4jFzenr+BP13Ui+OPael1eFEnmMmiHZDnM5wPHF9dQRHJADoBn7rDMcA/gCuBM2pagYiMBkYDpKen10nQxoSDA2XlvPLtWp79ZCX7Ssu5fGA6vz69yxFf8hrK2jRL5N8jB/DpD1t4YFoOw8bO4tIB7bnv3O60iICkGC6CmSyqu9Sipjav4cCbqlruDv8KmK6qebXdtKOqY4Gx4DRDHUWsxoQFVWVGzhYe+98y1hXu47SuKfzfz7vTJbWJ16EF3endWjP4mFY8++kKxn65ms9zt/LksL6cnOn38dGmDgQzWeQDHXyG2wMbayg7HLjFZ3gwcLKI/ApoDCSIyB5VvScokRoTBuau28HfPviB79ZsJzO1Ma9cO5BTs6JrR5mUEMvvhnbjgr5p3DZpPle99D03ndqZ35yVRXysncsIpmCe4I7DOcF9BrAB5wT35aqaU6VcV2AG0EmrCUZERgHZdoLbRKPS8gqmL97Ey9+sZUHeTlo0SuDOs7IYcVwH4qJ857i/pJyH31vKpO/X0z89mWdG9KN9c7ti6nB5foJbVctE5FacRBALvKSqOSLyMDBHVd9xi44AJleXKIyJVjv2lvDa9+sZP3Mdm3cV06lVIx6+sCeX9G9PowbWSw84RxmPXXwsJ3RuyX1vLebcp7/ib5f2Zmivtl6HFpHspjxjQkjhngM888kKJs/O40BZBSdntuLaEztxalZKSNxxHarWF+7j15PmsTC/iOtO6sT9P+9unRQGyPMjC2NM4IpLy3npmzX867NV7C8t59L+7bnu5E5ktY78E9d1Ib1lQ9646QQefX8pL369hg7Nkxh1Yievw4ooliyM8VBFhTJt4QYe/yCXjUXFnNm9Nfec0y1ku+UIZQlxMTx0fk827izm0feX0SOtGQM7tfA6rIgR3WfIjPHQt6u2ccGYr7lzykJaNm7ApBsG8cLV2ZYojkJMjPDEsD6kt2jIrybOY3NRsdchRQxLFsbUs6UbdzHq5e+5/Pnv2L6nhKeG9WXaLScyuLPdlVwXmibG89yVA9hXUsbNE+dyoKzc/0zGL0sWxtSTvO37uHPKAn7+7FfMX7+T+87txqe/HcJF/drZyes6ltm6CX//ZR/mr9/JI+/V1MOQORx2zsKYINu+t4R/frqSCbPWIQI3ntKZm0/tHPI9wYa7c49ty42nHsNzX6ymd/tkLsvu4H8mUyNLFsYEiary9vwN/PHdpewuLuWy7A7cfmYmbZsleR1a1Lj77K4s2VDE/VOX0K1NE3q3T/Y/k6mWNUMZEwSbi4q57pU5/Ob1hWSmNmbGHafwl0t6W6KoZ3GxMTwzvB8pjRtw84R5FO0r9TqksGXJwpg6pKq8PjuPs578gm9XbeOB83ow5cbBZNr9Ep5p2bgBY67oz5Zdxdw3dTGRciNyfbNmKGPqyIad+7n3rcV8ubyAgZ1a8LdLetOxVSOvwzJA3w7J3HlWFo/PyOX0rqlcMqC91yGFHUsWxtSBr1ds4+aJcymvUB6+sCcjj8+wK5xCzE2nduaL3AIemLaE7I7NyWhpifxwWDOUMUdp0vfrufrl70lrlsQHt5/CVYM7WqIIQbHuDXsxMcIdUxZQVl7hdUhhxZKFMUeookL58/Rl3PvWYk7q0oo3bx5MekvrIjuUtW/ekD/94ljmr9/Js5+u9DqcsGLJwpgjsK+kjJsmzGXsl6u5clAGL16dTZNEu28iHFzQJ42L+7Xj2U9XMHfddq/DCRuWLIw5TFt2FXPZczP5eNkWHjy/Bw9f2DPqH0QUbv54YU/aNU/ijikL2F1sl9MGwj7hxhyGNdv2ctGYb1hdsJfnr8rmmhM72XMTwlCTxHieGtaXDTv28+C0HP8zmOAmCxEZKiK5IrJSRA55fraIPCkiC9y/5SKy0x3fV0RmikiOiCwSkWHBjNOYQKzZtpfhY2dyoKyC128czBndW3sdkjkKAzJacMtpXXhr/gYW5xd5HU7IC1qyEJFYYAxwDtADGCEiPXzLqOqdqtpXVfsCzwJvuZP2AVepak9gKPCUiNh9+sYza7btZcTYWZSWK6/dcDy92jXzOiRTB64/+RiS4mOZMGud16GEvGAeWQwEVqrqalUtASYDF9ZSfgQwCUBVl6vqCvf1RmArkBLEWI2p0Vo3UZSUVzDx+uPp1qap1yGZOtIsKZ6L+qUxbeEG6wrEj2Ami3ZAns9wvjvuECKSAXQCPq1m2kAgAVgVhBiNqdXabXsZPnYWB8rKmXj98XRva4ki0owclEFxaQVvzsv3OpSQFsxkUd1Zv5o6ZRkOvKmqBz2lRETaAuOBa1T1kDtoRGS0iMwRkTkFBQVHHbAxvtZu28uI551E8doNgyxRRKieac3on57MhFnrqKiwfqNqEsxkkQ/4diDfHthYQ9nhuE1QlUSkKfA+cL+qzqpuJlUdq6rZqpqdkmKtVKbuLN+ymxHPz6K4tJyJ11uiiHRXDs5gzba9fLuq0OtQQlYwk8VsIFNEOolIAk5CeKdqIRHpCjQHZvqMSwDeBl5V1TeCGKMxh/hqRQGX/OtbyiqUidcPokeaJYpId06vtrRolMD4WWu9DiVkBS1ZqGoZcCswA1gGvK6qOSLysIhc4FN0BDBZD+43+DLgFGCUz6W1fYMVqzGVXvtuPaNenk275klMveVESxRRIjE+lsuyO/DR0i1sKtrvdTghSSKlb/fs7GydM2eO12GYMFVRofzlgx8Y++VqTs1K4Z+X97PuO6JM3vZ9nPL4Z/z6tC785uyuXodTb0Rkrqpm+ytnd3CbqLe/pJybJ1o/T9GuQ4uGnN41lUmz8ygpsx5pq7JkYaLa1l3FDBs7kw+XbuGB86yfp2g3cnAGBbsP8OHSzV6HEnLsW2Gi1tKNu7hwzDes3LqH56/M5tqTrJ+naHdqZgodWiQxfqbd0V2VJQsTlT5euoVL//MtAG/cNJgze1g/TwZiYoSRx2fw3Zrt5G7e7XU4IcWShYkqqsoLX63mhvFz6JLamGm3nEjPNOvnyfzkl9kdSIiLsf6iqrBkYaJGaXkF9729hEffX8bPerRhyujBpDZN9DosE2JaNErgvN5teWtePnsOlHkdTsiwZGGiwu7iUq55eTaTvl/Pr4Z05l9X9CcpIdbrsEyIuuL4dPaWlPPBEjvRXcmShYl4+0vKuW7cHGatLuTxS3vzu6HdiImxE9mmZv3Tm5PRsiFvz7fOBStZsjARraSsgpsmzGXOuu08Nbwvv8zu4H8mE/VEhIv6tuPbVYV2R7fLkoWJWOUVyp1TFvDF8gIeu/hYzuud5nVIJoxc1K8dqvDOgpr6P40ulixMRFJV7ntrMe8v3sT9P+/OsOPSvQ7JhJlOrRrRLz2Zt+dv8DqUkGDJwkQcVeXR95cxZU4et53ehetPPsbrkEyY+kW/dvyweTfLNu3yOhTPWbIwEeeZT1by4tdrGHVCR+48K8vrcEwYO693GnExYkcXWLIwEeaFr1bz5MfLuaR/ex44r4d132GOSotGCQzpmsK0BRsoj/Kn6FmyMBHj5W/W8Oj7yzj32Db89ZJj7fJYUyd+0a89W3YdYGaUP0XPkoWJCK/OXMsf313Kz3q25unh/aznWFNnzuieSpMGcVHfFGXfKBP2Jn63jgem5XBm99Y8O6I/8ZYoTB1KjI/l3GPb8sGSTewvKfc6HM8E9VslIkNFJFdEVorIPdVMf9LnsanLRWSnz7SrRWSF+3d1MOM04WvK7PX839tLOL1bKmOu6EdCnCUKU/d+0b8de0vKo/o5F3HBWrCIxAJjgLOAfGC2iLyjqksry6jqnT7lfw30c1+3AB4EsgEF5rrz7ghWvCb8vDEnj3veWsypWSn864r+NIizvp5McAzs2IJ2yUm8PX8DF/Zt53U4ngjmz7CBwEpVXa2qJcBk4MJayo8AJrmvfwZ8pKrb3QTxETA0iLGaMPPGnDx+999FnNSlFc9dOYDEeEsUJnhiYoQL+6bx1YptFOw+4HU4nghmsmgH5PkM57vjDiEiGUAn4NPDmVdERovIHBGZU1BQUCdBm9D3wleruftNJ1GMvTLbEoWpF7/o147yCuXdhdHZ/Ucwk0V11y3WdKHycOBNVa08exTQvKo6VlWzVTU7JSXlCMM04UJV+fuM3B8vj33h6mzrZtzUm8zWTejVrilTF0TnVVHBTBb5gG8Xn+2BmlLycH5qgjrceU0UKK9Q7p+6hH9+tpLhx3Xg2RF2jsLUv4v6tmNRfhErt+7xOpR6F8xkMRvIFJFOIpKAkxDeqVpIRLoCzYGZPqNnAGeLSHMRaQ6c7Y4zUaikrILbJ89n4nfruenUzjx28bHE2g13xgPn93F6Lp6RE31XRQUtWahqGXArzk5+GfC6quaIyMMicoFP0RHAZFVVn3m3A4/gJJzZwMPuOBNl9pWUccOrc3hv0SbuPacb95zTzbrwMJ5p3TSRvh2S+TAKk0XQLp0FUNXpwPQq4x6oMvxQDfO+BLwUtOBMyCsuLef6V5wn3P31kmOtm3ETEs7u2Zq/fZDLpqL9tG2W5HU49cbuYDIhqbxCuWPyAr5dVcjff9nHEoUJGWf3aAPAx0u3eBxJ/bJkYUKOqnL/1MV8kLOZP5zXg4v7t/c6JGN+1CW1McekNOJDSxbGeOvvH+Yy6fs8bjmtM9ed1MnrcIw5xNk92jBzVSFF+0u9DqXeWLIwIeXFr9cw5rNVjBiYzm/P7up1OMZU6+yerSmrUD7P3ep1KPXGkoUJGW/Ny+eR95ZyTq82PHpRL7vqyYSsvu2TSWnSgA9zoqcpypKFCQmf/rCFu99cxAmdW/LU8L52H4UJaTExwlk9WvN57laKS6Oj23JLFsZzs9du5+YJ8+jetgnPXTnA7sw2YeHsHq3ZW1IeNU/Qs2RhPLV04y6uHTebdslJvHLNQJokxnsdkjEBGdy5JY0bxEXNMy4sWRjPrCvcy1UvfU/jBnGMv/54WjZu4HVIxgSsQVwsQ7qm8NHSLZRX1NRHauSwZGE8sWVXMSNf/I7yigrGXzeQdsnRcyesiRxn92zDtj0lLMiL/OeyWbIw9a5oXylXvfg9hXtKGHfNQLqkNvE6JGOOyJCuKcTHSlRcFWXJwtSrfSVlXPvKbNZs28vzV2XTp0Oy1yEZc8SaJsYzuHMrZuRsxqcv1IhkycLUm7LyCm6ZOI/563fwzIi+nNilldchGXPUzu7RmrWF+yL+GReWLEy9UFUeejeHz3ILeOSiXgzt1dbrkIypE2f1aA0Q8X1F+U0WItJQRP4gIs+7w5kicl7wQzOR5MWv1zBh1npuPPUYrjg+w+twjKkz0fKMi0COLF4GDgDxDzT/AAAgAElEQVSD3eF84NGgRWQizoyczfxp+jLO6dWG3/+sm9fhGFPnzu7ZmoX5RWwq2u91KEETSLLorKp/A0oBVHU/YH0xmIAsyt/J7ZPn06d9Mk8O60uMdeNhItBZ3Z2mqC9yCzyOJHgCSRYlIpIEKICIdMY50vBLRIaKSK6IrBSRe2ooc5mILBWRHBF5zWf839xxy0TkGbFe5cJO/o59XPfKHFo1bsDzV2WTGG/deJjI1CW1Mc0bxjNvfeTebxHIY1UfBD4AOojIROBEYJS/mUQkFhgDnIXTdDVbRN5R1aU+ZTKBe4ETVXWHiKS6409w19PbLfo1cCrweWDVMl7bVVzKdePmUFxazmvXH09KE7s720QuEWFARnPmrovcZFHrkYX7a/4H4GKcBDEJyFbVzwNY9kBgpaquVtUSYDJwYZUyNwBjVHUHgKpWdg6vQCKQADQA4oHIvtQgghwoK+eWifNYVbCH/4wcQGZru+nORL5+6c1ZVbCXHXtLvA4lKGpNFurcZTJVVQtV9X1VfU9VtwW47HZAns9wvjvOVxaQJSLfiMgsERnqrncm8Bmwyf2boarLqq5AREaLyBwRmVNQELltheGkrLyC2ybN56sV2/jLJb3tXgoTNQZkNAdgfoR2/RHIOYtZInLcESy7unMMVW9xjAMygSHACOAFEUkWkS5Ad6A9ToI5XUROOWRhqmNVNVtVs1NSUo4gRFOXKiqUu99cxIycLTx0fg8uHWDPzjbRo0/7ZGJjJGKbogI5Z3EacKOIrAP24iQBVdXetc9GPtDBZ7g9sLGaMrNUtRRYIyK5/JQ8ZqnqHgAR+R8wCPgygHiNB1SVP0xbwtvzN3D3z7oy6kR7draJLkkJsfRMaxqxySKQI4tzgM7A6cD5wHnuf39mA5ki0klEEoDhwDtVykzFSUaISCucZqnVwHrgVBGJE5F4nJPbhzRDmdCgqjz2vx+Y+N16bh7SmVtO6+J1SMZ4on96cxbmFVFaXuF1KHXOb7JQ1XVAMk6COB9Idsf5m68MuBWYgbOjf11Vc0TkYRG5wC02AygUkaU45yjuVtVC4E1gFbAYWAgsVNV3D7t2pl4888lKxn65mqsHZ/C7n3X1OhxjPDMgozn7S8v5YdNur0Opc36boUTkdpyrlt5yR00QkbGq+qy/eVV1OjC9yrgHfF4r8Bv3z7dMOXCj3+iN5174ajVPfrycSwe058Hze2K3w5hoVnmSe+667RzbvpnH0dStQJqhrgOOV9UH3B39IJzkYaLcR0u38Oj7yzj32Db85eJj7e5sE/XSkpNo2yyRuet3eh1KnQskWQhQ7jNcjnX3EfVWbt3DnVMW0Lt9M564rC9xsdaBsTEA/TOaMy8CT3IHcjXUy8B3IvK2O3wR8GLwQjKhbldxKaNfnUNifAz/GTnAuvEwxseA9Oa8v2gTm4r207ZZ5DwuOJAT3E8A1wDbgR3ANar6VLADM6GpokK5Y/IC1m/fx5jL+5Nmz8425iCV5y3mrYuspqhAnmcxCFihqs+o6tPAShE5PvihmVD05MfL+fSHrTx4fg+OP6al1+EYE3J6pDUlMT4m4joVDKSh+d+A7/MC97rjTJT53+JNPPvpSoZld2DkIHuAkTHViY+NoXe75Ii7OS+gE9zq8yRyVa0gsHMdJoLkbt7NXW8spF96Mg9fZJfIGlOb/hnNydlYRHFpuf/CYSKQZLFaRG4TkXj373acu6xNlNixt4TR4+fQqEEc/xk5gAZxdkLbmNoMyGhOabmyeEOR16HUmUCSxU3ACcAGnL6cjgdGBzMoEzpKyiq4acJcNu0s5j8jB9C6aaLXIRkT8vqnJwNEVFOU3+Yk9xkTw+shFhNiVJUH31nCd2u28+SwPj9e5WGMqV3Lxg3o1KpRRCWLQK6G+puINHWboD4RkW0iMrI+gjPeeumbtUz6Po9bTuvML/pZd+PGHI7+6c7NeT6nfMNaIM1QZ6vqLpzeZvNxeoa9O6hRGc999sNW/vT+Uob2bMNdZ1nngMYcrgEZzSncW8K6wn1eh1InAkkW8e7/c4FJqro9iPGYEJC7eTe/njSf7m2b8sSwPtbnkzFH4KdOBSOjKSqQZPGuiPwAZAOfiEgKUBzcsIxXCvcc4LpXZpOUEMsLV2fTMMGukjbmSGSmNqZJgzjmRsjNeYF093EPMBjIdp9otw+4MNiBmfpXXFrOTRPmUrD7AC9clR1R/doYU99iYoR+EdSpYEBdharqDvcZE6jqXlXdHNywTH0rK6/gtknzmb12B/+4rA99OiR7HZIxYW9AenNyt+xmV3Gp16EctaD2Ky0iQ0UkV0RWisg9NZS5TESWikiOiLzmMz5dRD4UkWXu9I7BjDWaqSr3vrWYD5du4aHze3Be7zSvQzImIgzIaI4qLIiA51sELVmISCwwBucZ3j2AESLSo0qZTOBe4ERV7Qnc4TP5VeBxVe0ODAS2BivWaKaq/Hn6Mt6Ym8/tZ2Qy6sROXodkTMTo08F5Wt6i/ChNFiLSLYBiA4GVqrpaVUuAyRx6ruMGYIyq7oAfbwDETSpxqvqRO36PqkbG9Wch5l+fr+L5r9Zw9eAM7jgz0+twjIkoTRLjaZecxPIte/wXDnFHemTxYQBl2gF5PsP57jhfWUCWiHwjIrNEZKjP+J0i8paIzBeRx90jFVOHJn63jsdn5HJR3zR7frYxQZLVujHLt+z2OoyjVuN1kSLyTE2TgEDOfla356l6K2MckAkMAdoDX4lIL3f8yUA/YD0wBRhFlSf0icho3H6q0tPTAwjJVHpv0Ubun7qE07ul8vgv7V4KY4Ilq00TvllZSFl5RVg/fri2yK8BlgBzq/zNAUoCWHY+0MFnuD2wsZoy01S1VFXXALk4ySMfmO82YZUBU4H+VVegqmNVNVtVs1NSUgIIyQDMXFXInVMWkJ3RnDGX9yc+jD/AxoS6rNQmlJRXsDbM7+Su7Y6r2cASVf226gQReSiAZc8GMkWkE06PtcOBy6uUmQqMAMaJSCuc5qfVwE6guYikqGoBcDpOkjJHac22vdw0YS4dWzbihauPIynBWveMCaas1k0AWL5lN11SG3sczZGr7SflpcCC6iaoqt9LZtwjgluBGcAy4HVVzRGRh0XkArfYDKBQRJYCnwF3q2qhe0/Hb3HuGF+M06T1fKCVMtUr2lfKdeNmExsjvHj1cTRLivc/kzHmqHRJbYwIYX/eorYji8ZH2w+Uqk4HplcZ94DPawV+4/5VnfcjoPfRrN/8pLS8gpsnziVvxz4mXj+I9JYNvQ7JmKiQlBBLeouGrAjzK6JqO7KYWvlCRP5bD7GYIFFVHpiWw7erCnns4t4M7NTC65CMiSpZrZuQG+ZHFrUlC9/LY44JdiAmeJznUqzn5iGduXSAPZfCmPqW1boxa7ft5UBZ+D6Tu7ZkoTW8NmGk8rkUP+vZmrvPtudSGOOFrNZNKKtQ1mzb63UoR6y2cxZ9RGQXzhFGkvsad1hVtWnQozNHZfmWn55L8eSwvnYvhTEe+emKqD10axOeu84ak4Wq2jWVYaxofymjX51DYrw9l8IYrx2T0ojYGGH55t3Qx+tojozdjRWBKiqUOybPJ3/Hfv49sr89l8IYjzWIi6Vjy4ZhffmsJYsI9NQnK/gst4AHz+/BcR3tyidjQkFW6yas2Bq+l89asogwH+Zs5plPVvDLAe0ZOSjD63CMMa6s1k1YW7iX4tLwvCLKkkUEWbl1D795fSG92zfjkYt6WS+yxoSQrNZNUHW+p+HIkkWE2F1cyo3j59AgLob/jBxAYrxdn2BMKOnaxukXKlzPW9glMhGgokK56/WFrC3cx4Trjict2U5oGxNqMlo2Ij5WwvZBSHZkEQH+/cUqPly6hfvO7c7gzi29DscYU4342Bg6p4Tvg5AsWYS5z3O38vcPc7mwbxrXntjR63CMMbXIbN3EkoWpf+sL93H75AV0a9OUv1zc205oGxPislIbk79jP3sPlHkdymGzZBGm9peUc+OEuagqz40cYA8xMiYMZLVxuv0Ix/stLFmEIVXl3rcW8cPmXTw9op89m8KYMOH71LxwY8kiDI37di1TF2zkN2dmcVrXVK/DMcYEKL1FQxrExTh9RIWZoCYLERkqIrkislJE7qmhzGUislREckTktSrTmorIBhH5ZzDjDCffrS7kT+8v48zurbnltC5eh2OMOQyxMUKX1MYsD8NmqKDdZyEiscAY4CwgH5gtIu+o6lKfMpnAvcCJqrpDRKr+TH4E+CJYMYabzUXF3PLafNJbNOSJYX2sy3FjwlDX1k2YubrQ6zAOWzCPLAYCK1V1taqWAJOBC6uUuQEYo6o7AFR1a+UEERkAtAY+DGKMYWN/STmjx89hX0kZz105gKaJ8V6HZIw5Apmtm7CpqJii/aVeh3JYgpks2gF5PsP57jhfWUCWiHwjIrNEZCiAiMQA/wDurm0FIjJaROaIyJyCgoI6DD20qCq/fXMhizcU8fTwfmS6J8mMMeEnq7XT7cfKreF13iKYyaK6NpKqj2eNAzKBIcAI4AURSQZ+BUxX1TxqoapjVTVbVbNTUlLqIOTQ9PQnK3h/0SZ+P7QbZ/Vo7XU4xpijUHlFVO7m8DpvEcy+ofKBDj7D7YGN1ZSZpaqlwBoRycVJHoOBk0XkV0BjIEFE9qhqtSfJI9n7izbx1McruLh/O2485RivwzHGHKV2yUk0TIgNu8tng3lkMRvIFJFOIpIADAfeqVJmKnAagIi0wmmWWq2qV6hquqp2BH4LvBqNiWJR/k7uemMB2RnNeeziY+0ObWMiQEyMhGW3H0FLFqpaBtwKzACWAa+rao6IPCwiF7jFZgCFIrIU+Ay4W1XD7zKBINhcVMwNr86hZaMG/OfKATSIszu0jYkUWamNw6732aB2Ua6q04HpVcY94PNagd+4fzUtYxwwLjgRhqbKK5/2FJfx5s0n0KpxA69DMsbUoa5tmvDG3Hy27y2hRaMEr8MJiN3BHWIqKpS73ljw45VP3ds29TokY0wdywzDbj8sWYSYv37wA9MXb+a+c7pzpl35ZExEqrx8doUlC3Mkxs9ax3NfruaqwRlcf3Inr8MxxgRJm6aJNEqIZfW2vV6HEjBLFiHisx+28uC0JZzRLZUHzuthVz4ZE8FEhLbJSWzaWex1KAGzZBEClmwo4pbX5tEjrSnPjOhHXKy9LcZEurTkJDYW7fc6jIDZXsljG3fu59pxs0lOiuelq4+jUYOgXqBmjAkRac0S2WhHFiYQu4pLuebl2ewvKeflawaS2jTR65CMMfUkLTmJbXsOcKCs3OtQAmLJwiOl5RXcMnEeqwr28O+RA+jaxjoHNCaatG3m/DjcXBQeRxeWLDzgPBZ1MV+t2MZjFx/LSZmtvA7JGFPP2iUnAYRNU5QlCw88/ckK3pybzx1nZvLL7A7+ZzDGRJy2PyaL8DjJbcminr0xJ4+nPl7BpQPac/sZmV6HY4zxSGUz1KYwuSLKkkU9+mpFAfe+tZiTM1tZL7LGRLnE+FhaNkpggzVDGV/LNu3i5gnz6JLamH9d0Z94u5fCmKiXlpxkRxbmJ5uK9nPNy7Np3CCOl685jib2/GxjDE5TlJ2zMAAUl5Zz/Stz2HOgjJevOY62zZK8DskYEyLSwqjLD0sWQfbAtCXkbNzF08P7WnfjxpiDpCUnsvtAGbuKS70OxS9LFkE0ZfZ6Xp+Tz69P78IZ3a27cWPMwdLcy2fD4egiqMlCRIaKSK6IrBSRap+hLSKXichSEckRkdfccX1FZKY7bpGIDAtmnMGwZEMRf5iWw0ldWnHHmVleh2OMCUGVzdLhcN4iaL3WiUgsMAY4C8gHZovIO6q61KdMJnAvcKKq7hCRVHfSPuAqVV0hImnAXBGZoao7gxVvXdq5r4SbJsylVaMEnh7el9gYu0TWGHOoH+/iDoMrooJ5ZDEQWKmqq1W1BJgMXFilzA3AGFXdAaCqW93/y1V1hft6I7AVSAlirHWmokK5c8oCtuwqZswV/Wlpz882xtQgpUkD4mIkLI4sgpks2gF5PsP57jhfWUCWiHwjIrNEZGjVhYjIQCABWFXNtNEiMkdE5hQUFNRh6EduzGcr+Sy3gAfO60G/9OZeh2OMCWGxMULrpolRf86iurYXrTIcB2QCQ4ARwAsikvzjAkTaAuOBa1S14pCFqY5V1WxVzU5J8f7A48vlBTzx8XIu6pvGyEEZXodjjAkDacmJbIjyI4t8wLeXvPbAxmrKTFPVUlVdA+TiJA9EpCnwPnC/qs4KYpx1YuuuYu6YsoDM1Mb82bryMMYEyLmLO7qPLGYDmSLSSUQSgOHAO1XKTAVOAxCRVjjNUqvd8m8Dr6rqG0GMsU5UVCi/eX0h+0rK+NcV/WmYYE+7M8YEpm0zp8uPioqqDS+hJWjJQlXLgFuBGcAy4HVVzRGRh0XkArfYDKBQRJYCnwF3q2ohcBlwCjBKRBa4f32DFevRev6r1Xy9chsPnNeTLqn2ECNjTODSkhMpLVe27T3gdSi1CupPYFWdDkyvMu4Bn9cK/Mb98y0zAZgQzNjqysK8nTw+I5ehPdswYqA9m8IYc3jSmv30EKTUJqH7aGW7g/so7DlQxm2T55PapAF/ucTOUxhjDl/bZPe5FiF+ktsa14/CA9OWkLd9H5NHDya5YYLX4RhjwlDljXmhfkWUHVkcoWkLNvDWvA3cenomAzu18DocY0yYapYUT1J8bMhfEWXJ4gisL9zH/729hOyM5tx2ehevwzHGhDERIS059J9rYcniMJWWV3Db5PmIwFPD+xJnT7wzxhyltOQkNtqRRWR59pMVLMjbyWMXH0v75g29DscYEwHSmiXZkUUk+X7Ndv752UouHdCe83qneR2OMSZCtE1OpGD3AQ6UlXsdSo0sWQSoaH8pd05ZQIcWDXnogp5eh2OMiSCVD0HaUhS6N+ZZsgiAqvKHqUvYvKuYp4b1pXEDu+LYGFN3frwxL4Sfa2HJIgBTF2zgnYUbueOMTOt23BhT59Iqb8yzZBG+1hfu4w9TcxjYsQW/Os0ukzXG1L22Pl1+hCpLFrUoK6/gjinOZbJPDOtjj0c1xgRFUkIsLRolhPQVUdb4XotnP13JvPU7eWZEP7tM1hgTVG2bhfaNeXZkUYNvVm7j2U9XcHG/dlzQxy6TNcYEV6g/BMmSRTXyd+zj1tfm0SW1MY9c1MvrcIwxUSCtWWg/XtWSRRXFpeXcPGEeZeXKc1dm08gukzXG1IO05CR2F5exu7jU61CqFdRkISJDRSRXRFaKyD01lLlMRJaKSI6IvOYz/moRWeH+XR3MOH09OC2HxRuKeGJYXzq1alRfqzXGRLm27o15odoUFbSfzSISC4wBzgLygdki8o6qLvUpkwncC5yoqjtEJNUd3wJ4EMgGFJjrzrsjWPECTPp+PVPm5PHr07twVo/WwVyVMcYcpJ17r8XGnfvJah16j2cO5pHFQGClqq5W1RJgMnBhlTI3AGMqk4CqbnXH/wz4SFW3u9M+AoYGMVYW5O3kwWk5nJKVwh1nZgVzVcYYc4hQv9cimMmiHZDnM5zvjvOVBWSJyDciMktEhh7GvHVm254D3DxhLqlNG/DM8L52P4Uxpt6lNmlAbIyE7F3cwTx7W90eV6tZfyYwBGgPfCUivQKcFxEZDYwGSE9PP6Igy8oruPW1eWzfW8J/bz7BHo9qjPFEXGwMrZs0CNkrooJ5ZJEPdPAZbg9srKbMNFUtVdU1QC5O8ghkXlR1rKpmq2p2SkrKkQW5Yz+rCvby518cS692zY5oGcYYUxfSkpPYFIXNULOBTBHpJCIJwHDgnSplpgKnAYhIK5xmqdXADOBsEWkuIs2Bs91xda5jq0Z8ctepXDKgfTAWb4wxAWubnBSyPc8GLVmoahlwK85OfhnwuqrmiMjDInKBW2wGUCgiS4HPgLtVtVBVtwOP4CSc2cDD7rigaJoYH6xFG2NMwNKSE9lUVExFxSGt7p4L6h1nqjodmF5l3AM+rxX4jftXdd6XgJeCGZ8xxoSStGZJlJRVULi3hJQmDbwO5yB2B7cxxoSItB9vzAu9pihLFsYYEyLaNvvpxrxQY8nCGGNCRLvk0L0xz5KFMcaEiOSG8STGx9iRhTHGmJqJCCd2bkXzRqF3c7D1v22MMSHkxVHHeR1CtezIwhhjjF+WLIwxxvhlycIYY4xfliyMMcb4ZcnCGGOMX5YsjDHG+GXJwhhjjF+WLIwxxvglTi/h4U9ECoB1R7GIVsC2OgonXERbnaOtvmB1jhZHU+cMVfX7qNGISRZHS0TmqGq213HUp2irc7TVF6zO0aI+6mzNUMYYY/yyZGGMMcYvSxY/Get1AB6ItjpHW33B6hwtgl5nO2dhjDHGLzuyMMYY45clC2OMMX5FVbIQkaEikisiK0XknmqmjxKRAhFZ4P5d70Wcdclfnd0yl4nIUhHJEZHX6jvGuhbA+/ykz3u8XER2ehFnXQqgzuki8pmIzBeRRSJyrhdx1qUA6pwhIp+49f1cRNp7EWddEZGXRGSriCypYbqIyDPu9lgkIv3rNABVjYo/IBZYBRwDJAALgR5VyowC/ul1rPVc50xgPtDcHU71Ou5g17lK+V8DL3kddz28z2OBm93XPYC1XsddD3V+A7jafX06MN7ruI+yzqcA/YElNUw/F/gfIMAg4Lu6XH80HVkMBFaq6mpVLQEmAxd6HFOwBVLnG4AxqroDQFW31nOMde1w3+cRwKR6iSx4AqmzAk3d182AjfUYXzAEUucewCfu68+qmR5WVPVLYHstRS4EXlXHLCBZRNrW1fqjKVm0A/J8hvPdcVVd4h7CvSkiHeontKAJpM5ZQJaIfCMis0RkaL1FFxyBvs+ISAbQCfi0HuIKpkDq/BAwUkTygek4R1ThLJA6LwQucV//AmgiIi3rITavBPzZPxLRlCykmnFVrxt+F+ioqr2Bj4FXgh5VcAVS5zicpqghOL+yXxCR5CDHFUyB1LnScOBNVS0PYjz1IZA6jwDGqWp7nOaK8SISzt//QOr8W+BUEZkPnApsAMqCHZiHDuezf9jC+cNyuPIB3yOF9lQ5FFfVQlU94A4+Dwyop9iCxW+d3TLTVLVUVdcAuTjJI1wFUudKwwn/JigIrM7XAa8DqOpMIBGn87lwFcj3eaOqXqyq/YD/c8cV1V+I9e5wPvuHLZqSxWwgU0Q6iUgCzo7iHd8CVdr3LgCW1WN8weC3zsBU4DQAEWmF0yy1ul6jrFuB1BkR6Qo0B2bWc3zBEEid1wNnAIhId5xkUVCvUdatQL7PrXyOnu4FXqrnGOvbO8BV7lVRg4AiVd1UVwuPq6sFhTpVLRORW4EZOFdSvKSqOSLyMDBHVd8BbhORC3AOVbfjXB0VtgKs8wzgbBFZCpQDd6tqoXdRH50A6wxOs8xkdS8jCWcB1vku4HkRuROnaWJUONc9wDoPAR4TEQW+BG7xLOA6ICKTcOrUyj339CAQD6Cq/8E5F3UusBLYB1xTp+sP48+LMcaYehJNzVDGGGOOkCULY4wxflmyMMYY45clC2OMMX5ZsjDGGOOXJQvzIxHpKCKXB1h2ktstyp0i0s3twXW+iHQ+yhjWuvd7HBURaSAiH7txDTva5flZ13QRSXb/fhVA+YO2s4hki8gzh7G+k90egheISDsRefNIY68LIjJORC71U2aUiKQd4fKPanuZumHJwvjqCPhNFiLSBjhBVXur6pPARTh3gfdT1VVBjjFQ/YB4Ve2rqlMCmUFEYo9kRap6rqruBJIBv8mCKttZVeeo6m2HscorgL+7ddugqrXuqAMlIsG872oUcETJgqPfXqYueN3trv0F9w+4CliE06naeHfcOOBSnzJ73P+zgCJgAXAnzl2+LwOLcboxP80ttwjY75Z7ENiM0+/OZ1XWfTPwN5/hUcCz7uupwFwgBxjtU2YtTjcUHfHpihmnn5+H3NedgQ/c+b8CulVZbyrOjUmVdemMc/fyfLcuLwENfNb3APA1MLzKcsYB/8bpsXQ1Tv9CL+Hc2T+umpgn+2yXx3H66nkcWOKud1gN23kI8J47rbHPNl8EXFIlputxbhhdA0z03U5AQ5wuPRYBU4DvgGzf99h9fWll/G4dn3Dr+A+gkVvH2e72urCaz5QA/wSWAu/j3Ax2qTttAPCF+97MANq669uD05XMAiCpunLu/F1w+mVbCMxz37vatlcLnM/SIrdcb3f8Q249Pnffu9u8/i6G+5/nAdhfEN9c6Ol+QVu5wy3c/+OoPln8+CV0h+8CXnZfd8PpMiKRQ3fkDwG/rWb9KTjdSFcO/w84qUosSTg705bu8Fr8J4tPgEz39fHAp9Ws23eHkojTG2eWO/wqcIfP+n5Xw/Ybh5MABKf7513AsThH5HOBvn5ivgT4COcO49bu9mtbzXb2jfWvwFM+05rXEFflzvnHdbrb6Dn3dS+cnggCSRbvAbHu8J+Bke7rZGA50KjK+i/2qVcasNNdZjzwLZDilhuG+6wQnJ12ZSy1lfsO+IXP+9bQz/Z6FnjQfX06sMDnM/kt0MB9bwpxjjQ9/16G61/UdPcRpU7H6VV1G4Cq1tYXfnVOwvkyoqo/iMg6nL6jdgUys6oWiMhqt5+aFUBX4Bt38m0i8gv3dQeczgv9djMiIo2BE4A3RH7sZLOBn9m6AmtUdbk7/ApO1w9PucO1NVO9q6oqIouBLaq62I0jB2dHvaCWeU8CJqnTq+0WEfkCOI7at9+ZOP0cAaDuc0YCdBLwtDvfEhFZFOB8b+hPPe+eDVwgIr91hxOBdA7uJ+0UfqrXRhGp7OK9K06S+sh9b2KB6vomqraciDQB2qnq224digF83uea6nyJW/5TEWkpIs3cae+r0zHoARHZipOw8/1tDFM9SxaRTai+i+Iy3PNV4nwTE2qZ/2hNAS4DfgDedne8Q3B2ioNVdZ+IfI6zU6o2RhbH7PUAAAgfSURBVFfl9Bhgp6r2PYwY/NVjby3TKnshrvB5XTns7/tzJNuvpvcs0Hlr4rvMqtvat/6C0/SV62dd1cUoQI6qDvYzb7XlRKRpDeX9Laum2Hzfr3Jsf3dU7AR3ZPsEuKzygS8i0sIdv5aful+/ELczMmA30MRn/i9xTqYiIlk4vzD97USqegvnBPgIfvoF3wzY4SaKbjiPgKxqC5Dq/lJsAJwHoKq7gDUi8ks3LhGRPn5i+AHoKCJd3OErcdrL61p122+YiMSKSArOL/Lvqynn60Pg1soB+f/27j7UsqoO4/j3SW1S0UlCK7NUcqy0kkEZMMKiMsVe1AhxMlPTKHWsRia0iBAtMytfIEVIjIrwJUOwMo1UNCdLs+bVl9EycowCyRKLfJtff+x183g996wZNHTufD9wmbPXWXvvtfZc9nP2Wvvuk2y3Afu/hSGYSbIHw5DZlL8leVN7Cuuh41ZurgNOah8iSDJ/TJ2bgcNbv15Ne2oxw+/G9kn2betukWTP9t5on8fWa/+3a5Mc0srnJNmKycdr9Hf0ncBDbTt6nhkWs1hVrQa+AtyUZDnDRCYM39XxjiS3MYz5T32yXAE8mWR5ezrphcBmbQjmcoYnlT7GBmjDKHcCO1fVba34WmDzNkxyBsPE5PT1ngBOZxjD/gnDCX/KEcCxrU+r6XxdZhvOOIZh6Golw1XBRRvSj/VRw9N6lyZZleTrwFU8fXPBDQxzI3/l2cd51JeB7do2lvP0iXh9XMhwEl4BnNL2M/X9DacyHMcbGD80NOUMhg8PK5KsasvTXcUwrLiS4QaAm1r/H2eYu/haa/syhiFDGOZGLkqyjGHYaaZ6RzIMUa5gmHN4FZOP12nAPq3+WcBRE/qm58CnzkqzRLv1d4uq+k/7e5frGSb1H3+Bm6ZZwDE8afbYCrgxyRYMY/nHGxR6vnhlIUnqcs5CktRlWEiSugwLSVKXYSFJ6jIsJEldhoUkqcuwkCR1GRaSpC7DQpLUZVhIkroMC0lSl2EhSeoyLCRJXYaFJKnLsJAkdRkWkqQuw0KS1GVYSJK6DAtJUpdhIUnqMiwkSV2GhSSpy7CQJHUZFpKkLsNCktRlWEiSugwLSVKXYSFJ6jIsJEldhoUkqcuwkCR1GRaSpC7DQpLUZVhIkroMC0lSl2EhSeoyLCRJXYaFJKnLsJAkdRkWkqQuw0KS1GVYSJK6DAtJUpdhIUnqMiwkSV2GhSSpy7CQJHUZFpKkLsNCktRlWEiSugwLSVKXYSFJ6jIsJEldhoUkqcuwkCR1GRaSpC7DQpLUZVhIkroMC0lSl2EhSeoyLCRJXYaFJKnLsJAkdRkWkqQuw0KS1GVYSJK6DAtJUpdhIUnqMiwkSV2GhfR/lOTAJPckuS/JqTPUOTfJsvazJsk/pr2/bZIHk3yrLW8zUn9ZkoeSnDehDe9K8uMkK5PcmuSzSTabUP/zrb33JDlghjrvTvK7tv9bkuzWyj/V9jNVvsf6HCe9+KWqXug2SLNSOyGvAfYH1gK3Awur6s4J65wEzK+qj4+UnQ9sD/y9qhaNWecOYHFV3TzmveOBg4ElVbUqydbAZ4D5wGE17QTQTu6XAguAHYFfALtX1VPT6q0BDq6qu5KcACyoqqOTbFtVj7Q6HwROqKoDJx8pbQy8stAmI8kuSe5OcnGSVUl+kOQ9SZYmuTfJglZvQZJfJfl9+/cNrfzkJJe0129p29hqwi4XAPdV1R+r6nHgMoYT9yQLGU7WU23eG3gl8PMZ+jQP2AH45QzvHQa8v6pWAVTVv6rqTOBu4MNjNnkwcFlVPVZV9wP3tX5MV8C27fVc4C9t+4+M1Nm61dMsYFhoU7MbcD7wVuCNwEeAtwNLgC+0OncD+1XVfOBLwJmt/DxgtySHAt8BPllV/06yT5KLx+zrNcADI8trW9lYSXYGdgVuaMsvAb4JfG5CfxYCl0+/QmiOaW1fl+SCJHckOa1dqZwDfPQ5tPk44Joka4EjgbNG+nFikj8AZwOfntB2bUQMC21q7q+qlVW1DlgNXN9OtCuBXVqducAPk6wCzgX2BGjrHA18H7ipqpa28t9W1XFj9pUxZZM+aR8OXDky5HMCcE1VPdBZ59IZ3tsL+DXwAeCJqtobeASYW1UPA9s8hzYvBg6qqp0YgvOc/1WuuqCqXg+cAnxxQtu1ETEstKl5bOT1upHldcDm7fUZwI1V9WaGE+3LRtaZBzzKMJ7fsxZ47cjyTrThmhlMP/HvCyxK8ifgG8DHkox+gt8L2Lyq7phhewGeYriCuraV/aytO4dnHov1bnOS7YG9quo3rehy4G1jtnUZcMgMbdNGxrCQnm0u8GB7ffRUYZK5DENY+wGvSDJuzH/U7cC8JLsmeSlDGFw9rmKbF9kOuHWqrKqOqKrXVdUuDMNk36uq0TuqnjG/McZKhsC5B3hvKzuAIUROAa4cs87VwOFJ5iTZlSEcb5tW52FgbpLd2/L+wF2tH/NG6r0PuHdC+7QRMSykZzsb+GqSpcDoLabnAhdW1RrgWOCsJDvMNGdRVU8Ci4DrGE6mV1TVaoAkp7e7haYsZJhY3pAJ4cOYHBbfZZiz+CmwZbtr6uUMw2qPApeMafNq4ArgToarkROnhsWSXJNkx9avTwA/SrKcYc5ial5lUZLVSZYBJwNHbUB/9CLmrbPSLJZkCcPVxeKq+nOSLYEPATd35kKkZzAspFkuyUEMf1uxA/BPhquRb7cJe2m9GBaSpC7nLCRJXYaFJKnLsJAkdRkWkqQuw0KS1PVfxOJI6U4KUMkAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7f358a978d68>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "resfile = 'results/2ndstage_RetinaNet-MEL-export.pth-CCMCT_MEL-val-inference_results_boxes.p.bz2'\n",
    "databasefile = 'databases/MITOS_WSI_CCMCT_MEL.sqlite'\n",
    "thr, F1scores, thrs = optimize_threshold(databasefile=databasefile, resfile=resfile)\n",
    "\n",
    "import matplotlib.pyplot as plt \n",
    "plt.plot(thrs, F1scores)\n",
    "plt.xlabel('cutoff value for mitotic figure detection')\n",
    "plt.ylabel('F1 score')\n",
    "plt.title('Optimization of detection threshold (trainval set)')\n",
    "plt.text(0.7, 0.5, 'max: %.3f @ %.2f' % (np.max(np.array(F1scores)),np.arange(0.5,0.99,0.01)[np.argmax(F1scores)]))\n",
    "plt.plot([0.5, 1.0], np.max(np.array(F1scores))*np.array([1,1]),'r--')\n",
    "\n",
    "plt.savefig('results/figures/Opt_F1_CCMCT_MEL_RetinaNet_2ndstage.pdf')\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Inference on test set\n",
    "\n",
    "1. MEL on MEL"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Calculating F1 for test set of 11 files\n",
      "Overall: \n",
      "TP: 17467 FP: 5431 FN:  4084 F1: 0.7859344417197237\n",
      "F1 score:  0.786\n"
     ]
    }
   ],
   "source": [
    "resfile = 'results/2ndstage_RetinaNet-MEL-export.pth-CCMCT_MEL-inference_results_boxes.p.bz2'\n",
    "databasefile = 'databases/MITOS_WSI_CCMCT_MEL.sqlite'\n",
    "F1, individ = calculate_F1(databasefile=databasefile, resfile=resfile, det_thres=0.83)\n",
    "\n",
    "print('F1 score: ', np.round(F1,3))\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "2. MEL on HEAEL"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Calculating F1 for test set of 11 files\n",
      "Overall: \n",
      "TP: 17478 FP: 5420 FN:  4093 F1: 0.7860756931795183\n",
      "F1 score:  0.786\n"
     ]
    }
   ],
   "source": [
    "resfile = 'results/2ndstage_RetinaNet-MEL-export.pth-CCMCT_MEL-inference_results_boxes.p.bz2'\n",
    "databasefile = 'databases/MITOS_WSI_CCMCT_HEAEL.sqlite'\n",
    "F1, individ = calculate_F1(databasefile=databasefile, resfile=resfile, det_thres=0.83)\n",
    "\n",
    "print('F1 score: ', np.round(F1,3))\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "3. MEL on ODAEL"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Calculating F1 for test set of 11 files\n",
      "Overall: \n",
      "TP: 18036 FP: 4869 FN:  4440 F1: 0.7948700998215112\n",
      "F1 score:  0.795\n"
     ]
    }
   ],
   "source": [
    "resfile = 'results/2ndstage_RetinaNet-MEL-export.pth-CCMCT_MEL-inference_results_boxes.p.bz2'\n",
    "databasefile = 'databases/MITOS_WSI_CCMCT_ODAEL.sqlite'\n",
    "F1, individ = calculate_F1(databasefile=databasefile, resfile=resfile, det_thres=0.83)\n",
    "\n",
    "print('F1 score: ', np.round(F1,3))\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## HEAEL data set variant"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Finding thresholds"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Optimizing threshold for validation set of 21 files: \n",
      "Best threshold: F1= 0.7173388015493278 Threshold= 0.8000000000000003\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYsAAAEWCAYAAACXGLsWAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4xLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvAOZPmwAAIABJREFUeJzt3Xd8FWXWwPHfSUIKBEINEDokoSNC6HZEwIIuKqBYsIBlre+qq762V3fVbbZddcEGFkDEhgVQQUWldwiCBBAINQQIIARSzvvHTPASktwL5GZukvP9fPLJvXfaeWbmzrnzPDPPiKpijDHGlCTM6wCMMcaEPksWxhhj/LJkYYwxxi9LFsYYY/yyZGGMMcYvSxbGGGP8smRRxkSkqYgcEJHwk5z+gIi0DKWYTmG59UVklojsF5F/BTD+CBH5sSxi8xPHmSKypoyWVWZlFhEVkcSTmK65O21EMcOfEJF3S5g+SkRWiUiDE112CfMcLiJfldb8SljOdyJyc5CXESUiq0UkPpjL8ceShR/ul3WFiBwUke0i8qqI1DyB6X8VkfML3qvqJlWNVdW8k4nHnXb9yUwbrJhOwShgF1BDVf9UmjMWkbEi8pdSmtcxB1FV/UFVW5fGvAstp8SDbgU2CpilqtuhdLadqr6nqheUSnRlrHACUtXDwJvAn72LypJFiUTkT8DfgPuBOKAn0Az4WkQivYytgmgGrFK7M7RUiKM8fqdvAd4JdORKmEwBxgPXi0iUZxGoqv0V8QfUAA4AQwp9HgvsBG503z8BTAbeB/YDi4HT3GHvAPnAIXdeDwDNAQUi3HG+A/4CzHbH+QyoA7wH7AMWAM19lq9AIpDgjl/wd9DZnArQCpgJZOL8cn8PqHkCMSUAU4DdQBow0mf5TwCTgLfd8qYCKSWsx95uGbLc/73dz8cCOcARN47zi5i2jhvHPmA+8BTwo8/wNsDXbpxrCrYVzi9V33l/5lOuD4EMYANwl8+8woGHgXVuuRYBTYBZ7rr5zZ3XUOAcIN1n2rbudtzrro9BPsPGAi8DX7jznQe0KmZdbXKXVbBNewEjgB+BfwJ73LgH+kzzHfBX4Cd3mybi/LB5A9gGbMHZv8Ld8ROB793tsQt4v9C+dSuw1l3Wy4C4w8KAR4CNOPv/20CcO6w5x+4/Ldxl7He3z3+Ad4spc1M37gg/2+5XnF/Wy4HDQATwoM/2WgX8wWe+Izh2XymybECUu906+Ixbz40pHqgFfI6zz+xxXzcutP5vLqZs3YGFOPvvDuA5n2E9cb7ze4FlwDnu538F8oBst/z/8ZlmLXC2Z8dErxYc6n/AACC3YCcuNGwcMMF9/YS7c18BVAHuw/lCV/HZyc/3mbbwF+s7nANyK5wv+SrgF+B89wvxNvBWoZ0+sYiY3vOJKRHo534R6uEc8F7wGddfTN8DrwDRQGf3i9LXp7zZwIU4B9hngLnFrMPa7hfsWrcsV7nv67jDxwJ/KWEbTMRJTNWADjgHvh/dYdWAzcAN7ry74Bz82hc1b5yD3SLgMSASaAmsB/q7w+8HVgCtcQ4ip/nEecw6xydZuNs8DSfRRALn4Ry8WvvEsRvnwBHhbqeJxZT3mO3gfjYCZ/8a6a7v24Ct/H4Q/w4nybR3518F+AQY7a6jeJxEe4s7/gTgf931EQ2cUWjf+hyoiXMQzwAGuMNudMvZEucH00fAO8XsP3OA53D2v7Pc9VFcsrgISC302XH7Bc4+uxQngce4n12J8wMgDCeJ/wY09FlvhZNFcWV7E/irz7h/BKa5r+sAlwNVgerAB8AnPuN+R/HJYg5wrfs6Fujpvm6E80PuQjf2fu77eiXNE+eH011FLass/srjKWtZqQvsUtXcIoZtc4cXWKSqk1U1B+dLEo3zyyFQb6nqOlXNAqYC61T1G3fZHwCnlzSxiPwZ51f2jQCqmqaqX6vqYVXNcGM6O5BARKQJcAbwZ1XNVtWlwOs4B/wCP6rql+q0cbyDc2AtykXAWlV9R1VzVXUCsBq4JIA4wnG+pI+p6m+quhInSRe4GPhVVd9y570Y56zhimJm2Q3ny/ikqh5Rp93nNWCYO/xm4BFVXaOOZaqa6S9OnO0cCzzrzncmzkHpKp9xPlLV+e72fA8nAZ+Ijar6mru+xwENgfo+w8eqaqo7/9rAQOAed73tBJ73KWcOTvVfgrt9CzeeP6uqe1V1E/CtT6zDcX4Zr1fVA8BDwLDCVUIi0hRnXT/q7n+zcM6Wi1MTJ5kE4iVV3ayqhwBU9QNV3aqq+ar6Ps4v7+4lTF9c2cZz7Pa62v0MVc1U1Q9V9aCq7sf55R/QdwlnXSeKSF1VPaCqc93PrwG+dL9D+ar6Nc4ZyIV+5rcfZ315wpJF8XYBdYupH23oDi+wueCFquYD6Ti/eAK1w+f1oSLexxY3oYgMBO4GLiv4EolIvIhMFJEtIrIPeJdjk1tJEoDd7hejwEacX0MFtvu8PghEF7OeEtxpfRWeV3Hq4fxS3uzzme+8mgE9RGRvwR/OAa24K2qaAQmFxn+Y3w+6TXCqNE5UArDZ3e6+cZa0vordnsU4Or2qHnRf+s7Ddx01wzm72OZTztE4ZxjgVDsKMF9EUkXkxuKWVSjWwttyI8728U1aBePtUdXfCo1bnD04v9gD4VtOROQ6EVnqU84OlLyfF1e2mUCMiPQQkWY4SeRjdxlVRWS0iGx0v0uzgJoBXjl4E5AMrBaRBSJysft5M+DKQvviGTjHlZJUx6m28kRlbCgK1BycutHBOFUhAIhINZxfbg/7jNvEZ3gY0BinqgCc09+gEJHWOL80B6uq7xfpGXe5nVQ1U0Quw6k3LlBSTFuB2iJS3SdhNMWpAjpRW3G+GL6aAtMCmDYDpxqwCc7ZSMG0BTYD36tqv2KmL1zGzcAGVU0qZvzNOFWBKwOIzddWoImIhPkkjKY4VYkn6mT3Fd/pNuPst3WLOitW54qjkQAicgbwjYjMUtU0P8sovC2b4myfHTj7e4FtQC0RqeaTMJpSfNmWAy1FJMIn3uLGPfq5e1B/DegLzFHVPBFZipMIT4iq5ovIJJyzix3A5z77/p9wqiZ7qOp2EekMLAlkOaq6FrjKPSYMBiaLSB2cbfSOqo70V85C2gJ+LzEPFjuzKIZbJfR/wL9FZICIVBGR5jjVQukce/VGVxEZ7P66vgfny1pwyrkDp563VIlIDeBTnKqTwlUJ1XEax/aKSCOc+nhfxcbkJp3ZwDMiEi0inXB+Ib13EmF+CSSLyNUiEiEiQ4F2ONU0JXKrXD4CnnB/3bUDrvcZ5XN33te626aKiHQTkbbFlHE+sE9E/iwiMSISLiIdRKSbO/x14CkRSXKvKurkfrGLmpeveTh15Q+4MZyDU8020V8Zi5CBc/HBSe8vqroN+Ar4l4jUEJEwEWklImcDiMiVIlJwcN+Dc2AK5JLpCcC9ItJCRGKBp3Eax49JSKq6EadK5f9EJNJNSMVWO6pqOsdXHwXynanmxp7hlusGnDOLkzUep91juPu6QHWcs/u9IlIbeDzQGYrINSJSz/0RUXBGkIdzpn+JiPR398NoETnHZ7scV373e1yb348rZc6SRQlU9e84ZxD/xLmiYR7Or4K+6lz7XOBTnB2toDF3sNt+Ac6v/Efc0837SjG8Lji/eJ4T54a6AyJywB32f+7wLJyrcD4qNK2/mK7CabTcinM6/rhbr3pC3Dr/i3F+nWXiVIFcrKq7Spzwd3fgVBVsx2n0fMtn3vuBC3Dq4re64/wNp1EVnKuB2rll/MRNPpfgVDFswKlGfB3nogJw2nUm4Rxo97nTx7jDngDGufMaUqiMR4BBOGebu3AuDLhOVVdzgtwqpr8CP7nLOpF2L1/X4TS2r8LZJyfzexVHN2Ceu69MAe5W1Q0BzPNNnB9Is3DWXzZwZzHjXg30wGnYfxznIo2SjObYNrFjtl1RE6jqKpxf2XNwDq4dca4IOymqWpD0E3DaDQu8gLMf7MI5UAdyVlxgAJDqrusXgWFuO9Fm4FKcY0sGzjHlfn4/Hr8IXCEie0TkJfezq4FxhY47ZargigpzkkTkCZwrZa7xOhZjyiNx7h1YgvMjbJvX8YQad/0sA85yL1jwhLVZGGM85f5abud1HKHKXT9tvI7DqqGMMcb4ZdVQxhhj/LIzC2OMMX5VmDaLunXravPmzb0OwxhjypVFixbtUtV6/sarMMmiefPmLFy40OswjDGmXBGRku6wP8qqoYwxxvhlycIYY4xfliyMMcb4ZcnCGGOMX5YsjDHG+GXJwhhjjF+WLIwxxvhVYe6zOGXnnHP8Z0OGwO23w8GDcGERTzwcMcL527ULrijiaZ633QZDh8LmzXDttccP/9Of4JJLYM0auOWW44c/8gicfz4sXQr33HP88Kefht69YfZsePjh44e/8AJ07gzffAN/+cvxw0ePhtat4bPP4F9FPFPlnXegSRN4/3149dXjh0+eDHXrwtixzl9hX34JVavCK6/ApEnHD//uO+f/P/8Jnxd6xEVMDEx1e4p+6imYMePY4XXqwIcfOq8fegjmzDl2eOPG8O67zut77nHWoa/kZBgzxnk9ahT8UuhZRZ07O+sP4JprID392OG9esEzzzivL78cMgs9gbVvX3j0Uef1wIFw6NCxwy++GO5ze4cPwr639/qbmNWlL9tXrmXg3x9AAPF5XE/qVSPJHnAhzTI20frx+6kSEUa47wi27zmvy8u+V1CeILJkYUw5p0B2Th77s3PZl53D/uxc3vhkJZ+vqU3DfRl02nPwuGneX7CJGXuW0DIznac3O8/lCQ8TqoSHER4mfPrFKtanx5G0NY0hGQcIDxOqRUVQI7oKURFWIVEZVZiOBFNSUtTu4DaVwY592SzbvJfl6VksS3f+Zx1ynrVVp1okKc1r0a15bbq3qE27hjUIDxPyFfJVyctXVOFwbh4Z+w+zY99htu/LZse+bHbuyybjwGF+O5zHoSN5HMzJ5eAR53XWoRwOHnEeqNe4Vgw9W9ahR4va9GxZhya1q3q5OswpEpFFqpribzw7szAmRKkqW7OySd2SRerWfaRu3cfKLVls35cNOGcCyfWrM7BDA05rUpNuzWvTql41RI5/PHS4QDhClXDnfUxkODWrRpJUv3pAseTlK6u372Pe+t3M25DJNz/vYPIip3qkZb1q9GtXn35t63N601qEh53wY7BNOWBnFsaEiPx8ZcWWLGas3smijbtJ3bqPvQedM4YwgZb1YumQUINOjWtyWpM42jWMIyYy3LNYf9m5nznrMpm5eidz1mWSm6/UrhbJeW3iOb9tfc5pXY/oKt7EZwIX6JmFJQtjPHTgcC4/rt3FzNU7mLk6g10HDhMm0C6hBh0S4mifUIN2CXG0bVidqpGhWxGwLzuHWb9k8M2qHcxcvZN92bnUqlqFYd2bMrxHUxrXsqqqUGXJwpgQlXngMF+t2sHUlduZuy6TI3n5VI+O4OzkevRtG8/ZyfHUrhbpdZgnLScvn7nrM3lv7ia+WrUdgPPb1uf63s3p3apOkdVkxjvWZmFMCNmelc301O1MXbmN+Rt2k6/QtHZVruvVjL5t65PSvBZVwivGVUZVwsM4M6keZybVY8veQ4yft5EJ8zfz1aodJMbHckXXxgzs0IBmdap5Hao5AXZmYUwQrUjP4tlpP/NTmnMfRlJ8LAM7NKB/hwa0a1ij0vzKzs7J44vl23h33kaWbHIu1W3bsAYDOzRgYIcGATe0m9Jn1VDGeGjHvmz+MX0NHy5Op061SK7v1ZyBHRuSGB/rdWie27z7oHuWtZ1FG/cA0KpeNa7q3pRrejazRvEyZsnCGA9k5+Tx2qz1vPr9OnLzlBvPaMEfz21F9egqXocWknbsc6rnPl+2jfm/7iYhLpp7+yUzuEtjuwS3jFiyMKYM5ecrny3fyt+mrmZrVjYDOzTgoYFtaVrHrgIK1Oy0XTw7bTXL07NIrh/Lnwe04bw28ZWmqs4rgSaLoLaoicgAEVkjImki8mARw58XkaXu3y8istdn2PUistb9uz6YcRpzKuasy+SyV37i7olLqVUtkomjevLqNV0tUZyg3ol1+fSPfXj56i4cyc3npnELGTp6LsvT9/qf2ARd0M4sRCQc+AXoB6QDC4CrVHVVMePfCZyuqjeKSG1gIZCC0/XNIqCrqu4pbnl2ZmHK2i879vPs1NXMXL2ThLho/nRBay47vZFVn5SCnLx8Ji7YzIvfrGXPwSPceV4id5ybSEQFuWIslITCpbPdgTRVXe8GNBG4FCgyWQBXAY+7r/sDX6vqbnfar4EBwIQgxmtMQLZnZfP817/wwaLNVIuK4MGBbRjRu7k1zJaiKuFhXNuzGZd2TuDxT1N54Zu1fLcmgxeGdqZ5Xbvk1gvBTBaNgM0+79OBHkWNKCLNgBbAzBKmbVTEdKOAUQBNmzY99YiNKcGR3Hxe/3E9/56RRm5+Pjf0acEd5yZSqxzfQBfqakRX4fmhnTm3TTyPfLyCC1/6gccubsfQbk2sLaOMBTNZFLUli6vzGgZMVtW8E5lWVccAY8CphjqZII0JxA9rM3h8SirrM36jf/v6/O+F7axNogwNOi2BlGa1uO+DZTz40QpmrN7Js4M7Uic2yuvQKo1gVgCmA0183jcGthYz7jCOrWI6kWmNCZptWYf443uLufaN+eTlK2/d0I3R16ZYovBAQs0Y3r2pB49c1Jbv12Rw2Ss/sXN/ttdhVRrBbOCOwGng7gtswWngvlpVUwuN1xqYDrRQNxi3gXsR0MUdbTFOA/fu4pZnDdymNB3OzeOtn37lpRlryctX7jg3kZFntbR2iRCxeNMehr82j8T4WCaO6km1KOu56GR5fumsquYCd+Akgp+BSaqaKiJPisggn1GvAiaqT9Zyk8JTOAlmAfBkSYnCmNKiqny+fCvnP/c9z05dTZ/EunzzP2dzZ98kSxQhpEvTWrw8/HRWbdvH7e8tJicv3+uQKjy7Kc8Y16KNe/jLF6tYsmkvbRpU5+EL23JWcj2vwzIlmDh/Ew9+tIIrujbmH1d0skbvkxAKl84aUy5syjzI36at5osV26hXPYq/Xd6RK7o2sfslyoFh3ZuyLSubF2esJSEumv+5oLXXIVVYlixMpbU9K5tXvktjwvxNRISFcXffJEad1dLqv8uZe85PYntWNi/NTKN+XDTDezTzOqQKyb4VptLZuT+bV79bx3vzNpGfr1yZ0oR7zk+ifo1or0MzJ0FE+OsfOrBzfzaPfrKS+tWjOb9dfa/DqnAsWZhKI/PAYUbPWs/bc34lJ0+5vEsj7jwviSa17TLY8i4iPIyXh3fhqjFzuXPCEj66vTdtG9bwOqwKxRq4TYWXm5fPGz9u4MUZa8nOyeOyzo24s28SLazbiApn575sLvnPj0RFhDPljj7UrGp31/vj+aWzxoSC5el7GfSfn3hm6mp6t6rLV/eezXNDO1uiqKDia0Tz6jVd2Z6VzZ0TlpCXXzF+DIcCSxamQvrtcC5PfraKy17+iV0HDvPfa7ry+vUp9qS6SqBL01o8eWl7fli7i39MX+N1OBWGtVmYCmfm6h08+kkqW/Ye4pqeTXlgQBtq2JPqKpVh3ZuyfEsW//1+HR0a1eDiTgleh1TuWbIwFUbWwRwen7KST5ZuJSk+lsm39iKleW2vwzIeeeKS9qzZvp/7P1hOq3qx1uB9iqwaylQI367ZyQUvfM/ny7dxd98kvrjrTEsUlVxkRBivDu9CjZgIRr2zkL0Hj3gdUrlmycKUa/uzc3jww+Xc8NYC4mKq8Mkf+3Bvv2QiI2zXNr83eO/IOsydE5aQbw3eJ82+Uabcmr1uFwNe+IFJCzdz2zmt+OzOM+jQKM7rsEyI6dK0Fo9d0o4f1u7i3XkbvQ6n3LI2C1Pu5Obl84+v1jD6+/W0qFuND27tTddmtbwOy4Sw4T2a8tWqHTzz5WrOSY6355GcBDuzMOXKrgOHufaN+Yz+fj3DezTly7vOtERh/BIRnh3ckYgw4YEPl1l11EmwZGHKjSWb9nDJv39k8aY9/OvK0/jrHzoSE2nPmDCBSagZwyMXt2Xu+t1WHXUSLFmYkKeqvDt3I0NGzyEiXPjo9t5c3rWx12GZcmhIShPOSq7Hs1NXsynzoNfhlCuWLExIy87J474PlvPIJyvpk1iXz+44g/YJ1ohtTk5BdVSYWHXUibJkYULWr7t+4w+vzObDxenc3TeJN6/vZh3DmVOWUDOGRy5yqqPes+qogFmyMCFp2srtXPLvH9m69xBvjejGvf2SCbMn15lSMrRbE85MqsszU1ezebdVRwXCkoUJKTl5+Tz95c/c+u4iWtSrxhd3ncG5beK9DstUMCLCs5d3IkyE+ycvo6I8qiGYLFmYkLFjXzbDX5vHmFnrubZnMz64tReNa9n18CY4GtWM4eELneqoT5du9TqckGfJwoSE2Wm7uOilH1ixJYsXh3Xmqcs6EBVhl8Wa4BrWrQkdG8Xx7NTVHDyS63U4Ic2ShfFUbl4+z321huFvzCMupgpT7ujDpZ0beR2WqSTCwoTHL2nH9n3Z/Pe7dV6HE9IsWRjPbN17iKtfm8dLM9O4vEtjPrvzDJLqV/c6LFPJpDSvzaDTEhg9az3pe6yxuziWLIwnvl61gwtf+oHUrVk8P/Q0/nnlaVSNtK7KjDceHNgGEXhm6mqvQwlZlixMmTqcm8cTU1IZ+fZCGteK4fO7zuQPp9vd2MZbCTVjuPXsVnyxfBvz1md6HU5IsmRhyszO/dkM+e8cxs7+lRv6NOfD23rTom41r8MyBoBbzmpFQlw0T36+ijy7s/s4lixMmVizfT9/eHk2v+w4wH+v6crjl7S3q51MSImJDOfBC9uSunUfHyzc7HU4IceShQm6Wb9kcMWrs8nJy2fSLb0Y0KGB1yEZU6RLOjUkpVkt/jF9Dfuyc7wOJ6RYsjBBNX7eJm4Yu4BGtWL45I996NjYOgE0oUtEePyS9uw+eIT/zEzzOpyQEtRkISIDRGSNiKSJyIPFjDNERFaJSKqIjPf5PE9Elrp/U4IZpyl9+fnK01/+zMMfr+DMpLpMvq03CTVjvA7LGL86No7jyq6NeeunDdZvlI+gJQsRCQdeBgYC7YCrRKRdoXGSgIeAPqraHrjHZ/AhVe3s/g0KVpym9B3OzeOOCYuPdtvx+nUpxEbZZbGm/Ljn/GTy8tUekuQjmGcW3YE0VV2vqkeAicClhcYZCbysqnsAVHVnEOMxZeDQkTxGvr2IL1ds538vbMuTl7YnItxqO035klAzhgvaNWDSgs1k5+R5HU5ICOa3uBHge0lBuvuZr2QgWUR+EpG5IjLAZ1i0iCx0P78siHGaUrI/O4fr35rPD2sz+PvlnRh5VktErFtxUz5d16sZew7m8Nky62QQgpssijpKFL54OQJIAs4BrgJeF5Ga7rCmqpoCXA28ICKtjluAyCg3oSzMyMgovcjNCdt78AjXvD6PxRv38OKw0xnSrYnXIRlzSnq1qkNifCxvz9loXZgT3GSRDvgeMRoDhVN0OvCpquao6gZgDU7yQFW3uv/XA98BpxdegKqOUdUUVU2pV69e6ZfABCRj/2GGjZnLz9v28+o1XRl0WoLXIRlzykSE63o1Y8WWLJZu3ut1OJ4LZrJYACSJSAsRiQSGAYWvavoEOBdAROriVEutF5FaIhLl83kfYFUQYzUnaeveQwwdPYeNmQd5c0Q3+rWr73VIxpSawV0aExsVwTtzrKE7aMlCVXOBO4DpwM/AJFVNFZEnRaTg6qbpQKaIrAK+Be5X1UygLbBQRJa5nz+rqpYsQsyvu37jyv/OIWP/Yd65qTtnJNX1OiRjSlVsVASDuzTi8+Xb2HXgsNfheEoqSl1cSkqKLly40OswKo012/dzzRvzyM3L5+0be9jNdqbCStu5n/Ofm8X9/Vvzx3MTvQ6n1InIIrd9uER2TaM5YcvT9zJ0zBwEmHRLL0sUpkJLjK9O71Z1GD9vE7l5+V6H4xlLFuaEzN+wm6tfm0dsVASTb+1tDysylcJ1vZqzZe8hZqyuvLeCWbIwAfv+lwyue3Me9WtE8cGtvWhap6rXIRlTJs5vG09CXHSlbui2ZGECMm3ldm4et4CWdWN5/5ZeNIyzfp5M5RERHsbwns34MW0XaTsPeB2OJyxZGL/enbuR299bRMdGcUwY1ZO6sVFeh2RMmRvarQmR4WG8O7dynl1YsjDFys9Xnp26mkc+Wcm5reN556YexMVU8TosYzxRNzaKizo1ZPKidA4czvU6nDJnycIU6XBuHve8v5T/fr+O4T2aMvrarlSznmNNJXdNz6YcOJzL16u2ex1KmbNkYY6TdTCH696Yz5RlW/nzgDb85bIO1nOsMcDpTWpRv0YU01fu8DqUMmc/Fc0xNu8+yA1jF7Ap8yAvDuvMpZ0LdxRsTOUVFiZc0K4Bkxelc+hIHjGRlec58vZz0Ry1cksWg1+dzc592bx9U3dLFMYUoX/7BhzKyWPW2srV07UlCwPAD2szGDp6DlXChA9v603PlnW8DsmYkNSjZW3iYqowPbVytVtYNZTho8XpPDB5OYnxsYy9oTsN4qK9DsmYkFUlPIy+beKZ8fNOcvLyqVJJ2vMqRylNkVSVV79bx/9MWka35rWZdGsvSxTGBOCC9g3IOpTD/A27vQ6lzFiyqKTy8pUnpqTyt2mrGXRaAmNv7EaNaLuHwphAnJ1cj+gqYZWqKsqSRSWUnZPHHeMXM27ORkae2YIXhnYmKqLyXNVhzKmKiQznrKR6fJW6g/z8ivGYB38sWVQyh47kMfLthUxduZ1HLmrL/17UjrCwoh6XbowpSf/2Ddi+L5vlW7K8DqVMWLKoRA4czuX6t+bzY9ou/n5FJ24+s6XXIRlTbvVtG094mFSaqihLFpVE1qEcrn1jHos27uGFoZ0ZktLE65CMKddqVo2kZ8valixMxbHntyMMf30uK7dk8fLVXexmO2NKSf/2DVif8RtpO/d7HUrQWbKo4DL2H2bYmLn8suMAY65NYUCHBl6HZEyFcUE75/s0PbXi9xVlyaIC256VzdAxc9i0+yBvjejGuW3ivQ7JmAqlQVw0pzWpWSmqovwmCxGpKiKPisjsVba9AAAgAElEQVRr7vskEbk4+KGZU5G+5yBDRs9hR1Y2427sTp/Eul6HZEyF1L99fZanZ7F17yGvQwmqQM4s3gIOA73c9+nAX4IWkTllGzN/Y+jouew9eIR3b+5B9xa1vQ7JmAqrf3unKuqrCn52EUiyaKWqfwdyAFT1EGAX5oeotJ0HGDJ6DgeP5DJ+ZE9Ob1rL65CMqdBa1YslMT62wrdbBJIsjohIDKAAItIK50zDhJg12/czbMwc8vKViaN60aFRnNchGVMp9G9fn/m/7mbPb0e8DiVoAkkWjwPTgCYi8h4wA3ggqFGZE7ZySxbDxswhPEyYOKoXrRtU9zokYyqN/u0bkJevfLtmp9ehBE2JyUJEBFgNDAZGABOAFFX9LuiRmYAt3byXq1+bS9XICCbd0ovE+FivQzKmUumQEEetqlX4KS3T61CCpsTnWaiqisgnqtoV+KKMYjInYOGvuxnx1gJqV4tk/MgeNK5V1euQjKl0wsKEXq3qMHvdLlQV53d2xRJINdRcEekW9EjMCZu7PpPr3pxPfPUo3r+lpyUKYzzUu1VdtmVls2HXb16HEhSBJItzgTkisk5ElovIChFZHuzATMl+StvFiLfmk1AzhomjetIwLsbrkIyp1AruZfppXcWsigrksaoDgx6FOSHfrdnJLe8sokXdarx7cw/qxkZ5HZIxlV7zOlVJiItmdtouru3ZzOtwSp3fMwtV3QjUBC5x/2q6n/klIgNEZI2IpInIg8WMM0REVolIqoiM9/n8ehFZ6/5dH1hxKr5vVu1g1NuLaFUvlvEje1qiMCZEiAi9E+syZ31mhXwgUiDdfdwNvAfEu3/visidAUwXDryMc2bSDrhKRNoVGicJeAjoo6rtgXvcz2vjXLLbA+gOPC4ilf7usmkrt3Pru4to27A6E0b2pHa1SK9DMsb46JNYh70Hc1i1bZ/XoZS6QNosbgJ6qOpjqvoY0BMYGcB03YE0VV2vqkeAicClhcYZCbysqnsAVLXgIuX+wNequtsd9jUwIIBlVlhfpW7njvGL6dg4jndu7kFcVXtetjGhpncrt90ibZfHkZS+QJKFAHk+7/MIrLuPRsBmn/fp7me+koFkEflJROaKyIATmBYRGSUiC0VkYUZGRgAhlU9fr9rBH8cvpkOjOMbd2J0a0ZYojAlF9WtEkxgfWyEbuQNp4H4LmCciH7vvLwPeCGC6ohJK4Yq8CCAJOAdoDPwgIh0CnBZVHQOMAUhJSal4lYQ4bRS3v7eIdglxvH2TJQpjQl2fVnWYtDCdI7n5REZUnKdABNLA/RxwA7Ab2APcoKovBDDvdMD32Z2Nga1FjPOpquao6gZgDU7yCGTaCm/Gzzu47b1FtG1Yg7ftjMKYcqF3Yl0O5eSxZNMer0MpVYE0cPcE1qrqS6r6IpAmIj0CmPcCIElEWohIJDAMmFJonE9w7uNAROriVEutB6YDF4hILbdh+wL3s0rj29U7ue3dxbRpUIN3buxBXIwlCmPKg54t6xAmFe9+i0DOkV4FDvi8/839rESqmgvcgXOQ/xmYpKqpIvKkiAxyR5sOZIrIKuBb4H5VzVTV3cBTOAlnAfCk+1ml8K17H0Vyg1jevckas40pT+JiqtCxURyzK1gjdyBtFqKqR9sDVDVfRAKZDlX9Eviy0GeP+bxW4H/cv8LTvgm8GchyKpKvUrfzx/GLSa5f3RKFMeVU78S6vDZrPb8dzqVaVECHy5AXyJnFehG5S0SquH9341QVmVL22bKt3P7eYtonxDF+ZE9qVrX7KIwpj/q0qktuvjJ/Q8WpEAkkWdwK9Aa24DQ89wBGBTOoymjyonTunriELk1r8e7N1kZhTHmW0rwWkRFhFep+C7/nR+6NcsPKIJZKa/y8TTz88QrOSKzLmOu6UjWyYpy2GlNZRVcJp2vTWhWqkTuQq6H+LiI13CqoGSKyS0SuKYvgKoO3ftrAwx+v4Lw28bx+fYolCmMqiD6Jdfh52z4yD1SMp1AHUg11garuAy7GqYZKBu4PalSVxKvfreP/PlvFgPYN+O81XYmuEu51SMaYUtLb7bJ8zvqKcXYRSLIoqDy/EJhQmS5hDRZV5bmvf+Fv01Yz6LQE/nP16RXqTk9jDHRqFEf1qIgK86jVQOo8PhOR1cAh4HYRqQdkBzesiktVeWbqasbMWs/QlCY8Pbgj4WEV7xGMxlR2EeFh9GhZm9nrKkYjdyDdfTwI9AJSVDUHOMjxvceaAOTnK499msqYWeu5vlcznrFEYUyF1rtVXTZmHiR9z0GvQzllAdV9qOoeVc1zX/+mqtuDG1bFk5ev/PnD5bwzdyO3nNWSJwa1J8wShTEVWsGjVmdXgKui7NKbMpCTl8//TFrGZ8u2cnffJO45PwkRSxTGVHRJ8bHEVAnn5wrwMCRLFkF2JDefuyYsYVrqdh4c2IZbz27ldUjGmDISFiYk1Y/llx37vQ7llJ3UJTgi0qa0A6mIjuTmc+eExUxL3c5jF7ezRGFMJZQUX51fdhzwP2KIO9nrNb8q1SgqoIJEMT11B09c0o4bz2jhdUjGGA+0bhBLxv7D7D14xOtQTkmx1VAi8lJxg4CawQmnYiicKEb0sURhTGWVVL86AL/sOED3FrU9jubklXRmcQOwElhU6G8hUL5TZBBZojDG+Ep2k8Wact5uUVID9wJgparOLjxARJ4IWkTlmCUKY0xhCXHRxEZFsLYCJ4srKOZObVW1o2AhliiMMUURqRhXRJVUDRWrquX/tsMyYInCGFOS5PjqrC3nV0SVlCw+KXghIh+WQSzlUk6eJQpjTMmSG1Qn87cj7CrH3ZWXlCx8bzFuGexAyqOcvHzuGG+JwhhTsuT6sQDluiqqpGShxbw2WKIwxgSu4Iqo8lwVVVID92kisg/nDCPGfY37XlW1RtCjC1E5efncOX4J01N38LglCmOMH/HVo6gRHVGuL58tNlmoqj22rQi5eb/39fT4Je24wRKFMcYPEaF1g+rl+vJZezzbCcjPVx6YvJypK7fz6MWWKIwxgUuq7/QRpVo+a/UtWQRIVXlsyko+WrKF+y5I5ibr68kYcwKS42PJOpRDxv7yeUWUJYsAqCrPTlvNu3M3cevZrfjjuYleh2SMKWeSG5Tvbj8sWQTg5W/TGP39eq7t2Yw/D2htDy4yxpywZJ8OBcsjSxZ+vPnjBv751S8MPr0R/zeovSUKY8xJqRsbRe1qkeW2kduSRQkmLdjMk5+vYkD7Bvz9ik72zGxjzClJrh9r1VAVzZcrtvHgR8s5K7keL17VmYhwW1XGmFOTXL86aeX0iig7Ahbhh7UZ3D1xCV2a1mL0NV2JirBbTowxpy6pfnX2H85lW1aRHXqHtKAmCxEZICJrRCRNRB4sYvgIEckQkaXu380+w/J8Pp8SzDh9Ldq4h1FvLyIxvjpvjOhGTKQlCmNM6UiOL799RJXU3ccpEZFw4GWgH5AOLBCRKaq6qtCo76vqHUXM4pCqdg5WfEVZs30/N45dQHyNKMbd2I24mCpluXhjTAX3+xVR+zmndbzH0ZyYYJ5ZdAfSVHW9qh4BJgKXBnF5p2RT5kGufWMe0VXCePemHsRXj/Y6JGNMBVOrWiT1qkeVy8tng5ksGgGbfd6nu58VdrmILBeRySLSxOfzaBFZKCJzReSyohYgIqPccRZmZGScdKA792VzzRvzOJKXzzs39aBJ7aonPS9jjClJcv3Ycnn5bDCTRVHXmRa+BOAzoLmqdgK+Acb5DGuqqinA1cALItLquJmpjlHVFFVNqVev3kkFmXUwh2vfmM+uA4cZe0P3o6eJxhgTDMluH1H5+eXriqhgJot0wPdMoTGw1XcEVc1U1YKOUl4DuvoM2+r+Xw98B5wejCCP5OUTHRnOa9el0LlJzWAswhhjjkquX51DOXls2XvI61BOSDCTxQIgSURaiEgkMAw45qomEWno83YQ8LP7eS0RiXJf1wX6AIUbxktFvepRfHxbb/ok1g3G7I0x5hjl9al5QUsWqpoL3AFMx0kCk1Q1VUSeFJFB7mh3iUiqiCwD7gJGuJ+3BRa6n38LPFvEVVSlxu7MNsaUlaRy2kdU0C6dBVDVL4EvC332mM/rh4CHiphuNtAxmLEZY4wXakRXoWFctJ1ZGGOMKZnzICRLFsYYY0qQHB9L2s4D5JWjK6IsWRhjTBlLblCdw7n5bNp90OtQAmbJwhhjyphvtx/lhSULY4wpY0luh4Ll6U5uSxbGGFPGqkVFUDc2ivQ95efGPEsWxhjjgYSa0eXquRaWLIwxxgMNakSzLcvOLIwxxpQgoWYM2/bamYUxxpgSNIyLZv/hXPZn53gdSkAsWRhjjAca1owBKDftFpYsjDHGAw3jnKdxWrIwxhhTrKPJopw818KShTHGeKB+jWhEYKudWRhjjClOlfAw4qtH2ZmFMcaYkjWMi7E2C2OMMSVrGFd+bsyzZGGMMR4pOLNQDf3nWliyMMYYjyTUjObgkTz2Hcr1OhS/LFkYY4xHGsY5N+ZtLQdVUZYsjDHGIw1rFtyYZ8nCGGNMMcrTXdyWLIwxxiPx1aMJD5Ny0fusJQtjjPFIeJhQv3qUtVkYY4wpWcNy8lwLSxbGGOOhBnHRbN9nycIYY0wJEuKi2br3UMjfmGfJwpggmjZtGq1btyYxMZFnn322yHHuvfdeOnfuTOfOnUlOTqZmzZpHhw0YMICaNWty8cUXHzPNmWeeeXSahIQELrvssmJjWLJkCUOHDqVjx45069aNJ554gkOHiq8jHzduHElJSSQlJTFu3Lgix1m6dCk9e/akc+fOpKSkMH/+fABUlbvuuovExEQ6derE4sWLi12OcTSMi+Fwbj57Dob4E/NUtUL8de3aVY0JJbm5udqyZUtdt26dHj58WDt16qSpqaklTvPSSy/pDTfccPT9N998o1OmTNGLLrqo2GkGDx6s48aNK3LYp59+qr1799bZs2drfn6+Hj58WEePHq3nnnuuZmdnHzd+ZmamtmjRQjMzM3X37t3aokUL3b1793Hj9evXT7/88ktVVf3iiy/07LPPPvp6wIABmp+fr3PmzNHu3buXWF6jOnXFVm325891RfpeT5YPLNQAjrF2ZmEqjV9//ZU2bdpw880306FDB4YPH84333xDnz59SEpKOvrreP78+fTu3ZvTTz+d3r17s2bNGgCee+45brzxRgBWrFhBhw4dOHjwYLHLmz9/PomJibRs2ZLIyEiGDRvGp59+WmKMEyZM4Kqrrjr6vm/fvlSvXr3Y8ffv38/MmTOLPLPYu3cvTz75JNOnT6dXr16ICJGRkYwaNYrhw4fz0ksvHTfN9OnT6devH7Vr16ZWrVr069ePadOmHTeeiLBv3z4AsrKySEhIAODTTz/luuuuQ0To2bMne/fuZdu2bSWWubIruIs71O+1CGqyEJEBIrJGRNJE5MEiho8QkQwRWer+3ewz7HoRWev+XR/MOE3lkZaWxt13383y5ctZvXo148eP58cff+Sf//wnTz/9NABt2rRh1qxZLFmyhCeffJKHH34YgHvuuYe0tDQ+/vhjbrjhBkaPHk3VqlVZuHAhN99883HL2rJlC02aNDn6vnHjxmzZsqXY2DZu3MiGDRs477zzAi7Pxx9/TN++falRo8ZxwyZNmsQtt9xCbGwsTz31FF26dOH+++/npptu4vrrr2fq1KknHfMLL7zA/fffT5MmTbjvvvt45plnTqrM5vcb87aH+OWzQUsWIhIOvAwMBNoBV4lIuyJGfV9VO7t/r7vT1gYeB3oA3YHHRaRWsGI1lUeLFi3o2LEjYWFhtG/fnr59+yIidOzYkV9//RVwfilfeeWVdOjQgXvvvZfU1FQAwsLCGDt2LNdeey1nn302ffr0ASAlJYXXX3/9uGVpEQ2WIlJsbBMnTuSKK64gPDw84PIUPhPxtWzZMnr27MmyZctYunQpCxcupFevXqxbt46IiIgipwk05ldffZXnn3+ezZs38/zzz3PTTTed0PTmd3Vjo6gSLiH/xLxgnll0B9JUdb2qHgEmApcGOG1/4GtV3a2qe4CvgQFBitNUIlFRUUdfh4WFHX0fFhZGbq7T8+ejjz7Kueeey8qVK/nss8/Izv79S7x27VpiY2PZunWr32U1btyYzZs3H32fnp5+tLqmKBMnTiz2wF+UzMxM5s+fz0UXXVTkcFUlPDyc1atX069fP8LCwhg4cGCpxDxu3DgGDx4MwJVXXnm0Cu9Ey2wgLEyoXyM65J+YF8xk0QjY7PM+3f2ssMtFZLmITBaRgvPXQKc1ptRlZWXRqJGzu40dO/aYz++++25mzZpFZmYmkydPLnE+3bp1Y+3atWzYsIEjR44wceJEBg0aVOS4a9asYc+ePfTq1SvgOD/44AMuvvhioqOjixzesWNH5syZQ+vWrZkxYwb5+flMnz4dcA72BWdGvvr3789XX33Fnj172LNnD1999RX9+/c/bryEhAS+//57AGbOnElSUhIAgwYN4u2330ZVmTt3LnFxcTRs2DDgMlVWCXExlfrMoqhzz8LnqJ8BzVW1E/ANUHCdXiDTIiKjRGShiCzMyMg4pWCNKfDAAw/w0EMP0adPH/Ly8o5+fu+993L77beTnJzMG2+8wYMPPsjOnTuLbbOIiIjgP//5D/3796dt27YMGTKE9u3bA/DYY48xZcqUo+NOmDCBYcOGHVdlc+aZZ3LllVcyY8YMGjdufPRgD/7PRIYMGcK//vUvEhMTad++PSkpKfz000+oKmvXruXRRx89bpratWvz6KOP0q1bN7p168Zjjz1G7dq1Abj55ptZuHAhAK+99hp/+tOfOO2003j44YcZM2YMABdeeCEtW7YkMTGRkSNH8sorr/hd38a5MS/Ue56VouoYS2XGIr2AJ1S1v/v+IQBVfaaY8cOB3aoaJyJXAeeo6i3usNHAd6o6objlpaSkaMGObIxxvP/++4wePZqXX36Ztm3bkpOTw7Rp02jWrBmdOnXyOjzjembqz7z146+sfmoAYWFl28YjIotUNcXfeEW3cpWOBUCSiLQAtgDDgKt9RxCRhqpacF3dIOBn9/V04GmfRu0LgIeCGKsxFdLQoUNp1qwZDz30EBs3biQ2NpaLLrqICy64wOvQjI+EuBiO5OWT+dsR6lWP8j+BB4KWLFQ1V0TuwDnwhwNvqmqqiDyJcxPIFOAuERkE5AK7gRHutLtF5CmchAPwpKruDlasxlRkPXv25JNPPvE6DFOC359rcajyJQsAVf0S+LLQZ4/5vH6IYs4YVPVN4M1gxmeMMaEgoab7eNW92XRq7HEwxbA7uI0xxmMN4kL/8aqWLIwxxmN1qkUSGRHG9hC+fNaShTHGeExEaBgXHdL3WliyMMaYENAwLrTv4rZkYYwxISAhLiake561ZGGMMSGg4PGqefmh+cQ8SxbGGBMCGtaMIS9f2XXgsNehFMmShTHGhIAE9/LZrSHabmHJwhhjQkCoPzHPkoUxxoSAhnZmYYwxxp+aVasQXSXMziyMMcYUT0RIiIsJ2bu4LVkYY0yIaFgzmq0h2j+UJQtjjAkRDeNi2LbXziyMMcaUoGFcNDv3Z5Obl+91KMexZGGMMSGiYVwM+Qo79ofejXmWLIwxJkQ0rOlcPrs9BNstLFkYY0yISIj7/Yl5ocaShTHGhIhQfmKeJQtjjAkRNaIjqBYZbmcWxhhjiiciDOrciMT4WK9DOU6E1wEYY4z53TODO3odQpHszMIYY4xfliyMMcb4ZcnCGGOMX5YsjDHG+GXJwhhjjF+WLIwxxvhlycIYY4xfliyMMcb4JarqdQylQkQygI2nMIu6wK5SCqe8qGxlrmzlBStzZXEqZW6mqvX8jVRhksWpEpGFqpridRxlqbKVubKVF6zMlUVZlNmqoYwxxvhlycIYY4xflix+N8brADxQ2cpc2coLVubKIuhltjYLY4wxftmZhTHGGL8sWRhjjPGrUiULERkgImtEJE1EHixi+AgRyRCRpe7fzV7EWZr8ldkdZ4iIrBKRVBEZX9YxlrYAtvPzPtv4FxHZ60WcpSmAMjcVkW9FZImILBeRC72IszQFUOZmIjLDLe93ItLYizhLi4i8KSI7RWRlMcNFRF5y18dyEelSqgGoaqX4A8KBdUBLIBJYBrQrNM4I4D9ex1rGZU4ClgC13PfxXscd7DIXGv9O4E2v4y6D7TwGuM193Q741eu4y6DMHwDXu6/PA97xOu5TLPNZQBdgZTHDLwSmAgL0BOaV5vIr05lFdyBNVder6hFgInCpxzEFWyBlHgm8rKp7AFR1ZxnHWNpOdDtfBUwok8iCJ5AyK1DDfR0HbC3D+IIhkDK3A2a4r78tYni5oqqzgN0ljHIp8LY65gI1RaRhaS2/MiWLRsBmn/fp7meFXe6ewk0WkSZlE1rQBFLmZCBZRH4SkbkiMqDMoguOQLczItIMaAHMLIO4gimQMj8BXCMi6cCXOGdU5VkgZV4GXO6+/gNQXUTqlEFsXgl43z8ZlSlZSBGfFb5u+DOguap2Ar4BxgU9quAKpMwROFVR5+D8yn5dRGoGOa5gCqTMBYYBk1U1L4jxlIVAynwVMFZVG+NUV7wjIuX5+x9Ime8DzhaRJcDZwBYgN9iBeehE9v0TVp53lhOVDvieKTSm0Km4qmaq6mH37WtA1zKKLVj8ltkd51NVzVHVDcAanORRXgVS5gLDKP9VUBBYmW8CJgGo6hwgGqfzufIqkO/zVlUdrKqnA//rfpZVdiGWuRPZ909YZUoWC4AkEWkhIpE4B4opviMUqt8bBPxchvEFg98yA58A5wKISF2caqn1ZRpl6QqkzIhIa6AWMKeM4wuGQMq8CegLICJtcZJFRplGWboC+T7X9Tl7egh4s4xjLGtTgOvcq6J6Almquq20Zh5RWjMKdaqaKyJ3ANNxrqR4U1VTReRJYKGqTgHuEpFBOKequ3Gujiq3AizzdOACEVkF5AH3q2qmd1GfmgDLDE61zER1LyMpzwIs85+A10TkXpyqiRHluewBlvkc4BkRUWAW8EfPAi4FIjIBp0x13banx4EqAKr6X5y2qAuBNOAgcEOpLr8c7y/GGGPKSGWqhjLGGHOSLFkYY4zxy5KFMcYYvyxZGGOM8cuShTHGGL8sWZijRKS5iFwd4LgT3G5R7hWRNm4PrktEpNUpxvCre7/HKRGRKBH5xo1r6KnOz8+yvhSRmu7f7QGMf8x6FpEUEXnpBJZ3pttD8FIRaSQik0829tIgImNF5Ao/44wQkYSTnP8prS9TOixZGF/NAb/JQkQaAL1VtZOqPg9chnMX+Omqui7IMQbqdKCKqnZW1fcDmUBEwk9mQap6oaruBWoCfpMFhdazqi5U1btOYJHDgX+6ZduiqiUeqAMlIsG872oEcFLJglNfX6Y0eN3trv0F9w+4DliO06naO+5nY4ErfMY54P6fC2QBS4F7ce7yfQtYgdON+bnueMuBQ+54jwPbcfrd+bbQsm8D/u7zfgTwb/f1J8AiIBUY5TPOrzjdUDTHpytmnH5+nnBftwKmudP/ALQptNx4nBuTCsrSCufu5SVuWd4EonyW9xjwIzCs0HzGAq/i9Fi6Hqd/oTdx7uwfW0TME33Wyz9w+ur5B7DSXe7QYtbzOcDn7rBYn3W+HLi8UEw349wwugF4z3c9AVVxuvRYDrwPzANSfLex+/qKgvjdMj7nlvFfQDW3jAvc9XVpEfuUAP8BVgFf4NwMdoU7rCvwvbttpgMN3eUdwOlKZikQU9R47vSJOP2yLQMWu9uupPVVG2dfWu6O18n9/Am3HN+52+4ur7+L5f3P8wDsL4gbF9q7X9C67vva7v+xFJ0sjn4J3fd/At5yX7fB6TIimuMP5E8A9xWx/Ho43UgXvJ8KnFEolhicg2kd9/2v+E8WM4Ak93UPYGYRy/Y9oETj9MaZ7L5/G7jHZ3kPFLP+xuIkAMHp/nkf0BHnjHwR0NlPzJcDX+PcYVzfXX8Ni1jPvrH+DXjBZ1itYuIqODgfXaa7jka7rzvg9EQQSLL4HAh33z8NXOO+rgn8AlQrtPzBPuVKAPa686wCzAbqueMNxX1WCM5BuyCWksabB/zBZ7tV9bO+/g087r4+D1jqs0/OBqLcbZOJc6bp+feyvP5Vmu4+KqnzcHpV3QWgqiX1hV+UM3C+jKjqahHZiNN31L5AJlbVDBFZ7/ZTsxZoDfzkDr5LRP7gvm6C03mh325GRCQW6A18IHK0k80oP5O1Bjao6i/u+3E4XT+84L4vqZrqM1VVEVkB7FDVFW4cqTgH6qUlTHsGMEGdXm13iMj3QDdKXn/n4/RzBIC6zxkJ0BnAi+50K0VkeYDTfaC/97x7ATBIRO5z30cDTTm2n7Sz+L1cW0WkoIv31jhJ6mt324QDRfVNVOR4IlIdaKSqH7tlyAbw2c7Flflyd/yZIlJHROLcYV+o0zHoYRHZiZOw0/2tDFM0SxYVm1B0F8W5uO1V4nwTI0uY/lS9DwwBVgMfuwfec3AOir1U9aCIfIdzUCoyRlfB8DBgr6p2PoEY/JXjtxKGFfRCnO/zuuC9v+/Pyay/4rZZoNMWx3eehde1b/kFp+prjZ9lFRWjAKmq2svPtEWOJyI1ihnf37yKi813e+Vhx7tTYg3cFdsMYEjBA19EpLb7+a/83v36pbidkQH7geo+08/CaUxFRJJxfmH6O4gU9hFOA/hV/P4LPg7Y4yaKNjiPgCxsBxDv/lKMAi4GUNV9wAYRudKNS0TkND8xrAaai0ii+/5anPry0lbU+hsqIuEiUg/nF/n8Isbz9RVwR8EbEal1Asv/EScxIyLtcKrMCuwQkbZuL6x/KGpi13TgTvdHBCJyehHjzAKGueVqiNtrMc6+UU9EernTVhGR9u4w3zIXOZ67bdNF5DL38ygRqUrJ68t3Hz0H2OXOx5QySxYVmKqmAn8FvheRZTgNmUt+UGMAAAFASURBVOA8q+NsEZmPU+df8MtyOZArIsvc3klfAcLdKpj3cXoqPcwJcKtRVgHNVHW++/E0IMKtJnkKp2Gy8HQ5wJM4ddif4xzwCwwHbnLLlIqfx2W61Rk34FRdrcA5K/jviZQjEOr01vuTiKwUkX8AH/P7xQUzcdpGtnP8evb1F6CWO49l/H4gDsQrOAfh5cCf3eUUPL/hQZz1OJOiq4YKPIXz42G5iKx03xf2MU614gqcCwC+d8t/BKft4m9u7EtxqgzBaRv5r4gsxal2Km68a3GqKJfjtDk0oOT19QSQ4o7/LHB9CWUzp8B6nTWmgnAv/a2iqtnu/S4zcBr1j3gcmqkArA7PmIqjKvCtiFTBqcu/zRKFKS12ZmGMMcYva7MwxhjjlyULY4wxflmyMMYY45clC2OMMX5ZsjDGGOPX/wPmhQlORdCHkAAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7f35e00e84a8>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "resfile = 'results/2ndstage_RetinaNet-HEAEL-export.pth-CCMCT_HEAEL-val-inference_results_boxes.p.bz2'\n",
    "databasefile = 'databases/MITOS_WSI_CCMCT_HEAEL.sqlite'\n",
    "thr, F1scores, thrs = optimize_threshold(databasefile=databasefile, resfile=resfile)\n",
    "\n",
    "import matplotlib.pyplot as plt \n",
    "plt.plot(thrs, F1scores)\n",
    "plt.xlabel('cutoff value for mitotic figure detection')\n",
    "plt.ylabel('F1 score')\n",
    "plt.title('Optimization of detection threshold (trainval set)')\n",
    "plt.text(0.7, 0.5, 'max: %.3f @ %.2f' % (np.max(np.array(F1scores)),np.arange(0.5,0.99,0.01)[np.argmax(F1scores)]))\n",
    "plt.plot([0.5, 1.0], np.max(np.array(F1scores))*np.array([1,1]),'r--')\n",
    "\n",
    "plt.savefig('results/figures/Opt_F1_CCMCT_HEAEL_RetinaNet_2ndstage.pdf')\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Inference on test set"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "1. HEAEL on HEAEL"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Calculating F1 for test set of 11 files\n",
      "Overall: \n",
      "TP: 16265 FP: 5223 FN:  5306 F1: 0.7554750458672984\n",
      "F1 score:  0.755\n"
     ]
    }
   ],
   "source": [
    "resfile = 'results/2ndstage_RetinaNet-HEAEL-export.pth-CCMCT_HEAEL-inference_results_boxes.p.bz2'\n",
    "databasefile = 'databases/MITOS_WSI_CCMCT_HEAEL.sqlite'\n",
    "F1, individ = calculate_F1(databasefile=databasefile, resfile=resfile, det_thres=0.79)\n",
    "\n",
    "print('F1 score: ', np.round(F1,3))\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "2. HEAEL on MEL"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Calculating F1 for test set of 11 files\n",
      "Overall: \n",
      "TP: 16256 FP: 5233 FN:  5295 F1: 0.7553903345724907\n",
      "F1 score:  0.755\n"
     ]
    }
   ],
   "source": [
    "resfile = 'results/2ndstage_RetinaNet-HEAEL-export.pth-CCMCT_HEAEL-inference_results_boxes.p.bz2'\n",
    "databasefile = 'databases/MITOS_WSI_CCMCT_MEL.sqlite'\n",
    "F1, individ = calculate_F1(databasefile=databasefile, resfile=resfile, det_thres=0.79)\n",
    "\n",
    "print('F1 score: ', np.round(F1,3))\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "3. HEAEL on ODAEL"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Calculating F1 for test set of 11 files\n",
      "Overall: \n",
      "TP: 16803 FP: 4693 FN:  5673 F1: 0.7642590739561539\n",
      "F1 score:  0.764\n"
     ]
    }
   ],
   "source": [
    "resfile = 'results/2ndstage_RetinaNet-HEAEL-export.pth-CCMCT_HEAEL-inference_results_boxes.p.bz2'\n",
    "databasefile = 'databases/MITOS_WSI_CCMCT_ODAEL.sqlite'\n",
    "F1, individ = calculate_F1(databasefile=databasefile, resfile=resfile, det_thres=0.79)\n",
    "\n",
    "print('F1 score: ', np.round(F1,3))\n"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}