Results_InvestigateInterlabelConsistency.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Investigation of intra-label consistency\n",
    "\n",
    "In this experiment, we investigate model scores (as given by the classification head of a RetinaNet model).\n",
    "\n",
    "For this purpose, we trained networks in three-fold cross-validation on the training sets of both label sets. \n",
    "The training of the networks can be seen in the following notebooks:\n",
    "\n",
    "TUPAC original labels:\n",
    "- RetinaNet-TUPAC-CrossValidationTrainingset_Batch1.ipynb\n",
    "- RetinaNet-TUPAC-CrossValidationTrainingset_Batch2.ipynb\n",
    "- RetinaNet-TUPAC-CrossValidationTrainingset_Batch3.ipynb\n",
    "\n",
    "Alternative Labels:\n",
    "- RetinaNet-TUPAC_AL-CrossValidationTrainingset_Batch1.ipynb\n",
    "- RetinaNet-TUPAC_AL-CrossValidationTrainingset_Batch2.ipynb\n",
    "- RetinaNet-TUPAC_AL-CrossValidationTrainingset_Batch3.ipynb\n",
    "\n",
    "The inference was run as:\n",
    "\n",
    "``python Inference-LowThres.py RetinaNet-TUPAC-CrossValidationTrainingset-Batch1-512s.pth TUPAC_stitched.sqlite test_lowcut_b1 test slidelist_test_TUPAC_CrossValidataion_batch1.p stitched/\n",
    "python Inference-LowThres.py RetinaNet-TUPAC-CrossValidationTrainingset-Batch2-512s.pth TUPAC_stitched.sqlite test_lowcut_b2 test slidelist_test_TUPAC_CrossValidataion_batch2.p stitched/\n",
    "python Inference-LowThres.py RetinaNet-TUPAC-CrossValidationTrainingset-Batch3-512s.pth TUPAC_stitched.sqlite test_lowcut_b3 test slidelist_test_TUPAC_CrossValidataion_batch3.p stitched/``\n",
    "\n",
    "and, respectively:\n",
    "\n",
    "``python Inference-LowThres.py RetinaNet-TUPAC_AL-CrossValidationTrainingset-Batch1-512s.pth TUPAC_stitched.sqlite test_al_lowcut_b1 test slidelist_test_TUPAC_CrossValidataion_batch1.p stitched/\n",
    "python Inference-LowThres.py RetinaNet-TUPAC_AL-CrossValidationTrainingset-Batch2-512s.pth TUPAC_stitched.sqlite test_al_lowcut_b2 test slidelist_test_TUPAC_CrossValidataion_batch2.p stitched/\n",
    "python Inference-LowThres.py RetinaNet-TUPAC_AL-CrossValidationTrainingset-Batch3-512s.pth TUPAC_stitched.sqlite test_al_lowcut_b3 test slidelist_test_TUPAC_CrossValidataion_batch3.p stitched/```\n",
    "\n",
    "Here, we match the mitotic figures of the respective data set to the scores predicted by the models."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "%reload_ext autoreload\n",
    "%autoreload 2\n",
    "%matplotlib inline\n",
    "from SlideRunner.dataAccess.database import *\n",
    "import os\n",
    "import sys"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We define a matching function, based on a KDtree with a distance(radius) of 25 pixels."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.neighbors import KDTree\n",
    "\n",
    "def match(dataset1 : np.ndarray, dataset2 : np.ndarray):\n",
    "\n",
    "\n",
    "        center1_x = dataset1[:, 0] if dataset1.shape[0]>0 else np.array([])\n",
    "        center1_y = dataset1[:, 1] if dataset1.shape[0]>0 else np.array([])\n",
    "\n",
    "        center2_x = dataset2[:, 0] if dataset2.shape[0]>0 else np.array([])\n",
    "        center2_y = dataset2[:, 1] if dataset2.shape[0]>0 else np.array([])\n",
    "        \n",
    "        uids1 = dataset1[:,2] if dataset1.shape[0]>0 else np.array([])\n",
    "        uids2 = dataset2[:,2] if dataset2.shape[0]>0 else np.array([])\n",
    "        \n",
    "        mapping_DS1_to_DS2 = {}\n",
    "        mapping_DS2_to_DS1 = {}\n",
    "        \n",
    "        isDS1 = np.zeros(center1_x.shape[0]+center2_x.shape[0])\n",
    "        isDS1[0:center1_x.shape[0]]=1 # mark as detection, rest ist GT\n",
    "\n",
    "        center_x = np.hstack((center1_x, center2_x))\n",
    "        center_y = np.hstack((center1_y, center2_y))\n",
    "        uids = np.hstack((uids1, uids2))\n",
    "\n",
    "        radius=25\n",
    "        \n",
    "        # set up kdtree \n",
    "        X = np.dstack((center_x, center_y))[0]\n",
    "\n",
    "        if (X.shape[0]==0):\n",
    "            return 0,0,0,0,({},{},[],[])\n",
    "\n",
    "        \n",
    "        try:\n",
    "            tree = KDTree(X)\n",
    "        except:\n",
    "            print('Shapes of X: ',X.shape)\n",
    "            raise Error()\n",
    "\n",
    "        ind = tree.query_radius(X, r=radius)\n",
    "\n",
    "        annotationMatches = {x: 0 for x in np.where(isDS1==0)[0]}\n",
    "        DetectionMatchesAnnotation = {x: 0 for x in np.where(isDS1==1)[0]}\n",
    "\n",
    "        # check: already used results\n",
    "        alreadyused=[]\n",
    "        for i in ind:\n",
    "            if len(i)==0:\n",
    "                continue\n",
    "            if np.any(isDS1[i]) and np.any(isDS1[i]==0):\n",
    "                # at least 1 from DS1 and one from DS2 --> mark all as matches\n",
    "                fromDS1 = i[isDS1[i]==1][0]\n",
    "                fromDS2 = i[isDS1[i]==0][0]\n",
    "                for j in range(len(i)):\n",
    "                    if not isDS1[i][j]: # is from DS2\n",
    "                        if i[j] not in annotationMatches:\n",
    "                            print('Missing key ',j, 'in annotationWasDetected')\n",
    "                            raise ValueError('Ijks')\n",
    "                        annotationMatches[i[j]] = 1\n",
    "                        mapping_DS2_to_DS1[uids[i[j]]] = uids[fromDS1]\n",
    "                        mapping_DS1_to_DS2[uids[fromDS1]] = uids[i[j]]\n",
    "                        \n",
    "                    else:\n",
    "                        if i[j] not in DetectionMatchesAnnotation:\n",
    "                            print('Missing key ',j, 'in DetectionMatchesAnnotation')\n",
    "                            raise ValueError('Ijks')\n",
    "\n",
    "                        DetectionMatchesAnnotation[i[j]] = 1\n",
    "\n",
    "        \n",
    "        matching = np.sum([annotationMatches[x]==1 for x in annotationMatches.keys()])\n",
    "        nonlyDS2 = np.sum([annotationMatches[x]==0 for x in annotationMatches.keys()])\n",
    "\n",
    "        nonlyDS1 = np.sum([DetectionMatchesAnnotation[x]==0 for x in DetectionMatchesAnnotation.keys()])\n",
    "        onlyDS1 = uids[isDS1==1][[DetectionMatchesAnnotation[x]==0 for x in DetectionMatchesAnnotation.keys()]]\n",
    "        onlyDS2 = uids[isDS1==0][[annotationMatches[x]==0 for x in annotationMatches.keys()]]\n",
    "        \n",
    "        F1 = 2*matching/(2*matching + nonlyDS1 + nonlyDS2)\n",
    "\n",
    "        assert(matching==len(mapping_DS2_to_DS1.keys()))\n",
    "        assert(nonlyDS1 == len(onlyDS1))\n",
    "        assert(nonlyDS2 == len(onlyDS2))\n",
    "        return F1, matching, nonlyDS1, nonlyDS2, (mapping_DS1_to_DS2, mapping_DS2_to_DS1,onlyDS1,onlyDS2)\n",
    "\n",
    "    \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "offprob=0.00"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "    05_stitched1.tif F1: 0.800000, matching:    4, only TUPAC_AL:    0, only TUPAC:    2 (   1 NM)\n",
      "    06_stitched1.tif F1: 0.816901, matching:   87, only TUPAC_AL:   30, only TUPAC:    9 (   8 NM)\n",
      "    13_stitched1.tif F1: 0.500000, matching:    1, only TUPAC_AL:    0, only TUPAC:    2 (   0 NM)\n",
      "    17_stitched1.tif F1: 0.500000, matching:    2, only TUPAC_AL:    0, only TUPAC:    4 (   2 NM)\n",
      "    21_stitched1.tif F1: 0.606250, matching:   97, only TUPAC_AL:  108, only TUPAC:   18 (  17 NM)\n",
      "    23_stitched1.tif F1: 0.724138, matching:   21, only TUPAC_AL:    5, only TUPAC:   11 (   9 NM)\n",
      "              25.tif F1: 0.000000, matching:    0, only TUPAC_AL:    3, only TUPAC:    1 (   0 NM)\n",
      "              26.tif F1: 0.000000, matching:    0, only TUPAC_AL:    4, only TUPAC:    0 (   0 NM)\n",
      "              29.tif F1: 0.800000, matching:    2, only TUPAC_AL:    1, only TUPAC:    0 (   0 NM)\n",
      "              30.tif F1: 0.500000, matching:    1, only TUPAC_AL:    2, only TUPAC:    0 (   0 NM)\n",
      "              32.tif F1: 0.571429, matching:    4, only TUPAC_AL:    6, only TUPAC:    0 (   0 NM)\n",
      "              34.tif F1: 0.608696, matching:    7, only TUPAC_AL:    6, only TUPAC:    3 (   0 NM)\n",
      "              36.tif F1: 0.571429, matching:    4, only TUPAC_AL:    5, only TUPAC:    1 (   1 NM)\n",
      "              41.tif F1: 0.375000, matching:    3, only TUPAC_AL:    9, only TUPAC:    1 (   1 NM)\n",
      "              43.tif F1: 0.333333, matching:    1, only TUPAC_AL:    4, only TUPAC:    0 (   0 NM)\n",
      "              48.tif F1: 0.000000, matching:    0, only TUPAC_AL:    2, only TUPAC:    0 (   0 NM)\n",
      "              49.tif F1: 0.923077, matching:    6, only TUPAC_AL:    0, only TUPAC:    1 (   0 NM)\n",
      "              51.tif F1: 1.000000, matching:    5, only TUPAC_AL:    0, only TUPAC:    0 (   0 NM)\n",
      "              52.tif F1: 0.825397, matching:   26, only TUPAC_AL:   10, only TUPAC:    1 (   1 NM)\n",
      "              59.tif F1: 0.642857, matching:    9, only TUPAC_AL:    8, only TUPAC:    2 (   2 NM)\n",
      "              60.tif F1: 0.769231, matching:    5, only TUPAC_AL:    2, only TUPAC:    1 (   1 NM)\n",
      "              64.tif F1: 0.533333, matching:   16, only TUPAC_AL:   25, only TUPAC:    3 (   3 NM)\n",
      "              69.tif F1: 1.000000, matching:    2, only TUPAC_AL:    0, only TUPAC:    0 (   0 NM)\n"
     ]
    }
   ],
   "source": [
    "import pickle\n",
    "results_testset = pickle.load(open('RetinaNet-TUPAC-CrossValidationTrainingset-Batch1-512s.pth-test_lowcut_b1-inference_results_boxes.p','rb'))\n",
    "DB_TUPAC = Database().open('TUPAC_stitched.sqlite')\n",
    "DB = Database().open('TUPAC_alternativeLabels_augmented.sqlite')\n",
    "mapping_TUPACtoTUPACAL={}\n",
    "detections_probs = []\n",
    "detections_probs_TUPAC = []\n",
    "detections_hardexamples=[]\n",
    "slides_tested = []\n",
    "detections_probs_agreed=[]\n",
    "for [fname,tupacSlideID] in DB_TUPAC.execute('SELECT filename,uid from Slides order by filename').fetchall():\n",
    "    newDBSlide = DB.findSlideWithFilename(fname,'')\n",
    "    DB.loadIntoMemory(newDBSlide)\n",
    "    DB_TUPAC.loadIntoMemory(tupacSlideID)\n",
    "    if fname not in results_testset:\n",
    "        continue\n",
    "    res_testset = results_testset[fname]\n",
    "    probabilities = np.array([[0.5*(x[0]+x[2]),0.5*(x[1]+x[3]),(idx, x[5])] for idx,x in enumerate(res_testset)])\n",
    "    \n",
    "    annos_tupac = np.array([[DB_TUPAC.annotations[x].x1,DB_TUPAC.annotations[x].y1,DB_TUPAC.annotations[x].uid] for x in DB_TUPAC.annotations.keys()])\n",
    "    annos_tupac_al = np.array([[DB.annotations[x].x1,DB.annotations[x].y1,DB.annotations[x].uid] for x in DB.annotations.keys() if DB.annotations[x].agreedClass==1 and DB.annotations[x].deleted==0])\n",
    "    annos_tupac_al_nonmit = np.array([[DB.annotations[x].x1,DB.annotations[x].y1,DB.annotations[x].uid] for x in DB.annotations.keys() if DB.annotations[x].agreedClass==2])\n",
    "    annos_tupac_al_all = np.array([[DB.annotations[x].x1,DB.annotations[x].y1,DB.annotations[x].uid] for x in DB.annotations.keys() if DB.annotations[x].agreedClass>0])\n",
    "\n",
    "    mtch=match(annos_tupac_al,annos_tupac)\n",
    "    mtch_all=match(annos_tupac_al_all,annos_tupac)\n",
    "    mtch2=match(annos_tupac_al_nonmit,annos_tupac)\n",
    "    slides_tested.append(fname)\n",
    "    \n",
    "    if (annos_tupac_al.shape[0]==0):\n",
    "        continue\n",
    "    \n",
    "    (mapping_TUPACtoTUPACAL[fname], mapping_DS2_to_DS1,onlyTUPACAL,onlyTUPAC) = mtch[4]\n",
    "    \n",
    "    annosOnlyInTUPAC_AL = annos_tupac_al[np.in1d(annos_tupac_al[:,2], onlyTUPACAL),:]\n",
    "    annosOnlyInTUPAC = annos_tupac[np.in1d(annos_tupac[:,2], onlyTUPAC),:] if annos_tupac.shape[0]>0 else np.array([])\n",
    "    annosMatched = annos_tupac[np.in1d(annos_tupac[:,2], list(mapping_TUPACtoTUPACAL[fname].values())),:] if annos_tupac.shape[0]>0 else np.array([])\n",
    "    match_probs_TUPACALnewMitosis = match(annosOnlyInTUPAC_AL, probabilities)\n",
    "    match_probs_TUPACnewMitosis = match(annosOnlyInTUPAC, probabilities)\n",
    "    match_probs_TUPACHardExamples = match(annos_tupac_al_nonmit, probabilities)\n",
    "    match_probs_TUPACmatched = match(annosMatched, probabilities)\n",
    "    \n",
    "    print('%20s' % fname,'F1: %4f, matching: %4d, only TUPAC_AL: %4d, only TUPAC: %4d (%4d NM)' % (mtch[0:4]+mtch2[1:2]))\n",
    "    \n",
    "    detections_probs += [x[1] for x in match_probs_TUPACALnewMitosis[4][0].values()]\n",
    "    detections_probs_TUPAC += [x[1] for x in match_probs_TUPACnewMitosis[4][0].values()]\n",
    "    detections_hardexamples += [x[1] for x in match_probs_TUPACHardExamples[4][0].values()]\n",
    "    detections_probs_agreed += [x[1] for x in match_probs_TUPACmatched[4][0].values()]\n",
    "\n",
    "    for k in range(int(match_probs_TUPACALnewMitosis[2])):\n",
    "        detections_probs.append(offprob)\n",
    "    \n",
    "    for k in range(int(match_probs_TUPACnewMitosis[2])):\n",
    "        detections_probs_TUPAC.append(offprob)\n",
    "    \n",
    "    for k in range(int(match_probs_TUPACHardExamples[2])):\n",
    "        detections_hardexamples.append(offprob)\n",
    "\n",
    "    for k in range(int(match_probs_TUPACmatched[2])):\n",
    "        detections_probs_agreed.append(offprob)\n",
    "        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "    01_stitched1.tif F1: 0.690476, matching:   58, only TUPAC_AL:   37, only TUPAC:   15 (  13 NM)\n",
      "    02_stitched1.tif F1: 0.761905, matching:   32, only TUPAC_AL:   15, only TUPAC:    5 (   4 NM)\n",
      "    08_stitched1.tif F1: 1.000000, matching:    3, only TUPAC_AL:    0, only TUPAC:    0 (   0 NM)\n",
      "    09_stitched1.tif F1: 0.666667, matching:    1, only TUPAC_AL:    0, only TUPAC:    1 (   1 NM)\n",
      "    14_stitched1.tif F1: 0.315789, matching:    3, only TUPAC_AL:    0, only TUPAC:   13 (   2 NM)\n",
      "    15_stitched1.tif F1: 0.852941, matching:   58, only TUPAC_AL:   12, only TUPAC:    8 (   7 NM)\n",
      "    16_stitched1.tif F1: 0.666667, matching:    6, only TUPAC_AL:    3, only TUPAC:    3 (   2 NM)\n",
      "              24.tif F1: 0.761905, matching:    8, only TUPAC_AL:    3, only TUPAC:    2 (   1 NM)\n",
      "              27.tif F1: 0.750000, matching:    3, only TUPAC_AL:    2, only TUPAC:    0 (   0 NM)\n",
      "              28.tif F1: 1.000000, matching:    1, only TUPAC_AL:    0, only TUPAC:    0 (   0 NM)\n",
      "              31.tif F1: 0.600000, matching:    6, only TUPAC_AL:    8, only TUPAC:    0 (   0 NM)\n",
      "              33.tif F1: 0.200000, matching:    1, only TUPAC_AL:    3, only TUPAC:    5 (   4 NM)\n",
      "              37.tif F1: 0.400000, matching:    2, only TUPAC_AL:    4, only TUPAC:    2 (   2 NM)\n",
      "              40.tif F1: 0.888889, matching:    4, only TUPAC_AL:    1, only TUPAC:    0 (   0 NM)\n",
      "              42.tif F1: 0.363636, matching:    8, only TUPAC_AL:   28, only TUPAC:    0 (   0 NM)\n",
      "              50.tif F1: 0.444444, matching:    2, only TUPAC_AL:    2, only TUPAC:    3 (   3 NM)\n",
      "              54.tif F1: 0.333333, matching:    1, only TUPAC_AL:    3, only TUPAC:    1 (   1 NM)\n",
      "              66.tif F1: 0.933333, matching:   21, only TUPAC_AL:    2, only TUPAC:    1 (   1 NM)\n",
      "              67.tif F1: 0.786207, matching:   57, only TUPAC_AL:   25, only TUPAC:    6 (   5 NM)\n",
      "              68.tif F1: 0.962963, matching:   39, only TUPAC_AL:    1, only TUPAC:    2 (   1 NM)\n",
      "              73.tif F1: 0.615385, matching:    4, only TUPAC_AL:    4, only TUPAC:    1 (   1 NM)\n"
     ]
    }
   ],
   "source": [
    "import pickle\n",
    "results_testset = pickle.load(open('RetinaNet-TUPAC-CrossValidationTrainingset-Batch2-512s.pth-test_lowcut_b2-inference_results_boxes.p','rb'))\n",
    "DB_TUPAC = Database().open('TUPAC_stitched.sqlite')\n",
    "DB = Database().open('TUPAC_alternativeLabels_augmented.sqlite')\n",
    "mapping_TUPACtoTUPACAL={}\n",
    "#detections_probs = []\n",
    "#detections_probs_TUPAC = []\n",
    "for [fname,tupacSlideID] in DB_TUPAC.execute('SELECT filename,uid from Slides order by filename').fetchall():\n",
    "    newDBSlide = DB.findSlideWithFilename(fname,'')\n",
    "    DB.loadIntoMemory(newDBSlide)\n",
    "    DB_TUPAC.loadIntoMemory(tupacSlideID)\n",
    "    if fname not in results_testset:\n",
    "        continue\n",
    "    res_testset = results_testset[fname]\n",
    "    probabilities = np.array([[0.5*(x[0]+x[2]),0.5*(x[1]+x[3]),(idx, x[5])] for idx,x in enumerate(res_testset)])\n",
    "    \n",
    "    annos_tupac = np.array([[DB_TUPAC.annotations[x].x1,DB_TUPAC.annotations[x].y1,DB_TUPAC.annotations[x].uid] for x in DB_TUPAC.annotations.keys()])\n",
    "    annos_tupac_al = np.array([[DB.annotations[x].x1,DB.annotations[x].y1,DB.annotations[x].uid] for x in DB.annotations.keys() if DB.annotations[x].agreedClass==1 and DB.annotations[x].deleted==0])\n",
    "    annos_tupac_al_nonmit = np.array([[DB.annotations[x].x1,DB.annotations[x].y1,DB.annotations[x].uid] for x in DB.annotations.keys() if DB.annotations[x].agreedClass==2])\n",
    "    annos_tupac_al_all = np.array([[DB.annotations[x].x1,DB.annotations[x].y1,DB.annotations[x].uid] for x in DB.annotations.keys() if DB.annotations[x].agreedClass>0])\n",
    "\n",
    "    mtch=match(annos_tupac_al,annos_tupac)\n",
    "    mtch_all=match(annos_tupac_al_all,annos_tupac)\n",
    "    mtch2=match(annos_tupac_al_nonmit,annos_tupac)\n",
    "    \n",
    "    slides_tested.append(fname)\n",
    "    if (annos_tupac_al.shape[0]==0):\n",
    "        continue\n",
    "    \n",
    "    (mapping_TUPACtoTUPACAL[fname], mapping_DS2_to_DS1,onlyTUPACAL,onlyTUPAC) = mtch[4]\n",
    "    \n",
    "    annosOnlyInTUPAC_AL = annos_tupac_al[np.in1d(annos_tupac_al[:,2], onlyTUPACAL),:]\n",
    "    annosOnlyInTUPAC = annos_tupac[np.in1d(annos_tupac[:,2], onlyTUPAC),:] if annos_tupac.shape[0]>0 else np.array([])\n",
    "    annosMatched = annos_tupac[np.in1d(annos_tupac[:,2], list(mapping_TUPACtoTUPACAL[fname].values())),:] if annos_tupac.shape[0]>0 else np.array([])\n",
    "    match_probs_TUPACALnewMitosis = match(annosOnlyInTUPAC_AL, probabilities)\n",
    "    match_probs_TUPACnewMitosis = match(annosOnlyInTUPAC, probabilities)\n",
    "    match_probs_TUPACHardExamples = match(annos_tupac_al_nonmit, probabilities)\n",
    "    match_probs_TUPACmatched = match(annosMatched, probabilities)\n",
    "    \n",
    "    print('%20s' % fname,'F1: %4f, matching: %4d, only TUPAC_AL: %4d, only TUPAC: %4d (%4d NM)' % (mtch[0:4]+mtch2[1:2]))\n",
    "    \n",
    "    detections_probs += [x[1] for x in match_probs_TUPACALnewMitosis[4][0].values()]\n",
    "    detections_probs_TUPAC += [x[1] for x in match_probs_TUPACnewMitosis[4][0].values()]\n",
    "    detections_hardexamples += [x[1] for x in match_probs_TUPACHardExamples[4][0].values()]\n",
    "    detections_probs_agreed += [x[1] for x in match_probs_TUPACmatched[4][0].values()]\n",
    "\n",
    "    for k in range(int(match_probs_TUPACALnewMitosis[2])):\n",
    "        detections_probs.append(offprob)\n",
    "    \n",
    "    for k in range(int(match_probs_TUPACnewMitosis[2])):\n",
    "        detections_probs_TUPAC.append(offprob)\n",
    "    \n",
    "    for k in range(int(match_probs_TUPACHardExamples[2])):\n",
    "        detections_hardexamples.append(offprob)\n",
    "\n",
    "    for k in range(int(match_probs_TUPACmatched[2])):\n",
    "        detections_probs_agreed.append(offprob)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "    03_stitched1.tif F1: 0.551724, matching:    8, only TUPAC_AL:    3, only TUPAC:   10 (   6 NM)\n",
      "    04_stitched1.tif F1: 0.674033, matching:  183, only TUPAC_AL:  136, only TUPAC:   41 (  38 NM)\n",
      "    07_stitched1.tif F1: 0.625698, matching:   56, only TUPAC_AL:   55, only TUPAC:   12 (  10 NM)\n",
      "    11_stitched1.tif F1: 0.560000, matching:    7, only TUPAC_AL:    3, only TUPAC:    8 (   6 NM)\n",
      "    12_stitched1.tif F1: 0.545455, matching:    3, only TUPAC_AL:    0, only TUPAC:    5 (   5 NM)\n",
      "    18_stitched1.tif F1: 0.745536, matching:  167, only TUPAC_AL:   69, only TUPAC:   45 (  42 NM)\n",
      "    22_stitched1.tif F1: 0.790698, matching:   51, only TUPAC_AL:    6, only TUPAC:   21 (  20 NM)\n",
      "              35.tif F1: 0.000000, matching:    0, only TUPAC_AL:    1, only TUPAC:    0 (   0 NM)\n",
      "              39.tif F1: 0.400000, matching:    1, only TUPAC_AL:    3, only TUPAC:    0 (   0 NM)\n",
      "              44.tif F1: 0.461538, matching:    3, only TUPAC_AL:    7, only TUPAC:    0 (   0 NM)\n",
      "              45.tif F1: 0.470588, matching:    4, only TUPAC_AL:    9, only TUPAC:    0 (   0 NM)\n",
      "              46.tif F1: 0.641509, matching:   17, only TUPAC_AL:   16, only TUPAC:    3 (   0 NM)\n",
      "              47.tif F1: 0.710280, matching:   38, only TUPAC_AL:   31, only TUPAC:    0 (   0 NM)\n",
      "              55.tif F1: 0.666667, matching:    2, only TUPAC_AL:    1, only TUPAC:    1 (   1 NM)\n",
      "              56.tif F1: 0.571429, matching:    2, only TUPAC_AL:    3, only TUPAC:    0 (   0 NM)\n",
      "              57.tif F1: 1.000000, matching:    1, only TUPAC_AL:    0, only TUPAC:    0 (   0 NM)\n",
      "              58.tif F1: 0.666667, matching:    1, only TUPAC_AL:    1, only TUPAC:    0 (   0 NM)\n",
      "              61.tif F1: 0.000000, matching:    0, only TUPAC_AL:    3, only TUPAC:    2 (   1 NM)\n",
      "              62.tif F1: 0.750000, matching:    9, only TUPAC_AL:    4, only TUPAC:    2 (   1 NM)\n",
      "              63.tif F1: 0.642857, matching:    9, only TUPAC_AL:    6, only TUPAC:    4 (   4 NM)\n",
      "              65.tif F1: 0.666667, matching:    5, only TUPAC_AL:    1, only TUPAC:    4 (   4 NM)\n",
      "              71.tif F1: 0.428571, matching:    3, only TUPAC_AL:    3, only TUPAC:    5 (   5 NM)\n",
      "              72.tif F1: 0.742424, matching:   49, only TUPAC_AL:   16, only TUPAC:   18 (   9 NM)\n"
     ]
    }
   ],
   "source": [
    "import pickle\n",
    "results_testset = pickle.load(open('RetinaNet-TUPAC-CrossValidationTrainingset-Batch3-512s.pth-test_lowcut_b3-inference_results_boxes.p','rb'))\n",
    "DB_TUPAC = Database().open('TUPAC_stitched.sqlite')\n",
    "DB = Database().open('TUPAC_alternativeLabels_augmented.sqlite')\n",
    "mapping_TUPACtoTUPACAL={}\n",
    "#detections_probs = []\n",
    "#detections_probs_TUPAC = []\n",
    "for [fname,tupacSlideID] in DB_TUPAC.execute('SELECT filename,uid from Slides order by filename').fetchall():\n",
    "    newDBSlide = DB.findSlideWithFilename(fname,'')\n",
    "    DB.loadIntoMemory(newDBSlide)\n",
    "    DB_TUPAC.loadIntoMemory(tupacSlideID)\n",
    "    if fname not in results_testset:\n",
    "        continue\n",
    "    res_testset = results_testset[fname]\n",
    "    probabilities = np.array([[0.5*(x[0]+x[2]),0.5*(x[1]+x[3]),(idx, x[5])] for idx,x in enumerate(res_testset)])\n",
    "    \n",
    "    annos_tupac = np.array([[DB_TUPAC.annotations[x].x1,DB_TUPAC.annotations[x].y1,DB_TUPAC.annotations[x].uid] for x in DB_TUPAC.annotations.keys()])\n",
    "    annos_tupac_al = np.array([[DB.annotations[x].x1,DB.annotations[x].y1,DB.annotations[x].uid] for x in DB.annotations.keys() if DB.annotations[x].agreedClass==1 and DB.annotations[x].deleted==0])\n",
    "    annos_tupac_al_nonmit = np.array([[DB.annotations[x].x1,DB.annotations[x].y1,DB.annotations[x].uid] for x in DB.annotations.keys() if DB.annotations[x].agreedClass==2])\n",
    "    annos_tupac_al_all = np.array([[DB.annotations[x].x1,DB.annotations[x].y1,DB.annotations[x].uid] for x in DB.annotations.keys() if DB.annotations[x].agreedClass>0])\n",
    "\n",
    "    mtch=match(annos_tupac_al,annos_tupac)\n",
    "    mtch_all=match(annos_tupac_al_all,annos_tupac)\n",
    "    mtch2=match(annos_tupac_al_nonmit,annos_tupac)\n",
    "    \n",
    "    slides_tested.append(fname)\n",
    "    if (annos_tupac_al.shape[0]==0):\n",
    "        continue\n",
    "    \n",
    "    (mapping_TUPACtoTUPACAL[fname], mapping_DS2_to_DS1,onlyTUPACAL,onlyTUPAC) = mtch[4]\n",
    "    \n",
    "    annosOnlyInTUPAC_AL = annos_tupac_al[np.in1d(annos_tupac_al[:,2], onlyTUPACAL),:]\n",
    "    annosOnlyInTUPAC = annos_tupac[np.in1d(annos_tupac[:,2], onlyTUPAC),:] if annos_tupac.shape[0]>0 else np.array([])\n",
    "    annosMatched = annos_tupac[np.in1d(annos_tupac[:,2], list(mapping_TUPACtoTUPACAL[fname].values())),:] if annos_tupac.shape[0]>0 else np.array([])\n",
    "    match_probs_TUPACALnewMitosis = match(annosOnlyInTUPAC_AL, probabilities)\n",
    "    match_probs_TUPACnewMitosis = match(annosOnlyInTUPAC, probabilities)\n",
    "    match_probs_TUPACHardExamples = match(annos_tupac_al_nonmit, probabilities)\n",
    "    match_probs_TUPACmatched = match(annosMatched, probabilities)\n",
    "    \n",
    "    print('%20s' % fname,'F1: %4f, matching: %4d, only TUPAC_AL: %4d, only TUPAC: %4d (%4d NM)' % (mtch[0:4]+mtch2[1:2]))\n",
    "    \n",
    "    detections_probs += [x[1] for x in match_probs_TUPACALnewMitosis[4][0].values()]\n",
    "    detections_probs_TUPAC += [x[1] for x in match_probs_TUPACnewMitosis[4][0].values()]\n",
    "    detections_hardexamples += [x[1] for x in match_probs_TUPACHardExamples[4][0].values()]\n",
    "    detections_probs_agreed += [x[1] for x in match_probs_TUPACmatched[4][0].values()]\n",
    "\n",
    "    for k in range(int(match_probs_TUPACALnewMitosis[2])):\n",
    "        detections_probs.append(offprob)\n",
    "    \n",
    "    for k in range(int(match_probs_TUPACnewMitosis[2])):\n",
    "        detections_probs_TUPAC.append(offprob)\n",
    "    \n",
    "    for k in range(int(match_probs_TUPACHardExamples[2])):\n",
    "        detections_hardexamples.append(offprob)\n",
    "\n",
    "    for k in range(int(match_probs_TUPACmatched[2])):\n",
    "        detections_probs_agreed.append(offprob)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(760, 309)"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(detections_probs), len(detections_probs_TUPAC)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Median model score, agreed mitoses: 0.26642605662345886\n",
      "Median model score, TUPAC-only mitoses: 0.3259458839893341\n",
      "Median model score, hard examples: 0.07257059961557388\n",
      "Median model score, new mitotic fig: 0.26642605662345886\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAxAAAACsCAYAAADrGUpuAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+j8jraAAAgAElEQVR4nO3debgcVZ3/8fcnCTtCWCKEQAhIBMEFmCgojiCbkS2MC6KoBHEQxwUdZwTU34ACCuMo4gZGWYIoYXGLgEIAg6JAANlBJKxJCGtIwhoIfH9/nNOk0unbt2/fXm9/Xs/Tz+3aT90+36o6dU6dUkRgZmZmZmZWi2HtToCZmZmZmXUPFyDMzMzMzKxmLkCYmZmZmVnNXIAwMzMzM7OauQBhZmZmZmY1cwHCzMzMzMxq5gJEk0g6SNJlDVzfv0q6u47lJOlMSU9JmlXvesyaSdJkSVe3YDsNzf+Sxkp6RtLwOpY9XtITkh4ZzHqsdSQ9IGn3Jq5/pqRPNmv9jSLpNEn/r93psKFJUkjaokHrekbS5o1YV17fHZJ2qWO5f5M0J6dnu3rX00lcgGiSiPhFROxZGh5oQJTPHxF/iYgt60jKO4E9gI0j4m2DWI/VIR8sSp9XJD1fGD5I0lmSji9bZlz+/Ufk4QcKyz2al1mzMP/kPP+HKmz/9ZIuyBeqiyTdKuk/SxeqkqZIujunbXKF5TeXdJGkp/M6/rfh/6QWKs//A70gLJ8/Ih6KiDUj4uWBpEPSWOBLwNYRsWG96zFrtkqF+4g4PCKOa1eazGqVj6v3AVQ631ZTaf6I2CYiZtaRlP8DPpvTc9Mg1tMxXIAY+jYFHoiIZ5u9Id89XVE+WKwZEWsCDwH7Fsb9YgCr2jevY3tgAvC1wrSDgQXAx4sLSHodcB0wB3hTRKwNfDAv/5o82y3AfwB/L9+gpJWBGcCVwIbAxsA5A0iz9W0s8GREPNbsDZUKotY5cs2wz7/W9ZyXa7YpcEezN9LS431EdOUHeAD4L+BWYBFwHrBqYfo+wM3AQuBvwJvz+EOA3xfmuwe4oDA8B9i2wvbGAZGXnwM8BRwOvDWnYSHww8L8k4Gr8/c/52WfBZ4BPpTH/zswm3TxNx3YqK/5gV2AuYX1bwL8GngceLK47cI8hwIvAC/n9Xy9wnq2B24CngYuyP/H48v3oTB/AFvk72cBpwKX5LTuDmwE/Cqn637g84Vl3wbcACwGHgW+2+581IY8u3vZuLNK/+8KeW1EpeWAbwMX5e+bAq8A7weWAhsW5jsHuLjGtF0NTC4bdxjwlwHuY8U8Xcg7h5NibiHwI0AV4uVHwHfK1jsd+GIf2wxSIeienI+PA15HivvFwPnAynneV/M/8PP8v3s+x8eX8/j9SAf6hcBM4A19zV/ht1oXOBN4mHSM+G2F9O6e1/FKXs9ZFdazGek48DRwef6fnFO+D5XyFnAscGH+/RcDnwTWBk4H5gPzgOOB4Xn+LYCrSMfRJ4Dz2h0rnfqhynkHWAe4iHTseyp/37iw7EzgBOCv+fffglQ7/I+8rh/m3+GTfWz72JyXz8754g5gQmF6tWPvasDUnK67ct4tngeOAu7N670T+Lc8/g0sfw5ZmMefxbLzxF3APoV1jchp2D4P70iKxYWkGxa7FOadDNyXt3s/cFC7f+Ne+fT1m+dpw4Hv5OPB/cBnWf74VCkvb0W64bQAuBs4oLC+VUh34B8inftPA1YrTP9v0rHpYeATFK4zKqR7Jun49becJ38PrAf8gnS8ux4YV5g/cvoOA14CXiwtV8jjM3P+vAPYL4/va/4HWHasHQ58pfB/vBHYpCy9q+TlS9d091ZYT3/xudz/oyz+dgHmAkcCj5DOU8MKv++TpOPGunn+VUnnhifzPl8PbFBXHmp3Jh5E5n8AmEU6aK6b/+mH52nbAY8BO+Qf+OA8/yrA5vmfNiwv+yDLLig2zz/gsArbG5d/xNPyD7An6cD6W+C1wJi8zZ3z/JMpXHxXyAC7koJz+5yuHwB/rjL/LoV0DicdiE8G1sjpeWcf/6fydBTXs3Le/yOAlYD3kYJlIAWIRcBO+f+5OimA/ieve3PSyeE9ef5rgI/l72sCO7Y7H7Uhzw6qAEEqON4BHJeH/x8wK3+/DfhSYT2PAIfUmLZKBYgzSAejP+S8OpNUk9HXOmrJ0xcBI0l34B8HJpbnNVJB82FyHALrA8/Rx0Eur/d3wFrANsAS4Iqc/9YmnRwPLs//lX4T4PWkg/weOSa+TCoQrdzH/OW/1cWki8p18vI795Hm8nSUr+ca0gl3ZVIzxMUMrADxErA/KS5XA34D/IR0vHgt6dj5qTz/ucBX87x9Hkv86fe8sx6pIL86qYbvAgoFSFL8PJTz6AhgFOmi4wM5r3yRdBOgWgHiBWAv0jngW8C1edowqh97TyQVTtYh1STeWpb/Ppj3aRjphtWzwOjy2CzMfxbLzhP/A/yiMG1v4K78fQzpQmWvvO498vConBcXA1vmeUcD27T7N+6VTz+/+eGk4+bGOc9czooFiGJeXpt0Y/WQPLwd6VywdZ7/ZNJNoHVzbPwe+FaeNpFUqHhjzhO/pP8CxGzSTaLS8f2fpBszI0gF7DML85dfsxxfmLZSXtdXctzsSorJLSvNn8c9wLJj7X+TzrtbAgLeAqzXR7rLr+mK6+kvPvsrQCwFTiKdd1cjXdNdm9e1CunYf26e/1P5/7866TjyL8Ba9eShbq92+n5EPBwRC0j/kG3z+MOAn0TEdRHxckRMJV1U7BipLdzTed53AZcCD0vaCtiZdMf1lSrbPC4iXoiIy0gBd25EPBYR84C/kAKnFgcBZ0TE3yNiCXA08HZJ42pY9m2kwP/viHg2p6eeB1B3JAXc9yPipYj4NenkOBC/i4i/5v/Zm4BREfGNiHgx/69/ChyY530J2ELS+hHxTERcW0eae9VvJS0kXehfBXwzj/846YBL/ltsxrQe6a5OvTYm/XbfJ+W3i4Hf5aZNldSSp0+MiIUR8RDwJ5bF7KsiYhapYLpbHnUgMDMiHq2S1v+NiMURcQdwO3BZRNwXEYtIBaBa4/JDpFqbGRHxEukifjXgHf0tKGk08F7SBeVTOaauqnG7xfWMJdVs/k+Oo6tJJ9+BuCYifpvjci3SBdwX8vHiMdIJvRiXm5Jqi+o9lvSSiuediHgyIn4VEc9FxNOkO7Q7ly17VkTcERFLSXnljoi4MOe175EK/dVcHRGXRHpW5uekCxZI+aXasfcA4Js5X84lxfSrIuKCvE+vRMR5pNq8t9X4//glsJ+k1fPwR0iFUoCPApfkNL8SETNItdB75emvAG+UtFpEzM/xay3Qz29+AHBKRMyNiKdIF7jlinl5Iqmp9JkRsTQibiLVhn1QkkjXZF+MiAU5Nr7J8nnzzIi4PVJT62NrSP6ZEXFv4fh+b0RcntNyAbUf73ck3cw8McfNlaSbXB+ucflPAl+LiLsjuSUinqxx2aKq8VmDV4BjImJJRDxPKgB+Nf9+S0j/0w/k5k0vka4NtsjXxzdGxOI60tz1BYjiwfY5UkaAdDL8kqSFpQ/pzu1GefpVpFLbu/L3maQD/c55uJriRczzFYbXpDal2g8AIuIZ0p2ZMTUsuwnwYA6WwdgImBeRiqXZnAGuozj/psBGZf/3rwAb5OmHku7w/kPS9ZL2qTfhQ8hS0l2QopVIB4RiQXb/iBgZEZtGxH9ExPOSdiI1dZmW5/kl8CZJpYvyJ0l39er1POmC5Q8R8SLpYno94A35AfDSw+B/yPPXkqf7itlyU0kXH+S/P+8nrc2Ky1dIebzWuFyQT7iDsVFez3OFcYONy5WA+YW4/AmpJgJSLYuAWblnkE/Ume5eUTEPS1pd0k8kPShpMakJ2siyZ8OKv8tGxeF8HO7vdy7f9qr5oqC/Y+9y2yrfjqSPS7q5sOwbSTV//YqI2aSamH1zIWI/lt3U2JR0EVlM1ztJd7qfJRXYDyflzYvzjTxrgX5+86r5pcK4TYEdyn7ng0jPzo0it04oTPtjHl9pWw/Sv0Ye7+eU3TR+kNqO95CO+ffWOG+/6SgMD/R4/3hEvFAY3hT4TeH/fRepGeIGpHPppcA0SQ9L+l9J5dcgNRmqD9fNAU6IiBP6mH4VsC/p4uubpCZNBwFvJ7VDbYWHST8yAJLWIF2czath2TnAWEkjBlmImA+MkaRCIaIYEM+SAr+Uxg0rrKO88HF/RIyvtLGIuAf4cH7g6n3AhZLWixY84N3BStXARZux4kGtkoNJF343p5s8y42/mVTt/H5Sm/x63EpqnraCSA+Alz8EPpg8Xe4c4HZJbyG1Uf1tHeuoRZQNP0yqSQPSA4KkmJjXx/xFc4B1JY2MiIWDSNP8vJ7VC4WITQrTy+NyOMtOxiXlcbkEWL/S8SIiHiE9u4KkdwKXS/pzvjC02n2J1JRhh4h4JBfkbyLFaEnxd5lP4Xct5LV6VD325m1tTGruQdl2NyXVVuxGqrl6WdLNhXRXy/Ml55Lu2g4D7izknTnAzyPi3ystFBGXApdKWo3Urv2nwL/WsD0bhBp+81J+KamUL8uPMVdFxB4VtjWMdFG/TaSWGuWWiwNS89ZmqXS830TSsML5diypSVSl+cvNITWlun2Q6eozPrPnKBzzSQWzuYXh8nTOAT4REX/tY3tfB76eWwdcQnpm5fSBJrrbayD68lPgcEk75B4C1pC0t6RSzzNXAe8mPcQzl9T0aCLpYuemJqXpUVK71JJzgUMkbStpFVJB5rqIeKCP+YtmkTLciXnfVs13owfqGlKp9LOSRkiaxPLV1rcA2+Q0rkr/VYuzgKclHSlpNUnDJb1R0lsBJH1U0qgcqKULrP4ukoe6XwF7S9oz/782IvWwNK3aQvn3OIBUNbxt4fM54CP5ruQxwDskfbtU+JO0haRzJI3MwyvndQlYKeel0nHhHGBHSbvni9QvkNq13tVHsvrL0zXLcXk96W7Jr3K1bDOUx9n5pN9jt3xX5kuki++/9TF/Mc3zSdXpP5a0jqSVJL1roAmKiAdJzTyOzb/P20k3PEr+SbrzvHdO49dI7Vz7Wt984DLgO5LWkjRM0usk7Qwg6YOSShcLT5FORr0el/V4DelCaaGkdUnxV83FpOPr+3K8fp50YVCPqsdeUr4+OufLMaSHYkvWIP3mjwNIOoR0N7rkUWBj9d10EdLxak/g0yyrfYB0DNlX0ntymlaVtIukjSVtIGlSvtGwhPSgqfNda/T3m58PHCFpTD5XHNnP+i4CXi/pY/m4t5Kkt0p6Qz7f/xQ4WdJr8/bGSHpPYVuTJW2tVIPVX9wMRvnx+zrSxfmXc5p3IR1rp/Uxf7mfAcdJGp+vNd8sab060lUtPiHdEPxIjqGJrNg0stxpwAm5oIikUfn6DknvlvSmfE5fTGrSVFfcDckCRETcQLqj9kPSCXE26UGw0vR/kg5Wf8nDi0kPnP01mtcP+7HAVKUqpQMi4nLSA7C/IhUGXseyNoErzF9cUU7jvqSeBR4ilURXeAdAf3KzlPeRmhYtJDUVuYh0MC/9n75BupN9D6n9fbX1vUzq/WpbUs8NT5ACbO08y0TgDknPAKcABzbxwrAr5Da/HyY9ELmAVKi7jnSHoJr9SRcrZ0fEI6UP6cHnEaSHk+8l1aqNI/3fF5Hy2w2k54AgXVg+T2rjPyV/f1dO292kPHEaKY4mkXqoeLGPfekvTw/UVFJtQH/NlwbjW8DXcpz9V2Gff0DKv/uSutB9sdL8Fdb3MdIB+R+kThW+UGe6SjWiT5LuzJ7HsrhcROp16mekmpFnWf5uVCUfJz0keCfpt7yQZc3b3gpcl+NyOnBE5H7TbUC+R3pe5gnSA4x/rDZzRDxBepD1RNLvPJ7Uq82A1XDs/QYpj9xPOp5fyLL8dCepx51rSBdMbypLx5WkjhsekfREH9ufn5d/BymvlsbPIR03vkK6WJ1DevB0WP78J+ku8ALSRdGn69l/G5gafvOfks4Nt5Juql5Cam5b8fooP9ewJ+l4/zCpqV3poV5IBZDZwLVKzfsuJ9XWERF/IMXOlXmeKxu0m5WcDmydj9+/zcf1fUnPIz0B/Bj4eET8o9L8Fdb3XdLF/2Wki/HTSceAgeozPrMjcjpLrWX6q5E/hXQsv0zS06Tj0Q552oZ5/YtJNwOvos5zbKkLRTMAJF0HnBYR9TZ7MWuIfPf+HGDT6PEDlaTzgH9ERDPvzlmPkPRp0g2c/u5kmiHpvaTrgk37ndkGrVvic0jWQFjtJO0sacPchOlg4M30c+fMrNly05wjgJ/1YuEhV/+/Ljc3mki6i9us50BsiJM0WtJOOT9tSWqa95t2p8s6U24Gt1e+LhhDalbk/NIk3RqfQ/UhaqvdlqQquDVIzbg+kKujzdpC0htIzaxuIfUr3os2JL0ocj1S1fanI3WNaFaPlUk9b21GagYxjdRcw6wSkZrRnkdq1nox6X0f1hxdGZ9uwmRmZmZmZjVzEyYzMzMzM6tZVzRhWn/99WPcuHHtToa1wI033vhERJT3aW/9cIz0jm6Pkdx94A2kl1juI6n0MsT1gBuBj0XEi7kr4LOBfyH1UvShUpfAko4m9R73MvD5/D6BqhwjvaPbY6QdHB+9pREx0hUFiHHjxnHDDTe0OxnWApJqeQullXGM9I4hECNHkLoPXCsPnwScHBHTJJ1GKhicmv8+FRFbSDowz/chSVuTuovchvQG18slvb6/LrgdI71jCMRIyzk+eksjYsRNmMzMrCXyC+v2Jr2joPT25V1J/ZJDevfH/vn7pDxMnr5bnn8SMC0ilkTE/aS+44svwDQzsyZzAcLMzFrle8CXWfbm0/WAhRGxNA/PBcbk72NILx4jT1+U5391fIVlliPpMEk3SLrh8ccfb+R+mJn1NBcgzMys6STtAzwWETe2apsRMSUiJkTEhFGj3CTezKxRuuIZCDMz63o7AftJ2gtYlfQMxCnASEkjci3DxsC8PP88YBNgrqQRwNqkh6lL40uKy5iZWQt0RQHitnmLGHfUxSuMf+DEvduQGjOrRaWYBcdtr4qIo4GjASTtAvxXRBwk6QLgA6SemA4GfpcXmZ6Hr8nTr4yIkDQd+KWk75Ieoh4PzOpv+z6PmFk5n6fq1xUFCDMzG7KOBKZJOh64CTg9jz8d+Lmk2cACUs9LRMQdks4H7gSWAp/prwcmMzNrLBcgzMyspSJiJjAzf7+PCr0oRcQLwAf7WP4E4ITmpdDMzKrxQ9RmZmZmZlYzFyDMzMzMzKxmLkCYmZmZmVnNXIAwaxNJq0qaJekWSXdI+noev5mk6yTNlnSepJXbnVYzMzOzEhcgzNpnCbBrRLwF2BaYKGlH4CTg5IjYAngKOLSNaTQzMzNbjnthMmuTiAjgmTy4Uv4EsCvwkTx+KnAscGqr02dmZjYU9PW+B6tfVxcg/AIQ63aShgM3AlsAPwLuBRbmt/ICzAXG9LHsYcBhAGPHjm1+Ys3MzMxwEyaztoqIlyNiW2BjUl/4Ww1g2SkRMSEiJowaNappaTQzMzMrcgHCrANExELgT8DbgZGSSrWDGwPz2pYwMzMzszJNLUBI+mLuXeZ2SefmXmfcw4wZIGmUpJH5+2rAHsBdpILEB/JsBwO/a08KzczMzFbUtGcgJI0BPg9sHRHPSzofOBDYi9TDzDRJp5F6mPEDotaLRgNT83MQw4DzI+IiSXcC0yQdD9wEnN7ORLaKn2kyMzPrDs1+iHoEsJqkl4DVgfm4hxkzACLiVmC7CuPvIz0PYWZmZtZxmtaEKSLmAf8HPEQqOCwi9TZTUw8zZmZmZmbWeZpWgJC0DjAJ2AzYCFgDmDiA5Q+TdIOkG15+blGTUmlmZmZmZgPRzIeodwfuj4jHI+Il4NfATtTYw0yxi8rhq6/dxGSamZmZmVmtmlmAeAjYUdLqkgTsBtyJe5gxMzMzM+taTXuIOiKuk3Qh8HdgKak3mSnAxfRgDzNmnaSdPR71tW0zMzPrDk3thSkijgGOKRvtHmbMzMzMzLqU30RtZmZNl18kOkvSLfkFo1/P4yu+XFTSKnl4dp4+rrCuo/P4uyW9pz17ZGbWu1yAMDOzVlgC7BoRbwG2BSZK2hE4ifRy0S2Ap0gvFyX/fSqPPznPh6StSS8l3YbUs9+P88sYzcysRVyAMDOzpovkmTy4Uv4E6eWiF+bxU4H98/dJeZg8fbfcIcckYFpELImI+4HZuFmsDXGSzpD0mKTbC+PWlTRD0j357zp5vCR9P9fS3Spp+/al3IYqFyDMzKwlJA2XdDPwGDADuJe+Xy46BpgDkKcvAtYrjq+wjNlQdRYrvkvrKOCKiBgPXJGHAd4LjM+fw4BTW5RG6yFNfYi6XTqxh5lWbNus1dyjkg1ERLwMbCtpJPAbYKtmbk/SYaQLKIavNaqZmzJrqoj4c/E5oGwSsEv+PhWYCRyZx58dEQFcK2mkpNERMb81qbVeMCQLEGZm1rkiYqGkPwFvJ79cNNcyFF8uOg/YBJibXz66NvBkYXxJ1ReSkroPZ5XR46MZ+2LWRhsUCgWPABvk733V0rkAUSPfDO6fCxBmZtZ0kkYBL+XCw2rAHqQHo0svF53G8i8XnZ6Hr8nTr4yIkDQd+KWk7wIbkZppzGrpzph1mBwbAyokF2voxo4d25R0tZprxVvHBQgzM2uF0cDU3GPSMOD8iLhI0p1Ufrno6cDPJc0GFpB6XiIi7pB0PnAn6SWln8lNo8x6zaOlpkmSRpOeLYIaa+mKNXQTJkxwDZ0NiAsQZmbWdBFxK7BdhfEVXy4aES8AH+xjXScAJzQ6jWZdplRLdyIr1t59VtI0YAdgkZ9/sEZzAcLMzMysg0k6l/TA9PqS5gLHkAoO50s6FHgQOCDPfgmwF6mL4+eAQ1qeYBvyXIAwMzMz62AR8eE+Ju1WYd4APtPcFFmvcwHCzF7lB9DMzMysP36RnJmZmZmZ1cw1EGZmZmbWFdpZU+73QyzT1AJEftvoz4A3AgF8ArgbOA8YBzwAHBARTzUzHYPhzGLWXo5BMzOzztLsJkynAH+MiK2AtwB3AUcBV0TEeOCKPGxmZmZmZl2gaQUISWsD7yK/FCgiXoyIhcAkYGqebSqwf7PSYGZmZmZmjdXMJkybAY8DZ0p6C3AjcASwQeGFJo8AG1RauPiK9eFrjWpiMs3aQ9ImwNmkGAhgSkScImldBtjM77Z5i9yDkpmZmbVETTUQkq6oZVyZEcD2wKkRsR3wLGXNlXJfxRVfnx4RUyJiQkRMGL762rUk06xt6oyRpcCXImJrYEfgM5K2xs38rMPVmd/Nep5jx4aKqjUQklYFVie9+XAdQHnSWsCYftY9F5gbEdfl4QtJF0KPShodEfMljQYeqzv1Zm02mBjJNXHz8/enJd2Vl5lEeuMopGZ+M4EjG512s4Ea5DnBrGc5dmyo6a8J06eALwAbkZoglTL8YuCH1RaMiEckzZG0ZUTcTXpb4p35czDpFewHA7+rP/mN4S7BbBDqjpEiSeOA7YDrcDM/61wNye9mPcixY0NK1QJERJwCnCLpcxHxgzrW/zngF5JWBu4DDiE1mzpf0qHAg8ABdazXrCM0IEaQtCbwK+ALEbFY0qvTIiIk9dnMD5gCsMro8RXnMWukRuT3TuObONYKQzF2rLfV9BB1RPxA0jtID3WOKIw/u5/lbgYmVJi02wDSaNbx6o0RSSuRCg+/iIhf59Fu5mcdrd78btbrHDs2VNRUgJD0c+B1wM3Ay3l0kHqQMet59cSIUlXD6cBdEfHdwqTpdFgzP7MinxPM6uPYsaGi1m5cJwBb516TzGxF9cTITsDHgNsk3ZzHfYVUcHAzP+tkPieY1cexY0NCrQWI24ENyT3GmNkKBhwjEXE1yx6kK+dmftbJfE4wq49jp0Z+t1Fnq7UAsT5wp6RZwJLSyIjYrympMus+jhHrJc7vZvVx7NiQUGsB4thmJqIXuCQ95B3b7gSYtdCxA11goG9ez88InQLsBTwHTI6Iv+d1HQx8La/6+IiYOqi9MWudY9udALNGqLUXpquanZBu4wKBFTlGrJfUmd9Lb17/u6TXADdKmgFMJr15/URJR5FeOHok8F5gfP7sAJwK7JALHMeQ2pJHXs/0iHhqsPtl1mw+V9hQMayWmSQ9LWlx/rwg6WVJi5udOLNu4RixXlJPfo+I+aUahIh4Gii+eb1UgzAV2D9/nwScHcm1wMjcrfF7gBkRsSAXGmYAExu+k2ZN4HOFDRW11kC8pvQ9VytPAnZsVqLMuo1jxHrJYPN7jW9eHwPMKSw2N4/ra7xZx/O5Ymiq1iplqL6UstZnIF6Vux77raRjSFXNXcPNjqwVujlGzAZqoPm93jev10PSYcBhAMPXGtWo1Zo1hM8V1s1qfZHc+wqDw0htT19oSorMupBjxHpJvfl9gG9enwdsUlh84zxuHrBL2fiZlbYXEVOAKQCrjB7vfvet7XyusKGi1hqIfQvfl5J6ypjU8NSYdS/HSIv1VaM4VKuLO8yA83sdb16fDnxW0jTSQ9SLciHjUuCbktbJ8+0JHD243TFrGZ8rbEio9RmIQ5qdELNu5hixXlJnfh/om9cvIXXhOpvUjeshedsLJB0HXJ/n+0ZELKhrR8xazOeK3jNUb3bV2oRpY+AHpBMAwF+AIyJibrMSZtZNHCPWS+rJ7wN983puH/6ZPtZ1BnDGQNJs1gl8rrChoqZuXIEzSdXJG+XP7/O4fkkaLukmSRfl4c0kXSdptqTzJK1cT8LNOkzdMWLWhZzfzerj2LEhodZnIEZFRDGDnyXpCzUuewSpv++18vBJwMkRMU3SacChpBcEmXWzwcSIWbdxfjerj2PHgO5v2lRrDcSTkj6aaxOGS/oo8GR/C+Wqur2Bn+VhAbsCF+ZZii8NMutmdcWIWZdyfjerj2PHhoRaCxCfID3Y9ggwH/gAMLmG5b4HfBl4JQ+vByyMiKV52C8AsqGi3hgx60bO72b1cezYkFBrE6ZvAAdHxFMAktYF/o8UCBVJ2gd4LCJulLTLQBPmFwBZlxlwjJh1Med3s/o4dsr4Jb/dqdYCxJtLmR1e7UZvu36W2QnYT9JewKqkZyBOAUZKGpFrIUovBlqBX+DIN8oAAA0lSURBVABkXaaeGDHrVs7vZvVx7NiQUGsBYpikdcpKzFWXjYijyS/3yTUQ/xURB0m6gFRlN43lXxpkBdVK5N3ygE2PGXCMmHUx53ez+jQ8diQ9ADwNvAwsjYgJeb3nAeNIL6s7oFhwMRusWjPtd4Br8sU/wAeBE+rc5pHANEnHAzeR3kxq1u0aGSNmnc753aw+zYqdd0fEE4Xho4ArIuJESUfl4SMbsB0zoPY3UZ8t6QZSD0oA74uIO2vdSETMBGbm7/cBbxtYMs0622BjxKybOL+b1aeFsTMJ2CV/n0q6BnMBogt0S/euNVeb5QzuE4RZHxwj1kuc383q04TYCeAySQH8JD9DukFEzM/THwE2KF+o2FnN2LFjG5gc6wVus2pmZmbWvd4ZEfMkvRaYIekfxYkREblwQdn4VzurmTBhgjursQFxAcLMzMysS0XEvPz3MUm/ITUTf1TS6IiYL2k08FhbE4m7ax1qXIAwMzMr0y3tkK23SVoDGBYRT+fve5LeNTGd1NPlibjHS2uCWt9EbWYNJukMSY9Jur0wbl1JMyTdk/+u0840mplZR9sAuFrSLcAs4OKI+COp4LCHpHuA3fOwWcO4AGHWPmcBE8vGlbreGw9ckYfNzMxWEBH3RcRb8mebiDghj38yInaLiPERsXtELGh3Wm1ocQHCrE0i4s9A+UF9EqnLPfLf/VuaKDMzM7N+uABh1ln67XrPzMzMrJ38ELVZh+qr672SYh/ew9ca1bJ0dbpqPX34AVgzM7PBcw2EWWd5NHe5R39d70XElIiYEBEThq++dssSaGZmZr3NBQizzlLqeg/c9Z4NMQPpeUzJ9yXNlnSrpO0Lyxyc579H0sGVtmVmZs3jJkxmbSLpXGAXYH1Jc4FjSF3tnS/pUOBB4ID2pXDocd/+bXcW8EPg7MK4Us9jJ0o6Kg8fCbwXGJ8/OwCnAjtIWpcUKxOAAG6UND0inmrZXpj1kIEeN/3CuN7gAoRZm0TEh/uYtFtLE2LWIhHxZ0njykZPIhWkIfU8NpNUgJgEnB0RAVwraWRu1rcLMKPULaWkGaTukM9tcvLNzCxzAWII8d1VM+tCffU8NgaYU5hvbh7X1/gVuKMBM7PmaFoBQtImpGrqDUjVzFMi4pRc/XweMA54ADjAVc8D4+pBMxuK+ut5rI71TQGmAKwyenzD1mtmvhZptU67SdzMGoilwJci4u+SXkNqpzoDmEzl9q5mZtZ7HpU0OiLml/U8Ng/YpDDfxnncPJY1eSqNn9mCdJoNWbfNW+QCgQ1I0woQuUp6fv7+tKS7SNXMfbV3NTOz3lPqeexElu95bDrwWUnTSA9RL8qFjEuBb5Z6awL2BI5uVWI77S6gmVk7tOQZiPzQ3HbAddT4pl23XTWzdvPFYmMNsOexS4C9gNnAc8AhABGxQNJxwPV5vm+UHqg2M7PWaHoBQtKawK+AL0TEYkmvTqvW3tVtV83MhpaB9DyWe1/6TB/rOQM4o4FJMzOzAWjqi+QkrUQqPPwiIn6dR9f8pl0zMzMzM+ssTStAKFU1nA7cFRHfLUzym3bNzMzMzLpUM5sw7QR8DLhN0s153Ffwm3bNrMO49xEzM7PaNbMXpqsB9THZb9o1MzMzM+tCTX0GwszMzMzMhpaWdONqZmY2lLnLXzPrJa6BMDMzMzOzmrkAYWZmZmZmNXMTJjOzJqvWy5ObuJiZWbdxDYSZmZmZmdXMBQgzMzMzM6uZmzD1APcOYtZYfvGc1crHXzMbilwDYWZmZmZmNXMNhJmZWYv5wXoz62YuQJiZmZmZdaF2NZN0AaKHuW2umVnn8bHZzDqdCxC2gnpOXj7hmZmZmfUGFyCsZu55xsysfXyjxsw6RVsKEJImAqcAw4GfRcSJ7UiHWadyjJhV5xhZxgULq8QxYs3U8gKEpOHAj4A9gLnA9ZKmR8SdrU6LWSdyjJhV5xipjQsWvcsxYs3WjhqItwGzI+I+AEnTgEmAM7VZ4hgxq84xMgiNbI7qwkjHcoxYU7WjADEGmFMYngvsUD6TpMOAw/LgkgdP2uf2FqStkvWBJ3pouw3dtk4a8CJbNmK7Xa6bYmRI5NN2btsxUhfHSIdsu0r+bed+O0ZqiJEOiQ8Y4jHSrm33c24ZdIx07EPUETEFmAIg6YaImNCOdLRr2724z6Vtt2O73agTYqTdeaVXt92O7XYjx0jvbrsd2+02nRAf3nb3xsiwRiRkgOYBmxSGN87jzCxxjJhV5xgxq84xYk3VjgLE9cB4SZtJWhk4EJjehnSYdSrHiFl1jhGz6hwj1lQtb8IUEUslfRa4lNS12BkRcUc/i01pfso6btu9uM/t3nZH6LIY6dW80qvb7giOEW+7g7fdEeqIkV79vbztOikiGpEQMzMzMzPrAe1owmRmZmZmZl3KBQgzMzMzM6tZWwsQkiZKulvSbElHVZi+iqTz8vTrJI0rTDs6j79b0nuasO3/lHSnpFslXSFp08K0lyXdnD8Dfiiphm1PlvR4YRufLEw7WNI9+XNwE7Z9cmG7/5S0sDCt7v2WdIakxyRV7Gdayfdzum6VtH1h2qD2uZs5RhwjhemOkQocI62NkXbFR17eMVIHx4hjpDC9cTESEW35kB7quRfYHFgZuAXYumye/wBOy98PBM7L37fO868CbJbXM7zB2343sHr+/unStvPwM03e78nADyssuy5wX/67Tv6+TiO3XTb/50gPXjViv98FbA/c3sf0vYA/AAJ2BK5rxD5388cx4hgpm+4YqS+vOEaWH193fmlnfOTlHSPNySuOkeXHO0Zq2Od21kC8+pr1iHgRKL1mvWgSMDV/vxDYTZLy+GkRsSQi7gdm5/U1bNsR8aeIeC4PXkvqQ7kRatnvvrwHmBERCyLiKWAGMLGJ2/4wcO4A1t+niPgzsKDKLJOAsyO5FhgpaTSD3+du5hhxjBQ5RlbkGGltjLQtPsAxUifHiGOkqGEx0s4CRKXXrI/pa56IWAosAtarcdnBbrvoUFKJrWRVSTdIulbS/gPY7kC2/f5cvXShpNLLYFq237kacTPgysLowex3vWkb7D53M8dI9W07RhwjjpHq2250jHRyfFRLn2NkGcfI8hwjdcZIy98D0W0kfRSYAOxcGL1pRMyTtDlwpaTbIuLeBm7298C5EbFE0qdIdwZ2beD6a3EgcGFEvFwY1+z9ti7kGHGMWHU9GiOOD6uZY+RVXRMj7ayBqOU166/OI2kEsDbwZI3LDnbbSNod+CqwX0QsKY2PiHn5733ATGC7Rm47Ip4sbO9nwL8MJN2D2XbBgZRVqw1yv+tN22D3uZs5RvrYtmNkubQ5RpZxjCxbfzNipJPjo1r6HCPLOEaWrd8xMpgYifY92DOC9JDGZix70GSbsnk+w/IP9pyfv2/D8g/23MfAHuypZdvbkR6EGV82fh1glfx9feAeqjwgU+e2Rxe+/xtwbSx7yOX+nIZ18vd1G7ntPN9WwAPkFw02Yr/zcuPo+8GevVn+wZ5Zjdjnbv44RhwjZdMcI/XlFcdIg2Kk3fGRl3WMDOz/5RhxjDQlRtqdsfcC/pkzz1fzuG+QSqEAqwIXkB7cmQVsXlj2q3m5u4H3NmHblwOPAjfnz/Q8/h3AbTlT3AYc2oRtfwu4I2/jT8BWhWU/kf8fs4FDGr3tPHwscGLZcoPab1Ipez7wEqlt3aHA4cDhebqAH+V03QZMaNQ+d/PHMeIYcYw4RjopRtoVH44Rx4hjpLNiRHkhMzMzMzOzfvlN1GZmZmZmVjMXIMzMzMzMrGYuQJiZmZmZWc1cgDAzMzMzs5q5AGFmZmZmZjVzAcLMzMzMzGrmAkQXym+KNLMKHB9m1TlGzKpzjPTPBYgWkbSGpIsl3SLpdkkfkvRWSX/L42ZJeo2kVSWdKek2STdJendefrKk6ZKuBK7I6zsjL3eTpElt3kWzujk+zKpzjJhV5xhpLZewWmci8HBE7A0gaW3gJuBDEXG9pLWA54EjgIiIN0naCrhM0uvzOrYH3hwRCyR9E7gyIj4haSQwS9LlEfFsy/fMbPAcH2bVOUbMqnOMtJBrIFrnNmAPSSdJ+ldgLDA/Iq4HiIjFEbEUeCdwTh73D+BBoJSxZ0TEgvx9T+AoSTcDM0mvoh/bqp0xazDHh1l1jhGz6hwjLeQaiBaJiH9K2h7YCzgeuLKO1RRLvQLeHxF3NyJ9Zu3k+DCrzjFiVp1jpLVcA9EikjYCnouIc4BvAzsAoyW9NU9/jdJDO38BDsrjXk8q7VbKvJcCn5OkPO92zd8Ls+ZwfJhV5xgxq84x0lqugWidNwHflvQK8BLwaVLp9geSViO1y9sd+DFwqqTbgKXA5IhYkvNv0XHA94BbJQ0D7gf2acmemDWe48OsOseIWXWOkRZSRLQ7DWZmZmZm1iXchMnMzMzMzGrmAoSZmZmZmdXMBQgzMzMzM6uZCxBmZmZmZlYzFyDMzMzMzKxmLkCYmZmZmVnNXIAwMzMzM7Oa/X90fwYRtxaj0QAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 792x180 with 4 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "fig, ax = plt.subplots(1,4, figsize=(11,2.5))\n",
    "ax[0].hist(detections_probs,np.arange(0.0,1.01,0.05)), \n",
    "ax[1].hist(detections_probs_TUPAC,np.arange(0.0,1.01,0.05))\n",
    "ax[2].hist(detections_hardexamples,np.arange(0.0,1.01,0.05))\n",
    "ax[3].hist(detections_probs_agreed,np.arange(0.0,1.01,0.05))\n",
    "for k in range(4):\n",
    "    ax[k].set_xlim([0.00,1.0])\n",
    "    yl = ax[k].get_ylim()\n",
    "    ax[k].set_ylim(yl)\n",
    "    ax[k].set_xlabel('score')\n",
    "    ax[k].set_ylabel('count')\n",
    "    \n",
    "print('Median model score, agreed mitoses:', np.median(detections_probs))\n",
    "print('Median model score, TUPAC-only mitoses:', np.median(detections_probs_TUPAC))\n",
    "print('Median model score, hard examples:', np.median(detections_hardexamples))\n",
    "print('Median model score, new mitotic fig:', np.median(detections_probs))\n",
    "\n",
    "ax[0].set_title('new mitotic figures')\n",
    "ax[1].set_title('TUPAC16-only mitotic figures')\n",
    "ax[2].set_title('hard negatives')\n",
    "ax[3].set_title('agreed mitotic figures')\n",
    "plt.tight_layout()\n",
    "plt.savefig('Histograms_crossvalidation_TUPACorig.svg')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1239"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(detections_probs_agreed)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# AlternnativeLabels data set"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "    03_stitched1.tif F1: 0.551724, matching:    8, only TUPAC_AL:    3, only TUPAC:   10 (   6 NM)\n",
      "    04_stitched1.tif F1: 0.674033, matching:  183, only TUPAC_AL:  136, only TUPAC:   41 (  38 NM)\n",
      "    07_stitched1.tif F1: 0.625698, matching:   56, only TUPAC_AL:   55, only TUPAC:   12 (  10 NM)\n",
      "    11_stitched1.tif F1: 0.560000, matching:    7, only TUPAC_AL:    3, only TUPAC:    8 (   6 NM)\n",
      "    12_stitched1.tif F1: 0.545455, matching:    3, only TUPAC_AL:    0, only TUPAC:    5 (   5 NM)\n",
      "    18_stitched1.tif F1: 0.745536, matching:  167, only TUPAC_AL:   69, only TUPAC:   45 (  42 NM)\n",
      "    22_stitched1.tif F1: 0.790698, matching:   51, only TUPAC_AL:    6, only TUPAC:   21 (  20 NM)\n",
      "              35.tif F1: 0.000000, matching:    0, only TUPAC_AL:    1, only TUPAC:    0 (   0 NM)\n",
      "              39.tif F1: 0.400000, matching:    1, only TUPAC_AL:    3, only TUPAC:    0 (   0 NM)\n",
      "              44.tif F1: 0.461538, matching:    3, only TUPAC_AL:    7, only TUPAC:    0 (   0 NM)\n",
      "              45.tif F1: 0.470588, matching:    4, only TUPAC_AL:    9, only TUPAC:    0 (   0 NM)\n",
      "              46.tif F1: 0.641509, matching:   17, only TUPAC_AL:   16, only TUPAC:    3 (   0 NM)\n",
      "              47.tif F1: 0.710280, matching:   38, only TUPAC_AL:   31, only TUPAC:    0 (   0 NM)\n",
      "              55.tif F1: 0.666667, matching:    2, only TUPAC_AL:    1, only TUPAC:    1 (   1 NM)\n",
      "              56.tif F1: 0.571429, matching:    2, only TUPAC_AL:    3, only TUPAC:    0 (   0 NM)\n",
      "              57.tif F1: 1.000000, matching:    1, only TUPAC_AL:    0, only TUPAC:    0 (   0 NM)\n",
      "              58.tif F1: 0.666667, matching:    1, only TUPAC_AL:    1, only TUPAC:    0 (   0 NM)\n",
      "              61.tif F1: 0.000000, matching:    0, only TUPAC_AL:    3, only TUPAC:    2 (   1 NM)\n",
      "              62.tif F1: 0.750000, matching:    9, only TUPAC_AL:    4, only TUPAC:    2 (   1 NM)\n",
      "              63.tif F1: 0.642857, matching:    9, only TUPAC_AL:    6, only TUPAC:    4 (   4 NM)\n",
      "              65.tif F1: 0.666667, matching:    5, only TUPAC_AL:    1, only TUPAC:    4 (   4 NM)\n",
      "              71.tif F1: 0.428571, matching:    3, only TUPAC_AL:    3, only TUPAC:    5 (   5 NM)\n",
      "              72.tif F1: 0.742424, matching:   49, only TUPAC_AL:   16, only TUPAC:   18 (   9 NM)\n"
     ]
    }
   ],
   "source": [
    "results_testset = pickle.load(open('RetinaNet-TUPAC_AL-CrossValidationTrainingset-Batch3-512s.pth-test_al_lowcut_b3-inference_results_boxes.p','rb'))\n",
    "DB_TUPAC = Database().open('TUPAC_stitched.sqlite')\n",
    "DB = Database().open('TUPAC_alternativeLabels_augmented.sqlite')\n",
    "mapping_TUPACtoTUPACAL={}\n",
    "detections_probs = []\n",
    "detections_probs_TUPAC = []\n",
    "detections_hardexamples=[]\n",
    "slides_tested = []\n",
    "detections_probs_agreed=[]\n",
    "for [fname,tupacSlideID] in DB_TUPAC.execute('SELECT filename,uid from Slides order by filename').fetchall():\n",
    "    newDBSlide = DB.findSlideWithFilename(fname,'')\n",
    "    DB.loadIntoMemory(newDBSlide)\n",
    "    DB_TUPAC.loadIntoMemory(tupacSlideID)\n",
    "    if fname not in results_testset:\n",
    "        continue\n",
    "    res_testset = results_testset[fname]\n",
    "    probabilities = np.array([[0.5*(x[0]+x[2]),0.5*(x[1]+x[3]),(idx, x[5])] for idx,x in enumerate(res_testset)])\n",
    "    \n",
    "    annos_tupac = np.array([[DB_TUPAC.annotations[x].x1,DB_TUPAC.annotations[x].y1,DB_TUPAC.annotations[x].uid] for x in DB_TUPAC.annotations.keys()])\n",
    "    annos_tupac_al = np.array([[DB.annotations[x].x1,DB.annotations[x].y1,DB.annotations[x].uid] for x in DB.annotations.keys() if DB.annotations[x].agreedClass==1 and DB.annotations[x].deleted==0])\n",
    "    annos_tupac_al_nonmit = np.array([[DB.annotations[x].x1,DB.annotations[x].y1,DB.annotations[x].uid] for x in DB.annotations.keys() if DB.annotations[x].agreedClass==2])\n",
    "    annos_tupac_al_all = np.array([[DB.annotations[x].x1,DB.annotations[x].y1,DB.annotations[x].uid] for x in DB.annotations.keys() if DB.annotations[x].agreedClass>0])\n",
    "\n",
    "    mtch=match(annos_tupac_al,annos_tupac)\n",
    "    mtch_all=match(annos_tupac_al_all,annos_tupac)\n",
    "    mtch2=match(annos_tupac_al_nonmit,annos_tupac)\n",
    "    slides_tested.append(fname)\n",
    "    \n",
    "    if (annos_tupac_al.shape[0]==0):\n",
    "        continue\n",
    "    \n",
    "    (mapping_TUPACtoTUPACAL[fname], mapping_DS2_to_DS1,onlyTUPACAL,onlyTUPAC) = mtch[4]\n",
    "    \n",
    "    annosOnlyInTUPAC_AL = annos_tupac_al[np.in1d(annos_tupac_al[:,2], onlyTUPACAL),:]\n",
    "    annosOnlyInTUPAC = annos_tupac[np.in1d(annos_tupac[:,2], onlyTUPAC),:] if annos_tupac.shape[0]>0 else np.array([])\n",
    "    annosMatched = annos_tupac[np.in1d(annos_tupac[:,2], list(mapping_TUPACtoTUPACAL[fname].values())),:] if annos_tupac.shape[0]>0 else np.array([])\n",
    "    match_probs_TUPACALnewMitosis = match(annosOnlyInTUPAC_AL, probabilities)\n",
    "    match_probs_TUPACnewMitosis = match(annosOnlyInTUPAC, probabilities)\n",
    "    match_probs_TUPACHardExamples = match(annos_tupac_al_nonmit, probabilities)\n",
    "    match_probs_TUPACmatched = match(annosMatched, probabilities)\n",
    "    \n",
    "    print('%20s' % fname,'F1: %4f, matching: %4d, only TUPAC_AL: %4d, only TUPAC: %4d (%4d NM)' % (mtch[0:4]+mtch2[1:2]))\n",
    "    \n",
    "    detections_probs += [x[1] for x in match_probs_TUPACALnewMitosis[4][0].values()]\n",
    "    detections_probs_TUPAC += [x[1] for x in match_probs_TUPACnewMitosis[4][0].values()]\n",
    "    detections_hardexamples += [x[1] for x in match_probs_TUPACHardExamples[4][0].values()]\n",
    "    detections_probs_agreed += [x[1] for x in match_probs_TUPACmatched[4][0].values()]\n",
    "\n",
    "    for k in range(int(match_probs_TUPACALnewMitosis[2])):\n",
    "        detections_probs.append(offprob)\n",
    "    \n",
    "    for k in range(int(match_probs_TUPACnewMitosis[2])):\n",
    "        detections_probs_TUPAC.append(offprob)\n",
    "    \n",
    "    for k in range(int(match_probs_TUPACHardExamples[2])):\n",
    "        detections_hardexamples.append(offprob)\n",
    "\n",
    "    for k in range(int(match_probs_TUPACmatched[2])):\n",
    "        detections_probs_agreed.append(offprob)\n",
    "        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "    01_stitched1.tif F1: 0.690476, matching:   58, only TUPAC_AL:   37, only TUPAC:   15 (  13 NM)\n",
      "    02_stitched1.tif F1: 0.761905, matching:   32, only TUPAC_AL:   15, only TUPAC:    5 (   4 NM)\n",
      "    08_stitched1.tif F1: 1.000000, matching:    3, only TUPAC_AL:    0, only TUPAC:    0 (   0 NM)\n",
      "    09_stitched1.tif F1: 0.666667, matching:    1, only TUPAC_AL:    0, only TUPAC:    1 (   1 NM)\n",
      "    14_stitched1.tif F1: 0.315789, matching:    3, only TUPAC_AL:    0, only TUPAC:   13 (   2 NM)\n",
      "    15_stitched1.tif F1: 0.852941, matching:   58, only TUPAC_AL:   12, only TUPAC:    8 (   7 NM)\n",
      "    16_stitched1.tif F1: 0.666667, matching:    6, only TUPAC_AL:    3, only TUPAC:    3 (   2 NM)\n",
      "              24.tif F1: 0.761905, matching:    8, only TUPAC_AL:    3, only TUPAC:    2 (   1 NM)\n",
      "              27.tif F1: 0.750000, matching:    3, only TUPAC_AL:    2, only TUPAC:    0 (   0 NM)\n",
      "              28.tif F1: 1.000000, matching:    1, only TUPAC_AL:    0, only TUPAC:    0 (   0 NM)\n",
      "              31.tif F1: 0.600000, matching:    6, only TUPAC_AL:    8, only TUPAC:    0 (   0 NM)\n",
      "              33.tif F1: 0.200000, matching:    1, only TUPAC_AL:    3, only TUPAC:    5 (   4 NM)\n",
      "              37.tif F1: 0.400000, matching:    2, only TUPAC_AL:    4, only TUPAC:    2 (   2 NM)\n",
      "              40.tif F1: 0.888889, matching:    4, only TUPAC_AL:    1, only TUPAC:    0 (   0 NM)\n",
      "              42.tif F1: 0.363636, matching:    8, only TUPAC_AL:   28, only TUPAC:    0 (   0 NM)\n",
      "              50.tif F1: 0.444444, matching:    2, only TUPAC_AL:    2, only TUPAC:    3 (   3 NM)\n",
      "              54.tif F1: 0.333333, matching:    1, only TUPAC_AL:    3, only TUPAC:    1 (   1 NM)\n",
      "              66.tif F1: 0.933333, matching:   21, only TUPAC_AL:    2, only TUPAC:    1 (   1 NM)\n",
      "              67.tif F1: 0.786207, matching:   57, only TUPAC_AL:   25, only TUPAC:    6 (   5 NM)\n",
      "              68.tif F1: 0.962963, matching:   39, only TUPAC_AL:    1, only TUPAC:    2 (   1 NM)\n",
      "              73.tif F1: 0.615385, matching:    4, only TUPAC_AL:    4, only TUPAC:    1 (   1 NM)\n"
     ]
    }
   ],
   "source": [
    "results_testset = pickle.load(open('RetinaNet-TUPAC_AL-CrossValidationTrainingset-Batch2-512s.pth-test_al_lowcut_b2-inference_results_boxes.p','rb'))\n",
    "DB_TUPAC = Database().open('TUPAC_stitched.sqlite')\n",
    "DB = Database().open('TUPAC_alternativeLabels_augmented.sqlite')\n",
    "mapping_TUPACtoTUPACAL={}\n",
    "#detections_probs = []\n",
    "#detections_probs_TUPAC = []\n",
    "for [fname,tupacSlideID] in DB_TUPAC.execute('SELECT filename,uid from Slides order by filename').fetchall():\n",
    "    newDBSlide = DB.findSlideWithFilename(fname,'')\n",
    "    DB.loadIntoMemory(newDBSlide)\n",
    "    DB_TUPAC.loadIntoMemory(tupacSlideID)\n",
    "    if fname not in results_testset:\n",
    "        continue\n",
    "    res_testset = results_testset[fname]\n",
    "    probabilities = np.array([[0.5*(x[0]+x[2]),0.5*(x[1]+x[3]),(idx, x[5])] for idx,x in enumerate(res_testset)])\n",
    "    \n",
    "    annos_tupac = np.array([[DB_TUPAC.annotations[x].x1,DB_TUPAC.annotations[x].y1,DB_TUPAC.annotations[x].uid] for x in DB_TUPAC.annotations.keys()])\n",
    "    annos_tupac_al = np.array([[DB.annotations[x].x1,DB.annotations[x].y1,DB.annotations[x].uid] for x in DB.annotations.keys() if DB.annotations[x].agreedClass==1 and DB.annotations[x].deleted==0])\n",
    "    annos_tupac_al_nonmit = np.array([[DB.annotations[x].x1,DB.annotations[x].y1,DB.annotations[x].uid] for x in DB.annotations.keys() if DB.annotations[x].agreedClass==2])\n",
    "    annos_tupac_al_all = np.array([[DB.annotations[x].x1,DB.annotations[x].y1,DB.annotations[x].uid] for x in DB.annotations.keys() if DB.annotations[x].agreedClass>0])\n",
    "\n",
    "    mtch=match(annos_tupac_al,annos_tupac)\n",
    "    mtch_all=match(annos_tupac_al_all,annos_tupac)\n",
    "    mtch2=match(annos_tupac_al_nonmit,annos_tupac)\n",
    "    \n",
    "    slides_tested.append(fname)\n",
    "    if (annos_tupac_al.shape[0]==0):\n",
    "        continue\n",
    "    \n",
    "    (mapping_TUPACtoTUPACAL[fname], mapping_DS2_to_DS1,onlyTUPACAL,onlyTUPAC) = mtch[4]\n",
    "    \n",
    "    annosOnlyInTUPAC_AL = annos_tupac_al[np.in1d(annos_tupac_al[:,2], onlyTUPACAL),:]\n",
    "    annosOnlyInTUPAC = annos_tupac[np.in1d(annos_tupac[:,2], onlyTUPAC),:] if annos_tupac.shape[0]>0 else np.array([])\n",
    "    annosMatched = annos_tupac[np.in1d(annos_tupac[:,2], list(mapping_TUPACtoTUPACAL[fname].values())),:] if annos_tupac.shape[0]>0 else np.array([])\n",
    "    match_probs_TUPACALnewMitosis = match(annosOnlyInTUPAC_AL, probabilities)\n",
    "    match_probs_TUPACnewMitosis = match(annosOnlyInTUPAC, probabilities)\n",
    "    match_probs_TUPACHardExamples = match(annos_tupac_al_nonmit, probabilities)\n",
    "    match_probs_TUPACmatched = match(annosMatched, probabilities)\n",
    "    \n",
    "    print('%20s' % fname,'F1: %4f, matching: %4d, only TUPAC_AL: %4d, only TUPAC: %4d (%4d NM)' % (mtch[0:4]+mtch2[1:2]))\n",
    "    \n",
    "    detections_probs += [x[1] for x in match_probs_TUPACALnewMitosis[4][0].values()]\n",
    "    detections_probs_TUPAC += [x[1] for x in match_probs_TUPACnewMitosis[4][0].values()]\n",
    "    detections_hardexamples += [x[1] for x in match_probs_TUPACHardExamples[4][0].values()]\n",
    "    detections_probs_agreed += [x[1] for x in match_probs_TUPACmatched[4][0].values()]\n",
    "\n",
    "    for k in range(int(match_probs_TUPACALnewMitosis[2])):\n",
    "        detections_probs.append(offprob)\n",
    "    \n",
    "    for k in range(int(match_probs_TUPACnewMitosis[2])):\n",
    "        detections_probs_TUPAC.append(offprob)\n",
    "    \n",
    "    for k in range(int(match_probs_TUPACHardExamples[2])):\n",
    "        detections_hardexamples.append(offprob)\n",
    "\n",
    "    for k in range(int(match_probs_TUPACmatched[2])):\n",
    "        detections_probs_agreed.append(offprob)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "    05_stitched1.tif F1: 0.800000, matching:    4, only TUPAC_AL:    0, only TUPAC:    2 (   1 NM)\n",
      "    06_stitched1.tif F1: 0.816901, matching:   87, only TUPAC_AL:   30, only TUPAC:    9 (   8 NM)\n",
      "    13_stitched1.tif F1: 0.500000, matching:    1, only TUPAC_AL:    0, only TUPAC:    2 (   0 NM)\n",
      "    17_stitched1.tif F1: 0.500000, matching:    2, only TUPAC_AL:    0, only TUPAC:    4 (   2 NM)\n",
      "    21_stitched1.tif F1: 0.606250, matching:   97, only TUPAC_AL:  108, only TUPAC:   18 (  17 NM)\n",
      "    23_stitched1.tif F1: 0.724138, matching:   21, only TUPAC_AL:    5, only TUPAC:   11 (   9 NM)\n",
      "              25.tif F1: 0.000000, matching:    0, only TUPAC_AL:    3, only TUPAC:    1 (   0 NM)\n",
      "              26.tif F1: 0.000000, matching:    0, only TUPAC_AL:    4, only TUPAC:    0 (   0 NM)\n",
      "              29.tif F1: 0.800000, matching:    2, only TUPAC_AL:    1, only TUPAC:    0 (   0 NM)\n",
      "              30.tif F1: 0.500000, matching:    1, only TUPAC_AL:    2, only TUPAC:    0 (   0 NM)\n",
      "              32.tif F1: 0.571429, matching:    4, only TUPAC_AL:    6, only TUPAC:    0 (   0 NM)\n",
      "              34.tif F1: 0.608696, matching:    7, only TUPAC_AL:    6, only TUPAC:    3 (   0 NM)\n",
      "              36.tif F1: 0.571429, matching:    4, only TUPAC_AL:    5, only TUPAC:    1 (   1 NM)\n",
      "              41.tif F1: 0.375000, matching:    3, only TUPAC_AL:    9, only TUPAC:    1 (   1 NM)\n",
      "              43.tif F1: 0.333333, matching:    1, only TUPAC_AL:    4, only TUPAC:    0 (   0 NM)\n",
      "              48.tif F1: 0.000000, matching:    0, only TUPAC_AL:    2, only TUPAC:    0 (   0 NM)\n",
      "              49.tif F1: 0.923077, matching:    6, only TUPAC_AL:    0, only TUPAC:    1 (   0 NM)\n",
      "              51.tif F1: 1.000000, matching:    5, only TUPAC_AL:    0, only TUPAC:    0 (   0 NM)\n",
      "              52.tif F1: 0.825397, matching:   26, only TUPAC_AL:   10, only TUPAC:    1 (   1 NM)\n",
      "              59.tif F1: 0.642857, matching:    9, only TUPAC_AL:    8, only TUPAC:    2 (   2 NM)\n",
      "              60.tif F1: 0.769231, matching:    5, only TUPAC_AL:    2, only TUPAC:    1 (   1 NM)\n",
      "              64.tif F1: 0.533333, matching:   16, only TUPAC_AL:   25, only TUPAC:    3 (   3 NM)\n",
      "              69.tif F1: 1.000000, matching:    2, only TUPAC_AL:    0, only TUPAC:    0 (   0 NM)\n"
     ]
    }
   ],
   "source": [
    "results_testset = pickle.load(open('RetinaNet-TUPAC_AL-CrossValidationTrainingset-Batch1-512s.pth-test_al_lowcut_b1-inference_results_boxes.p','rb'))\n",
    "DB_TUPAC = Database().open('TUPAC_stitched.sqlite')\n",
    "DB = Database().open('TUPAC_alternativeLabels_augmented.sqlite')\n",
    "mapping_TUPACtoTUPACAL={}\n",
    "for [fname,tupacSlideID] in DB_TUPAC.execute('SELECT filename,uid from Slides order by filename').fetchall():\n",
    "    newDBSlide = DB.findSlideWithFilename(fname,'')\n",
    "    DB.loadIntoMemory(newDBSlide)\n",
    "    DB_TUPAC.loadIntoMemory(tupacSlideID)\n",
    "    if fname not in results_testset:\n",
    "        continue\n",
    "    res_testset = results_testset[fname]\n",
    "    probabilities = np.array([[0.5*(x[0]+x[2]),0.5*(x[1]+x[3]),(idx, x[5])] for idx,x in enumerate(res_testset)])\n",
    "    \n",
    "    annos_tupac = np.array([[DB_TUPAC.annotations[x].x1,DB_TUPAC.annotations[x].y1,DB_TUPAC.annotations[x].uid] for x in DB_TUPAC.annotations.keys()])\n",
    "    annos_tupac_al = np.array([[DB.annotations[x].x1,DB.annotations[x].y1,DB.annotations[x].uid] for x in DB.annotations.keys() if DB.annotations[x].agreedClass==1 and DB.annotations[x].deleted==0])\n",
    "    annos_tupac_al_nonmit = np.array([[DB.annotations[x].x1,DB.annotations[x].y1,DB.annotations[x].uid] for x in DB.annotations.keys() if DB.annotations[x].agreedClass==2])\n",
    "    annos_tupac_al_all = np.array([[DB.annotations[x].x1,DB.annotations[x].y1,DB.annotations[x].uid] for x in DB.annotations.keys() if DB.annotations[x].agreedClass>0])\n",
    "\n",
    "    mtch=match(annos_tupac_al,annos_tupac)\n",
    "    mtch_all=match(annos_tupac_al_all,annos_tupac)\n",
    "    mtch2=match(annos_tupac_al_nonmit,annos_tupac)\n",
    "    \n",
    "    slides_tested.append(fname)\n",
    "    if (annos_tupac_al.shape[0]==0):\n",
    "        continue\n",
    "    \n",
    "    (mapping_TUPACtoTUPACAL[fname], mapping_DS2_to_DS1,onlyTUPACAL,onlyTUPAC) = mtch[4]\n",
    "    \n",
    "    annosOnlyInTUPAC_AL = annos_tupac_al[np.in1d(annos_tupac_al[:,2], onlyTUPACAL),:]\n",
    "    annosOnlyInTUPAC = annos_tupac[np.in1d(annos_tupac[:,2], onlyTUPAC),:] if annos_tupac.shape[0]>0 else np.array([])\n",
    "    annosMatched = annos_tupac[np.in1d(annos_tupac[:,2], list(mapping_TUPACtoTUPACAL[fname].values())),:] if annos_tupac.shape[0]>0 else np.array([])\n",
    "    match_probs_TUPACALnewMitosis = match(annosOnlyInTUPAC_AL, probabilities)\n",
    "    match_probs_TUPACnewMitosis = match(annosOnlyInTUPAC, probabilities)\n",
    "    match_probs_TUPACHardExamples = match(annos_tupac_al_nonmit, probabilities)\n",
    "    match_probs_TUPACmatched = match(annosMatched, probabilities)\n",
    "    \n",
    "    print('%20s' % fname,'F1: %4f, matching: %4d, only TUPAC_AL: %4d, only TUPAC: %4d (%4d NM)' % (mtch[0:4]+mtch2[1:2]))\n",
    "    \n",
    "    detections_probs += [x[1] for x in match_probs_TUPACALnewMitosis[4][0].values()]\n",
    "    detections_probs_TUPAC += [x[1] for x in match_probs_TUPACnewMitosis[4][0].values()]\n",
    "    detections_hardexamples += [x[1] for x in match_probs_TUPACHardExamples[4][0].values()]\n",
    "    detections_probs_agreed += [x[1] for x in match_probs_TUPACmatched[4][0].values()]\n",
    "\n",
    "    for k in range(int(match_probs_TUPACALnewMitosis[2])):\n",
    "        detections_probs.append(offprob)\n",
    "    \n",
    "    for k in range(int(match_probs_TUPACnewMitosis[2])):\n",
    "        detections_probs_TUPAC.append(offprob)\n",
    "    \n",
    "    for k in range(int(match_probs_TUPACHardExamples[2])):\n",
    "        detections_hardexamples.append(offprob)\n",
    "\n",
    "    for k in range(int(match_probs_TUPACmatched[2])):\n",
    "        detections_probs_agreed.append(offprob)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Median model score, agreed mitoses: 0.5029786229133606\n",
      "Median model score, TUPAC-only mitoses: 0.2839334309101105\n",
      "Median model score, hard examples: 0.0658361092209816\n",
      "Median model score, new mitotic fig: 0.5029786229133606\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAxAAAACsCAYAAADrGUpuAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+j8jraAAAgAElEQVR4nO3dd/xcVZ3/8dc79CIdIQUICBawABsBywpSIy2sBVHUgLiIoouruwLqbwUBxXUVsRulBFFCsUVE6cEKgQgCAZHQTEJoKYQmEPj8/jhnyM0wM9+Z+X6nv5+Pxzy+M3duOfc753PvPfeUq4jAzMzMzMysHqM6nQAzMzMzM+sdLkCYmZmZmVndXIAwMzMzM7O6uQBhZmZmZmZ1cwHCzMzMzMzq5gKEmZmZmZnVzQWIFpF0qKTLRnB9/yrpjiaWk6SzJC2WNLPZ9Zi1kqTDJP2hDdsZ0fwvaXNJj0taqYllT5b0iKQHhrMeax9J90ras4XrnyHpQ61a/0iR9D1J/6/T6bD+JCkkbT1C63pc0lYjsa68vtmSdmtiuX+TNDenZ4dm19NNXIBokYj4cUTsXfrcaECUzx8Rv4+IVzSRlDcDewHjImKnYazHmpAPFqXX85KeKnw+VNLZkk4uW2Z8/v1Xzp/vLSz3YF5m7cL8h+X5311h+y+XdGG+UH1U0s2SPlm6UJU0RdIdOW2HVVh+K0kXS3osr+N/R/yf1Ebl+b/RC8Ly+SPiHxGxdkQ810g6JG0OfArYNiI2bXY9Zq1WqXAfEUdFxEmdSpNZvfJx9W6ASufbWirNHxHbRcSMJpLyf8DHcnpuHMZ6uoYLEP1vC+DeiHii1Rvy3dMXyweLtSNibeAfwAGFaT9uYFUH5HXsCEwAPlf4bjKwCPhAcQFJLwOuA+YCr4mIdYF35eVfkmf7K/BR4C/lG5S0KnA5cBWwKTAOOLeBNFt1mwMLI+KhVm+oVBC17pFrhn3+tZ7nvFy3LYDZrd5IW4/3EdGTL+Be4L+Am4FHgfOB1Qvf7w/cBCwB/gS8Nk8/HPhVYb47gQsLn+cC21fY3ngg8vJzgcXAUcDrcxqWAN8qzH8Y8If8/nd52SeAx4F35+n/DswhXfxNB8ZUmx/YDZhXWP9mwM+Ah4GFxW0X5jkC+CfwXF7PiRXWsyNwI/AYcGH+P55cvg+F+QPYOr8/G/gucElO657AGOCnOV33AP9RWHYn4AZgKfAg8LVO56MO5Nk9y6adXfp/V8hrK1daDvgKcHF+vwXwPPAOYBmwaWG+c4Ff15m2PwCHlU07Evh9g/tYMU8X8s5RpJhbAnwbUIV4+Tbw1bL1Tgf+s8o2g1QIujPn45OAl5HifilwAbBqnveF/A/8KP/vnsrx8ek8/UDSgX4JMAN4VbX5K/xWGwBnAfeTjhG/qJDePfM6ns/rObvCerYkHQceA67I/5Nzy/ehUt4CTgAuyr//UuBDwLrAGcACYD5wMrBSnn9r4BrScfQR4PxOx0q3vqhx3gHWBy4mHfsW5/fjCsvOAE4B/ph//61JtcN/y+v6Vv4dPlRl2yfkvHxOzhezgQmF72sde9cApuZ03Z7zbvE8cBxwV17vbcC/5emvYsVzyJI8/WyWnyduB/YvrGvlnIYd8+ddSLG4hHTDYrfCvIcBd+ft3gMc2unfeFBe1X7z/N1KwFfz8eAe4GOseHyqlJdfSbrhtAi4Azi4sL7VSHfg/0E6938PWKPw/X+Tjk33Ax+kcJ1RId0zSMevP+U8+StgQ+DHpOPd9cD4wvyR03ck8CzwTGm5Qh6fkfPnbODAPL3a/Pey/Fi7EvCZwv9xFrBZWXpXy8uXrunuqrCeoeJzhf9HWfztBswDjgUeIJ2nRhV+34Wk48YGef7VSeeGhXmfrwc2aSoPdToTDyPz3wvMJB00N8j/9KPydzsADwE75x94cp5/NWCr/E8blZe9j+UXFFvlH3BUhe2Nzz/i9/IPsDfpwPoL4KXA2LzNXfP8h1G4+K6QAXYnBeeOOV3fBH5XY/7dCulciXQgPg1YK6fnzVX+T+XpKK5n1bz/xwCrAG8nBUsjBYhHgTfl/+eapAD6n7zurUgnh33y/H8G3p/frw3s0ul81IE8O6wCBKngOBs4KX/+f8DM/P4W4FOF9TwAHF5n2ioVIM4kHYx+k/PqDFJNRrV11JOnLwbWI92BfxiYWJ7XSAXN+8lxCGwEPEmVg1xe7y+BdYDtgKeBK3P+W5d0cpxcnv8r/SbAy0kH+b1yTHyaVCBatcr85b/Vr0kXlevn5XetkubydJSv58+kE+6qpGaIS2msAPEscBApLtcAfg58n3S8eCnp2PnhPP95wGfzvFWPJX4Ned7ZkFSQX5NUw3chhQIkKX7+kfPoysDGpIuOd+a88p+kmwC1ChD/BPYlnQO+BFybvxtF7WPvqaTCyfqkmsSby/Lfu/I+jSLdsHoCGF0em4X5z2b5eeJ/gB8XvtsPuD2/H0u6UNk3r3uv/HnjnBeXAq/I844Gtuv0bzworyF+86NIx81xOc9cwYsLEMW8vC7pxurh+fMOpHPBtnn+00g3gTbIsfEr4Ev5u4mkQsWrc574CUMXIOaQbhKVju9/J92YWZlUwD6rMH/5NcvJhe9Wyev6TI6b3Ukx+YpK8+dp97L8WPvfpPPuKwABrwM2rJLu8mu64nqGis+hChDLgC+TzrtrkK7prs3rWo107D8vz//h/P9fk3Qc+RdgnWbyUK9XO30jIu6PiEWkf8j2efqRwPcj4rqIeC4ippIuKnaJ1BbusTzvW4BLgfslvRLYlXTH9fka2zwpIv4ZEZeRAu68iHgoIuYDvycFTj0OBc6MiL9ExNPA8cAbJI2vY9mdSIH/3xHxRE5PMx1QdyEF3Dci4tmI+Bnp5NiIX0bEH/P/7DXAxhHxhYh4Jv+vfwAckud9Ftha0kYR8XhEXNtEmgfVLyQtIV3oXwN8MU//AOmAS/5bbMa0IemuTrPGkX67b5Dy26+BX+amTZXUk6dPjYglEfEP4GqWx+wLImImqWC6R550CDAjIh6skdb/jYilETEbuBW4LCLujohHSQWgeuPy3aRam8sj4lnSRfwawBuHWlDSaOBtpAvKxTmmrqlzu8X1bE6q2fyfHEd/IJ18G/HniPhFjst1SBdwn8jHi4dIJ/RiXG5Bqi1q9lgySCqedyJiYUT8NCKejIjHSHdody1b9uyImB0Ry0h5ZXZEXJTz2tdJhf5a/hARl0TqK/Mj0gULpPxS69h7MPDFnC/nkWL6BRFxYd6n5yPifFJt3k51/j9+Ahwoac38+b2kQinA+4BLcpqfj4jLSbXQ++bvnwdeLWmNiFiQ49faYIjf/GDg9IiYFxGLSRe45Yp5eSKpqfRZEbEsIm4k1Ya9S5JI12T/GRGLcmx8kRXz5lkRcWukptYn1JH8syLirsLx/a6IuCKn5ULqP97vQrqZeWqOm6tIN7neU+fyHwI+FxF3RPLXiFhY57JFNeOzDs8Dn4+IpyPiKVIB8LP593ua9D99Z27e9Czp2mDrfH08KyKWNpHmni9AFA+2T5IyAqST4ackLSm9SHdux+TvryGV2t6S388gHeh3zZ9rKV7EPFXh89rUp1T7AUBEPE66MzO2jmU3A+7LwTIcY4D5EalYms1tcB3F+bcAxpT93z8DbJK/P4J0h/dvkq6XtH+zCe8jy0h3QYpWIR0QigXZgyJivYjYIiI+GhFPSXoTqanLtDzPT4DXSCpdlC8k3dVr1lOkC5bfRMQzpIvpDYFX5Q7gpc7gv8nz15Onq8Vsuamkiw/y3x8NkdZWxeXzpDxeb1wuyifc4RiT1/NkYdpw43IVYEEhLr9PqomAVMsiYGYeGeSDTaZ7UFTMw5LWlPR9SfdJWkpqgrZeWd+w4u8ypvg5H4eH+p3Lt716vigY6ti7wrbKtyPpA5JuKiz7alLN35AiYg6pJuaAXIg4kOU3NbYgXUQW0/Vm0p3uJ0gF9qNIefPX+UaetcEQv3nN/FJh2hbAzmW/86GkvnMbk1snFL77bZ5eaVv3MbSRPN7PLbtpfB/1He8hHfPvqnPeIdNR+Nzo8f7hiPhn4fMWwM8L/+/bSc0QNyGdSy8Fpkm6X9L/Siq/BqlLv3aumwucEhGnVPn+GuAA0sXXF0lNmg4F3kBqh9oO95N+ZAAkrUW6OJtfx7Jzgc0lrTzMQsQCYKwkFQoRxYB4ghT4pTRuWmEd5YWPeyJim0obi4g7gffkDldvBy6StGG0oYN3FytVAxdtyYsPapVMJl343ZRu8qww/SZStfM7SG3ym3EzqXnai0TqAF7eCXw4ebrcucCtkl5HaqP6iybWUY8o+3w/qSYNSB0ESTExv8r8RXOBDSStFxFLhpGmBXk9axYKEZsVvi+Py5VYfjIuKY/Lp4GNKh0vIuIBUt8VJL0ZuELS7/KFodXvU6SmDDtHxAO5IH8jKUZLir/LAgq/ayGvNaPmsTdvaxypuQdl292CVFuxB6nm6jlJNxXSXSvPl5xHums7CritkHfmAj+KiH+vtFBEXApcKmkNUrv2HwD/Wsf2bBjq+M1L+aWkUr4sP8ZcExF7VdjWKNJF/XaRWmqUWyEOSM1bW6XS8X4zSaMK59vNSU2iKs1fbi6pKdWtw0xX1fjMnqRwzCcVzOYVPpency7wwYj4Y5XtnQicmFsHXELqs3JGo4nu9RqIan4AHCVp5zxCwFqS9pNUGnnmGuCtpE4880hNjyaSLnZubFGaHiS1Sy05Dzhc0vaSViMVZK6LiHurzF80k5ThTs37tnq+G92oP5NKpR+TtLKkSaxYbf1XYLucxtUZumpxJvCYpGMlrSFpJUmvlvR6AEnvk7RxDtTSBdZQF8n97qfAfpL2zv+vMaQRlqbVWij/HgeTqoa3L7w+Drw335X8PPBGSV8pFf4kbS3pXEnr5c+r5nUJWCXnpdJx4VxgF0l75ovUT5Datd5eJVlD5em65bi8nnS35Ke5WrYVyuPsAtLvsUe+K/Mp0sX3n6rMX0zzAlJ1+nckrS9pFUlvaTRBEXEfqZnHCfn3eQPphkfJ30l3nvfLafwcqZ1rtfUtAC4DvippHUmjJL1M0q4Akt4lqXSxsJh0Mhr0uGzGS0gXSkskbUCKv1p+TTq+vj3H63+QLgyaUfPYS8rXx+d8OZbUKbZkLdJv/jCApMNJd6NLHgTGqXrTRUjHq72Bj7C89gHSMeQASfvkNK0uaTdJ4yRtImlSvtHwNKmjqfNdewz1m18AHCNpbD5XHDvE+i4GXi7p/fm4t4qk10t6VT7f/wA4TdJL8/bGStqnsK3DJG2rVIM1VNwMR/nx+zrSxfmnc5p3Ix1rp1WZv9wPgZMkbZOvNV8racMm0lUrPiHdEHxvjqGJvLhpZLnvAafkgiKSNs7Xd0h6q6TX5HP6UlKTpqbiri8LEBFxA+mO2rdIJ8Q5pI5gpe//TjpY/T5/XkrqcPbHaN047CcAU5WqlA6OiCtIHWB/SioMvIzlbQJfNH9xRTmNB5BGFvgHqST6omcADCU3S3k7qWnRElJTkYtJB/PS/+kLpDvZd5La39da33Ok0a+2J43c8AgpwNbNs0wEZkt6HDgdOKSFF4Y9Ibf5fQ+pQ+QiUqHuOtIdgloOIl2snBMRD5RepI7PK5M6J99FqlUbT/q/P0rKbzeQ+gFBurB8itTGf0p+/5actjtIeeJ7pDiaRBqh4pkq+zJUnm7UVFJtwFDNl4bjS8Dncpz9V2Gfv0nKvweQhtB9ptL8Fdb3ftIB+W+kQRU+0WS6SjWiC0l3Zs9neVw+Shp16oekmpEnWPFuVCUfIHUSvI30W17E8uZtrweuy3E5HTgm8rjp1pCvk/rLPELqwPjbWjNHxCOkjqynkn7nbUij2jSsjmPvF0h55B7S8fwiluen20gj7vyZdMH0mrJ0XEUauOEBSY9U2f6CvPwbSXm1NH0u6bjxGdLF6lxSx9NR+fVJ0l3gRaSLoo80s//WmDp+8x+Qzg03k26qXkJqblvx+ij3a9ibdLy/n9TUrtSpF1IBZA5wrVLzvitItXVExG9IsXNVnueqEdrNSs4Ats3H71/k4/oBpP5IjwDfAT4QEX+rNH+F9X2NdPF/Geli/AzSMaBRVeMzOyans9RaZqga+dNJx/LLJD1GOh7tnL/bNK9/Kelm4DU0eY4tDaFoBoCk64DvRUSzzV7MRkS+e38usEUM+IFK0vnA3yKilXfnbEBI+gjpBs5QdzLNkPQ20nXBFkPObMPWK/HZlzUQVj9Ju0raNDdhmgy8liHunJm1Wm6acwzww0EsPOTq/5fl5kYTSXdxW9UPxPqcpNGS3pTz0ytITfN+3ul0WXfKzeD2zdcFY0nNipxfWqRX47NfO1Fb/V5BqoJbi9SM6525OtqsIyS9itTM6q+kccUH0aakB0VuSKra/kikoRHNmrEqaeStLUnNIKaRmmuYVSJSM9rzSc1af0163oe1Rk/Gp5swmZmZmZlZ3dyEyczMzMzM6tYTTZg22mijGD9+fKeTYW0wa9asRyKifEx7G4JjZHA4RprjGBkcjpHGOT4Gy0jESEsLEHn84B+SxhcO4IOkB1acTxpa8l7g4KGe3Dp+/HhuuOGGVibVuoSkep5CaWUcI4PDMdIcx8jgcIw0zvExWEYiRlrdhOl04LcR8UrgdaQxZ48DrsxPzLwyfzYzMzMzsx7QsgKEpHVJD6Q6A9JDyyJiCWk4wql5tqmkB2KZmZmZmVkPaGUNxJakJ0+eJelGST/Mj6zfpDBM6APAJpUWlnSkpBsk3fDwww+3MJlmZmZmZlavVhYgVgZ2BL4bETsAT1DWXCk/IKriOLIRMSUiJkTEhI03dl8oMzMzM7Nu0MpO1POAeRFxXf58EakA8aCk0RGxQNJo4KEWpsEAnaiK0+PzfgZIv5h1/6yKv7N/Y7PEMWJm3aTXr81aVgMREQ8Ac/NjuQH2AG4DpgOT87TJwC9blQYzM+suklbKzVovzp+3lHSdpDmSzpe0ap6+Wv48J38/vrCO4/P0OyTt05k9MTMbXK0ehenjwI8l3QxsD3wROBXYS9KdwJ75s5mZDYZjSCPylXwZOC0itgYWA0fk6UcAi/P00/J8SNoWOATYDpgIfEfSSm1Ku5mZ0eICRETclPsxvDYiDoqIxRGxMCL2iIhtImLPiFjUyjSYmVl3kDQO2I/0fCAkCdid1MQVVhyZrzhi30XAHnn+ScC0iHg6Iu4B5gA7tWcPzMwMWl8DYWZmVvJ14NPA8/nzhsCSiFiWP88Dxub3Y4G5APn7R/P8L0yvsMwKiqP58eRI7oaZ2WBzAcLMzFpO0v7AQxExq13bLI7mx5rt2qqZWf9r5ShMZmZmJW8CDpS0L7A6sA5wOrCepJVzLcM4YH6efz6wGTBP0srAusDCwvSS4jJmZn2p20Ztcg2EmZm1XEQcHxHjImI8qRP0VRFxKHA18M48W3FkvuKIfe/M80eefkgepWlLYBtgZpt2w8zMcA2EmZl11rHANEknAzcCZ+TpZwA/kjQHWEQqdBARsyVdQBoWfBlwdEQ81/5km5kNLhcgzMysrSJiBjAjv7+bCqMoRcQ/gXdVWf4U4JTWpdDMzGpxAaKPVGsfZ90tj2F/AzA/IvbPzTKmkUacmQW8PyKe6WQazczMzErcB8Ks8+p9sJaZmZlZx7kAYdZBDT5Yy8zMzKzjXIAw66xGHqxlZmZm1nHuAzHAum1M4UFTfLCWpN2aWP5I4EggjZBvZmZmXaVf+6e6AGHWOY0+WGsFETEFmAKgMXKpz8zMzNrCTZjMOqSJB2uZmZmZdZxrIMy6T7UHa5mZmVkf65UmTy5AmHWBeh6sZWZmZtYNXICwF3HnajMzMzOrxn0gzMzMzMysbi5AmJmZmZlZ3VyAMDMzMzOzuvVlHwi34TczM7N+IelMoPTw0VfnaRsA5wPjgXuBgyNisSSRnim0L/AkcFhE/KUT6bb+5RoIMzMzs+52NjCxbNpxwJURsQ1wZf4M8DZgm/w6Evhum9JoA6QvayC6UaPj+rq2xMzMzAAi4neSxpdNngTslt9PJQ0Ffmyefk5EBHCtpPUkjY6IBe1JrQ0CFyDMzMzMes8mhULBA8Am+f1YYG5hvnl5mgsQLdIrD38bSW7CZGZmLSdpdUkzJf1V0mxJJ+bpW0q6TtIcSedLWjVPXy1/npO/H19Y1/F5+h2S9unMHpl1j1zb0FDTBUlHSrpB0g0PP/xwi1Jm/coFCDMza4engd0j4nXA9sBESbsAXwZOi4itgcXAEXn+I4DFefppeT4kbQscAmxHahP+HUkrtXVPzLrDg5JGA+S/D+Xp84HNCvONy9NWEBFTImJCREzYeOONW55Y6y8uQJiZWctF8nj+uEp+BbA7cFGePhU4KL+flD+Tv98jjy4zCZgWEU9HxD3AHGCnNuyCWbeZDkzO7ycDvyxM/4CSXYBH3f/BRpr7QJgNIA91bJ2QawpmAVsD3wbuApZExLI8S6mtNhTacUfEMkmPAhvm6dcWVltcpnx7R5JGoYF1R3JPzNpL0nmkDtMbSZoHfB44FbhA0hHAfcDBefZLSEO4ziEN43p42xNsfa/lBYh8wrgBmB8R+0vaEphGOhHMAt4fEc+0Oh1mZtZZEfEcsL2k9YCfA69s8famAFMANEYuHVvPioj3VPlqjwrzBnB0a1Nk3aJTNwTb0YTpGOD2wudq7V3NzGwARMQS4GrgDcB6kko3s4pttV9ox52/XxdYSJ3tu83MrHVaWoCQNA7YD/hh/iyqt3c1GyiNjkpj1sskbZxrHpC0BrAX6ebS1cA782zl7bhL7bvfCVyV76xOBw7JozRtSXpY1sz27IWZmUHrayC+DnwaeD5/3pDq7V1X4OHFbAA0OiqNWS8bDVwt6WbgeuDyiLiY9OCrT0qaQzpHnJHnPwPYME//JPkpuxExG7gAuA34LXB0bhplZmZt0rI+EJL2Bx6KiFmSdmt0+WLb1QkTJrjtqvWdfDe12qg0783TpwInAN9td/rMRlJE3AzsUGH63VQYRSki/gm8q8q6TgFOGek0mplZfVrZifpNwIGS9gVWB9YBTie3d821EG67agOtwVFpzMzMrEMG8YnT1bSsABERxwPHA+QaiP+KiEMlXUhqzzqNFdu79pRWZ6Ja6+/GoTY9LGhzhjMqjYeoNDMzs07oxIPkqrV3NRtYdY5KU77MC08RZc02JdTMzMwGXl01EJKujIg9hppWTUTMAGbk9xXbuzbDVUnt5f93dc3EiKSNgWcjYklhVJovs3xUmp6upbP+NdxzgtmgcuxYv6hZgJC0OrAm6cmH6wOlK8h1cLtss+HGyGhgau4HMQq4ICIulnQbME3SycCNuJbOuoTPCWbNcexYvxmqBuLDwCeAMaSOnqUMvxT4VgvTZdYrmo6RRkelMesCPieYNcex00Pc4mJoNQsQEXE6cLqkj0fEN9uUJrOe4RixQeL8btYcx471m7r6QETENyW9ERhfXCYizmlRulYw6/5ZLg0W+H/RfTodI2bt5Pxu1hzHjvWLejtR/wh4GXATUHriZwDO8GY4RmywOL+bNcexY/2i3udATAC2zU/O7Tu99swF60p9HSNmZZzfzZrj2LG+UG8B4lZgU2BBC9Ni1su6Mkbc3M1apCvzu1kPcOxYX6i3ALERcJukmcDTpYkRcWBLUmXWexwjNkj6Jr9XK2S79tlapG9ixwZbvQWIE1qZCLM+cEKnE2DWRid0OgFmPeqETifAbCTUOwrTNa1OiFkvc4zYIHF+N2uOY8f6Rb2jMD1GGiUAYFVgFeCJiFinVQnrFm5DbvUY5BixweP8btYcx471i3prIF5Sei9JwCRgl1YlyqzXOEZskDi/mzXHsWP9YlSjC0TyC2CfFqSnpXSiKr7MRlIvx4hZo+rN75I2k3S1pNskzZZ0TJ6+gaTLJd2Z/66fp0vSNyTNkXSzpB0L65qc579T0uSW7qBZi/hcYb2s3iZMby98HEUax/ifLUmRWQ/qlxjxiDRWjybz+zLgUxHxF0kvAWZJuhw4DLgyIk6VdBxwHHAs8DZgm/zaGfgusLOkDYDP521GXs/0iFg8Yjto1iL9cq4wq3cUpgMK75cB95Kq3cwscYzYIGk4v0fEAvLY9xHxmKTbgbF5ud3ybFOBGaQCxCTgnPzArWslrSdpdJ738ohYBJALIROB80Zgv8xazecK6wv19oE4vNUJMetljhEbJMPN75LGAzsA1wGb5MIFwAPAJvn9WGBuYbF5eVq16ZW2cyRwJADrDifFZiPD5wrrF3X1gZA0TtLPJT2UXz+VNK7ViTPrFc3ESKNtws26xXDOCZLWBn4KfCIilha/y7UNI9ZeLiKmRMSEiJjAmiO1VrPm+XrK+kW9najPAqYDY/LrV3mamSXNxEipTfi2pFE4jpa0LakN+JURsQ1wZf5s1k2aOidIWoVUePhxRPwsT34wN00i/30oT58PbFZYfFyeVm26WS/w9ZT1hXoLEBtHxFkRsSy/zgY2bmG6zHpNwzESEQsi4i/5/WNAsU341DzbVOCg1iXbrCkN5/c8ZOUZwO0R8bXCV9OB0khKk4FfFqZ/II/GtAvwaG7qdCmwt6T1c+3c3nmaWS/w9VQX8eiczau3ALFQ0vskrZRf7wMWtjJhZj1mWDFSZ5tws27RTH5/E/B+YHdJN+XXvsCpwF6S7gT2zJ8BLgHuBuYAPwA+CpA7T58EXJ9fXyh1qDbrAb6esr5Q7yhMHwS+CZxGap/6J9LQe2aWNB0j5W3C043aJCJCUsU24e4gah3UcH6PiD8A1W7t7VFh/gCOrrKuM4Ez60+uWdfw9ZT1hXoLEF8AJpfG2c7jcP8fKRDMrMkYqdUmPCIWlLUJX0FETAGmAGhM5UKGWYv4nGDWHMeO9YV6CxCvLT6kJyIWSdqhRWky60UNx0gdbcJPZcU24R3jB8xZGZ8TzJrj2LG+UG8fiFHFoSRzibnewofZIGgmRhptE27WLXxOMGvOiMeOpHsl3ZLPITeU1uvhwK2V6s20XwX+LOnC/PldwCmtSZJZT2o4RhptE27WRXxOMGtOq2LnrRHxSOFzaTjwUyUdlz8fOwLbMQPqfxL1OblUu3ue9PaIuK11yVrBZL8AAAwzSURBVDLrLY4RGyTO72bNaWPsTAJ2y++nAjNwAcJGUN3VZjmD+wRhVoVjxAaJ87tZc1oQOwFclkfs+34eYGPI4cCLI/ltvvnmI5gcGwQta7MqaTPgHFKmDWBKRJye2/udD4wH7gUOLnYoMrPB5I7aZmZNeXNEzJf0UuBySX8rflltOPDiSH4TJkzwgdYaUm8n6mYsAz4VEdsCuwBHS9qW5e3ytgGuzJ/NzMzMrEERMT//fQj4ObATeThwgFrDgZs1q2U1ELnqbEF+/5ik24GxuF2emTXANRNmZpVJWgsYla+z1gL2Jj1rouuGA++kaucRa15bht2TNB7YAbiOOtrl5WX8lN0+4Is/MzOzltkE+Hl6rBArAz+JiN9Kuh64QNIRwH3AwR1Mo/WhlhcgJK1NetLuJyJiac7kQPV2efk7P2XXzEZUowVaF4DNrJtFxN3A6ypMX4iHA7cWamUfCCStQio8/DgifpYnu12emZmZmVmPalkBQqmq4Qzg9oj4WuGrUrs8cLs8MzMzM7Oe0somTG8C3g/cIummPO0zpA49bpdn1gdqdUzrVDMfd5YzMxtMPv63TytHYfoDUO2XdLs8MzMzM7Me1JZRmMzMzMzMhsu1DPVp9f/JBQgzM2sLSWcC+wMPRcSr87QNgPOB8cC9wMERsTj3ozsd2Bd4EjgsIv6Sl5kMfC6v9uSImDriafUIXGZmVbkAYWY9aSTvrviOVtucDXwLOKcw7Tjgyog4VdJx+fOxwNuAbfJrZ+C7wM65wPF5YAIQwCxJ0yNicdv2wsxswLV0GFczM7OSiPgdsKhs8iSgVIMwFTioMP2cSK4F1stDf+8DXB4Ri3Kh4XJgYutTb2ZmJa6BMOuQRppzdCqNZm2wSUQsyO8fID1ZF2AsMLcw37w8rdr0F5F0JHAkAOuOXILN+s2s+2e52Z41xAUIs845m/qbc5j1vYgISSN2tRIRU4ApABozcus1s9Zz09Lu5iZMZh3SYHMOs371YG6aRP77UJ4+H9isMN+4PK3adDMzaxPXQJh1l2rNOV6k25tnDGJ1+CDu8wiYDkwmPWR0MvDLwvSPSZpG6kT9aEQskHQp8EVJ6+f59gaOb3OazcwGmgsQZl1qqOYcbp5hvUbSecBuwEaS5pFGUzoVuEDSEcB9wMF59ktIQ7jOIQ3jejhARCySdBJwfZ7vCxFRXpNnZj3CTZV6kwsQZt3lQUmj853WYnMOs54XEe+p8tUeFeYN4Ogq6zkTOHMEk2ZmZg1wHwiz7lJqzgErNucwMzMz6wqugTDrkAabc/SNfqiubnQfas3v/hFmZtZrXIAw65BGmnOYWXdwR3mzxLEw2NyEyczMzMzM6uYChJmZmZmZ1c0FCDMzMzMzq5v7QJiZmZnZiOiHgTJsaK6BMDMzMzOzurkGwszMbJg8Io31K9coWCWugTAzMzMzs7q5AGFmZmZmZnVzAcLMzMzMzOrmPhBmZmYt4r4RZtaPXIAwM+sgX2AOplodU/3bm1m3cxMmMzMzMzOrmwsQZmZmZmZWNzdhso7wuNJmZpWN1PHRTaHMrFU6UgMhaaKkOyTNkXRcJ9Jg1s0cI2a1OUaGphNV8WWDwTFirdT2GghJKwHfBvYC5gHXS5oeEbe1Oy1m3cgxYlabY2R42lGIcO1HZzlGrNU60YRpJ2BORNwNIGkaMAlwpjZLHCNmtTlGupybYXWcY8RaqhMFiLHA3MLnecDO5TNJOhI4Mn98mhO4tQ1pq2Qj4JEB2m6nt/2KDm23m/RSjAxqPm35tnVC1Qswx4hjZGC2XSMOanGM1BEjXRIf0Af5tAe3PewY6dpO1BExBZgCIOmGiJjQiXR0atuDuM+lbXdiu72oG2Kk03llULfdie32IsfI4G67E9vtNd0QH95278ZIJzpRzwc2K3wel6eZWeIYMavNMWJWm2PEWqoTBYjrgW0kbSlpVeAQYHoH0mHWrRwjZrU5Rsxqc4xYS7W9CVNELJP0MeBSYCXgzIiYPcRiU1qfsq7b9iDuc6e33RV6LEYGNa8M6ra7gmPE2+7ibXeFJmJkUH8vb7tJivAIB2ZmZmZmVp+OPEjOzMzMzMx6kwsQZmZmZmZWt44WIIZ6zLqk1SSdn7+/TtL4wnfH5+l3SNqnBdv+pKTbJN0s6UpJWxS+e07STfnVcKekOrZ9mKSHC9v4UOG7yZLuzK/JLdj2aYXt/l3SksJ3Te+3pDMlPSSp4jjTSr6R03WzpB0L3w1rn3uZY8QxUvjeMVKBY6S9MdKp+MjLO0aa4BhxjBS+H7kYiYiOvEideu4CtgJWBf4KbFs2z0eB7+X3hwDn5/fb5vlXA7bM61lphLf9VmDN/P4jpW3nz4+3eL8PA75VYdkNgLvz3/Xz+/VHcttl83+c1PFqJPb7LcCOwK1Vvt8X+A0gYBfgupHY515+OUYcI2XfO0aayyuOkRWnN51fOhkfeXnHSGvyimNkxemOkTr2uZM1EC88Zj0ingFKj1kvmgRMze8vAvaQpDx9WkQ8HRH3AHPy+kZs2xFxdUQ8mT9eSxpDeSTUs9/V7ANcHhGLImIxcDkwsYXbfg9wXgPrryoifgcsqjHLJOCcSK4F1pM0muHvcy9zjDhGihwjL+YYaW+MdCw+wDHSJMeIY6RoxGKkkwWISo9ZH1ttnohYBjwKbFjnssPddtERpBJbyeqSbpB0raSDGthuI9t+R65eukhS6WEwbdvvXI24JXBVYfJw9rvZtA13n3uZY6T2th0jjhHHSO1tj3SMdHN81EqfY2Q5x8iKHCNNxkjbnwPRayS9D5gA7FqYvEVEzJe0FXCVpFsi4q4R3OyvgPMi4mlJHybdGdh9BNdfj0OAiyLiucK0Vu+39SDHiGPEahvQGHF8WN0cIy/omRjpZA1EPY9Zf2EeSSsD6wIL61x2uNtG0p7AZ4EDI+Lp0vSImJ//3g3MAHYYyW1HxMLC9n4I/Esj6R7OtgsOoaxabZj73WzahrvPvcwxUmXbjpEV0uYYWc4xsnz9rYiRbo6PWulzjCznGFm+fsfIcGIkOtexZ2VSJ40tWd7RZLuyeY5mxY49F+T327Fix567aaxjTz3b3oHUEWabsunrA6vl9xsBd1Kjg0yT2x5deP9vwLWxvJPLPTkN6+f3G4zktvN8rwTuJT9ocCT2Oy83nuode/ZjxY49M0din3v55RhxjJR95xhpLq84RkYoRjodH3lZx0hj/y/HiGOkJTHS6Yy9L/D3nHk+m6d9gVQKBVgduJDUcWcmsFVh2c/m5e4A3taCbV8BPAjclF/T8/Q3ArfkTHELcEQLtv0lYHbextXAKwvLfjD/P+YAh4/0tvPnE4BTy5Yb1n6TStkLgGdJbeuOAI4CjsrfC/h2TtctwISR2udefjlGHCOOEcdIN8VIp+LDMeIYcYx0V4woL2RmZmZmZjYkP4nazMzMzMzq5gKEmZmZmZnVzQUIMzMzMzOrmwsQZmZmZmZWNxcgzMzMzMysbi5AmJmZmZlZ3VyA6EH5SZFmVoHjw6w2x4hZbY6RobkA0SaS1pL0a0l/lXSrpHdLer2kP+VpMyW9RNLqks6SdIukGyW9NS9/mKTpkq4CrszrOzMvd6OkSR3eRbOmOT7ManOMmNXmGGkvl7DaZyJwf0TsByBpXeBG4N0Rcb2kdYCngGOAiIjXSHolcJmkl+d17Ai8NiIWSfoicFVEfFDSesBMSVdExBNt3zOz4XN8mNXmGDGrzTHSRq6BaJ9bgL0kfVnSvwKbAwsi4nqAiFgaEcuANwPn5ml/A+4DShn78ohYlN/vDRwn6SZgBulR9Ju3a2fMRpjjw6w2x4hZbY6RNnINRJtExN8l7QjsC5wMXNXEaoqlXgHviIg7RiJ9Zp3k+DCrzTFiVptjpL1cA9EmksYAT0bEucBXgJ2B0ZJen79/iVKnnd8Dh+ZpLyeVditl3kuBj0tSnneH1u+FWWs4Psxqc4yY1eYYaS/XQLTPa4CvSHoeeBb4CKl0+01Ja5Da5e0JfAf4rqRbgGXAYRHxdM6/RScBXwduljQKuAfYvy17YjbyHB9mtTlGzGpzjLSRIqLTaTAzMzMzsx7hJkxmZmZmZlY3FyDMzMzMzKxuLkCYmZmZmVndXIAwMzMzM7O6uQBhZmZmZmZ1cwHCzMzMzMzq5gKEmZmZmZnV7f8D1ZD4sD7HnZ0AAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 792x180 with 4 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "fig, ax = plt.subplots(1,4, figsize=(11,2.5))\n",
    "ax[0].hist(detections_probs,np.arange(0.0,1.01,0.05),color='green'), \n",
    "ax[1].hist(detections_probs_TUPAC,np.arange(0.0,1.01,0.05),color='green')\n",
    "ax[2].hist(detections_hardexamples,np.arange(0.0,1.01,0.05),color='green')\n",
    "ax[3].hist(detections_probs_agreed,np.arange(0.0,1.01,0.05),color='green')\n",
    "print('Median model score, agreed mitoses:', np.median(detections_probs))\n",
    "print('Median model score, TUPAC-only mitoses:', np.median(detections_probs_TUPAC))\n",
    "print('Median model score, hard examples:', np.median(detections_hardexamples))\n",
    "print('Median model score, new mitotic fig:', np.median(detections_probs))\n",
    "\n",
    "for k in range(4):\n",
    "    ax[k].set_xlim([0.00,1.0])\n",
    "    yl = ax[k].get_ylim()\n",
    "    ax[k].set_ylim(yl)\n",
    "    ax[k].set_xlabel('score')\n",
    "    ax[k].set_ylabel('count')\n",
    "\n",
    "    \n",
    "\n",
    "ax[0].set_title('new mitotic figures')\n",
    "ax[1].set_title('TUPAC16-only mitotic figures')\n",
    "ax[2].set_title('hard negatives')\n",
    "ax[3].set_title('agreed mitotic figures')\n",
    "plt.tight_layout()\n",
    "plt.savefig('Histograms_crossvalidation_AL.svg')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}