Added even more stuff

2nd_stage/Inference-CellClassifier.py

@@ -0,0 +1,36 @@
+"""
+
+        Process 2nd stage of approach 
+        
+        Requires:
+            Results of first stage (pickle format: .p)
+            2nd stage model (.pth)
+
+        Outputs:
+            Will output a file with same name as first results, but with a prefix of "2ndstage_".
+        
+        Syntax:
+            classify_2ndStage.py results.p model.pth [imagePath]
+            
+        
+        Marc Aubreville, Pattern Recognition Lab, FAU Erlangen-Nürnberg, 2019
+
+"""
+import sys
+
+sys.path.append('../')
+from lib.processSecondStageClassification import *
+import pickle
+import sys,os
+if len(sys.argv)<2:
+    print('Syntax: classify_2ndStage.py results.p model.pth')
+    exit()
+fname = sys.argv[1]
+p = pickle.load(open(fname, 'rb'))
+path,fname = os.path.split(fname)
+if len(path)>0:
+    path+='/'
+basepath='../../WSI/' if len(sys.argv)<=3 else sys.argv[3]
+modelpath='CellClassifier_128px.pth' if len(sys.argv)<=2 else sys.argv[2]
+out_2ndstage = processSecondStageClassifier(p,  intermediate=path+'2ndstage_'+fname, modelpath=modelpath, basepath=basepath)
+pickle.dump(out_2ndstage, open(path+'2ndstage_'+fname,'wb'))

2nd_stage/README.md

@@ -0,0 +1,10 @@
+## Training of the second stage detector model
+
+Similar to the works of Li. et al (DeepMitosis -> Deep Detection and Verification), we train a second stage classifier with patches of the cell embedded in its immediate cellular context. The patches are of size 128px X 128px. 
+
+The process is as follows:
+1. We extract patches, allowing us to use standard pipelines used for image classification
+2. We train a classifier for each data set variant
+3. We process the results of the last stage with these classifiers
+
+

2nd_stage/exportDataset_HEAEL.py

@@ -0,0 +1,86 @@
+import numpy as np 
+import SlideRunner.general.dependencies
+from SlideRunner.dataAccess.database import Database
+from SlideRunner.dataAccess.annotations import ViewingProfile
+import os
+import openslide
+import sqlite3
+import cv2
+
+#os.system('cp Slides_final.sqlite Slides_final_cleaned.sqlite')
+
+DB = Database()
+
+vp = ViewingProfile()
+vp.majorityClassVote=True
+
+cm=np.zeros((7,7))
+
+threshold = 5
+
+disagreedclass = 0
+agreedclass = 0
+basepath='../WSI/'
+patchSize=128
+
+os.system('mkdir -p DataHEAEL')
+
+dirs = ['Mitosis', 'Mitosislike', 'Tumorcells', 'Granulocytes']
+for k in dirs:
+    os.system('mkdir -p DataHEAEL/train/%s' % k)
+    os.system('mkdir -p DataHEAEL/test/%s' % k)
+
+def listOfSlides(DB):
+    DB.execute('SELECT uid,filename from Slides')
+    return DB.fetchall()
+
+test_slide_filenames = ['3369_07_B_1_MCT Mitose 2017.svs',
+ '3786_09 A MCT Mitose 2017.svs',
+ '1659_08_1_MCT Mitose 2017.svs',
+ '28_08_A_1_MCT Mitose 2017.svs',
+ '3806_09_B_1_MCT Mitose 2017.svs',
+ '2253_06_A_1_MCT Mitose 2017.svs',
+ '1410_08_A_1_MCT Mitose 2017.svs',
+ '1490_08_1_MCT Mitose 2017.svs',
+ '2281_14_A_1_MCT Mitose 2017.svs',
+ '221_08 MCT Mitose 2017.svs',
+ '5187_11 B MCT Mitose 2017.svs']
+
+DB.open('MITOS_WSI_CMCT_HEAEL.sqlite')#Slides_final_cleaned_checked.sqlite')
+
+for slide,filename in listOfSlides(DB):
+    DB.loadIntoMemory(slide)
+
+
+    slide=openslide.open_slide(basepath+filename)
+
+    for k in DB.annotations.keys():
+
+        anno = DB.annotations[k]
+
+        coord_x = anno.x1
+        coord_y = anno.y1
+
+        lu_x = int(coord_x - int(patchSize/2))
+        lu_y = int(coord_y - int(patchSize/2))
+        img = np.array(slide.read_region(location=(lu_x, lu_y), level=0, size=(patchSize, patchSize)))
+        img = cv2.cvtColor(img, cv2.COLOR_RGBA2BGR)
+
+#        DB.annotations[k].draw(leftUpper=(lu_x,lu_y), image=img, thickness=2, vp=vp, zoomLevel=1.0)
+
+        istest = 'train/' if filename not in test_slide_filenames else 'test/'
+        if (anno.agreedClass==2):
+            if not cv2.imwrite('DataHEAEL/'+istest+'Mitosis/%d.png' % (k), img):
+                  print('Write failed: '+'DataHEAEL/'+istest+'Mitosis/%d.png' % (k))
+
+        if (anno.agreedClass==7):
+            cv2.imwrite('DataHEAEL/'+istest+'Mitosislike/%d.png' % k, img)
+
+        if (anno.agreedClass==3):
+            cv2.imwrite('DataHEAEL/'+istest+'Tumorcells/%d.png' %k, img)
+
+        if (anno.agreedClass==1):
+            cv2.imwrite('DataHEAEL/'+istest+'Granulocytes/%d.png' %k, img) 
+
+
+

2nd_stage/exportDataset_MEL.py

@@ -0,0 +1,80 @@
+# Export Dataset (patch-wise) to learn classifier easier
+
+
+import numpy as np 
+import SlideRunner.general.dependencies
+from SlideRunner.dataAccess.database import Database
+from SlideRunner.dataAccess.annotations import ViewingProfile
+import os
+import openslide
+import sqlite3
+import cv2
+import sys
+
+DB = Database()
+
+basepath='../WSI/'
+patchSize=128
+
+os.system('mkdir -p DataMEL')
+
+dirs = ['Mitosis', 'Mitosislike', 'Tumorcells', 'Granulocytes']
+for k in dirs:
+    os.system('mkdir -p DataMEL/train/%s' % (k))
+    os.system('mkdir -p DataMEL/test/%s' % (k))
+
+def listOfSlides(DB):
+    DB.execute('SELECT uid,filename from Slides')
+    return DB.fetchall()
+
+test_slide_filenames = ['3369_07_B_1_MCT Mitose 2017.svs',
+ '3786_09 A MCT Mitose 2017.svs',
+ '1659_08_1_MCT Mitose 2017.svs',
+ '28_08_A_1_MCT Mitose 2017.svs',
+ '3806_09_B_1_MCT Mitose 2017.svs',
+ '2253_06_A_1_MCT Mitose 2017.svs',
+ '1410_08_A_1_MCT Mitose 2017.svs',
+ '1490_08_1_MCT Mitose 2017.svs',
+ '2281_14_A_1_MCT Mitose 2017.svs',
+ '221_08 MCT Mitose 2017.svs',
+ '5187_11 B MCT Mitose 2017.svs']
+
+DB.open('../databases/MITOS_WSI_CCMCT_MEL.sqlite')
+
+for slide,filename in listOfSlides(DB):
+    DB.loadIntoMemory(slide)
+
+
+    slide=openslide.open_slide(basepath+filename)
+
+    for k in DB.annotations.keys():
+
+        anno = DB.annotations[k]
+
+        coord_x = anno.x1
+        coord_y = anno.y1
+
+        lu_x = int(coord_x - int(patchSize/2))
+        lu_y = int(coord_y - int(patchSize/2))
+        img = np.array(slide.read_region(location=(lu_x, lu_y), level=0, size=(patchSize, patchSize)))
+        img = cv2.cvtColor(img, cv2.COLOR_RGBA2BGR)
+
+        istest = 'train/' if filename not in test_slide_filenames else 'test/'
+        if (anno.agreedClass==2):
+            fname = ('DataMEL/')+istest+'Mitosis/%d.png' % (k)
+            if not cv2.imwrite(fname, img):
+                  print('Write failed: ',fname)
+
+        if (anno.agreedClass==7):
+            cv2.imwrite(('DataMEL/')+istest+'Mitosislike/%d.png' % k, img)
+
+        if (anno.agreedClass==3):
+            cv2.imwrite(('DataMEL/')+istest+'Tumorcells/%d.png' %k, img)
+
+        if (anno.agreedClass==1):
+            cv2.imwrite(('DataMEL/')+istest+'Granulocytes/%d.png' %k, img) 
+
+
+
+
+

2nd_stage/exportDataset_ODAEL.py

@@ -0,0 +1,78 @@
+# Export Dataset (patch-wise) to learn classifier easier
+
+
+import numpy as np 
+import SlideRunner.general.dependencies
+from SlideRunner.dataAccess.database import Database
+from SlideRunner.dataAccess.annotations import ViewingProfile
+import os
+import openslide
+import sqlite3
+import cv2
+import sys
+
+DB = Database()
+
+basepath='../WSI/'
+patchSize=128
+
+os.system('mkdir -p DataODAEL')
+
+dirs = ['Mitosis', 'Mitosislike', 'Tumorcells', 'Granulocytes']
+for k in dirs:
+    os.system('mkdir -p DataODAEL/train/%s' % (k))
+    os.system('mkdir -p DataODAEL/test/%s' % (k))
+
+def listOfSlides(DB):
+    DB.execute('SELECT uid,filename from Slides')
+    return DB.fetchall()
+
+test_slide_filenames = ['3369_07_B_1_MCT Mitose 2017.svs',
+ '3786_09 A MCT Mitose 2017.svs',
+ '1659_08_1_MCT Mitose 2017.svs',
+ '28_08_A_1_MCT Mitose 2017.svs',
+ '3806_09_B_1_MCT Mitose 2017.svs',
+ '2253_06_A_1_MCT Mitose 2017.svs',
+ '1410_08_A_1_MCT Mitose 2017.svs',
+ '1490_08_1_MCT Mitose 2017.svs',
+ '2281_14_A_1_MCT Mitose 2017.svs',
+ '221_08 MCT Mitose 2017.svs',
+ '5187_11 B MCT Mitose 2017.svs']
+
+DB.open('../databases/MITOS_WSI_CCMCT_ODAEL.sqlite')
+
+for slide,filename in listOfSlides(DB):
+    DB.loadIntoMemory(slide)
+
+
+    slide=openslide.open_slide(basepath+filename)
+
+    for k in DB.annotations.keys():
+
+        anno = DB.annotations[k]
+
+        coord_x = anno.x1
+        coord_y = anno.y1
+
+        lu_x = int(coord_x - int(patchSize/2))
+        lu_y = int(coord_y - int(patchSize/2))
+        img = np.array(slide.read_region(location=(lu_x, lu_y), level=0, size=(patchSize, patchSize)))
+        img = cv2.cvtColor(img, cv2.COLOR_RGBA2BGR)
+
+        istest = 'train/' if filename not in test_slide_filenames else 'test/'
+        if (anno.agreedClass==2):
+            fname = ('DataODAEL/')+istest+'Mitosis/%d.png' % (k)
+            if not cv2.imwrite(fname, img):
+                  print('Write failed: ',fname)
+
+        if (anno.agreedClass==7):
+            cv2.imwrite(('DataODAEL/')+istest+'Mitosislike/%d.png' % k, img)
+
+        if (anno.agreedClass==3):
+            cv2.imwrite(('DataODAEL/')+istest+'Tumorcells/%d.png' %k, img)
+
+        if (anno.agreedClass==1):
+            cv2.imwrite(('DataODAEL/')+istest+'Granulocytes/%d.png' %k, img) 
+
+
+