moving to python3 step 1. testEnvironment works

jkiesele · jkiesele · commit 93c6e60acdee · 2020-03-26T21:38:06.000+01:00
diff --git a/DataCollection.py b/DataCollection.py
@@ -16,7 +16,7 @@
 import copy
 import time
 import logging
-from stopwatch import stopwatch
+from DeepJetCore.stopwatch import stopwatch
 logger = logging.getLogger(__name__)
 
 
@@ -550,8 +550,7 @@ def generatorFunction(self):
             xout = data.transferFeatureListToNumpy()
             wout = data.transferWeightListToNumpy()
             yout = data.transferTruthListToNumpy()
-            
-            
+        
             if len(wout)>0:
                 yield (xout,yout,wout)
             else:
diff --git a/TrainData.py b/TrainData.py
@@ -59,7 +59,7 @@ def readIn(self,fileprefix,shapesOnly=False):
     def _maybeConvertToSimpleArray(self,a):
         if str(type(a)) == "<class 'DeepJetCore.compiled.c_simpleArray.simpleArray'>":
             return a
-        elif str(type(a)) == "<type 'numpy.ndarray'>":
+        elif str(type(a)) == "<type 'numpy.ndarray'>" or str(type(a)) == "<class 'numpy.ndarray'>":
             rs = np.array([])
             arr = simpleArray()
             arr.createFromNumpy(a, rs)
diff --git a/__init__.py b/__init__.py
@@ -10,7 +10,4 @@
 import tensorflow as tf
 sys.modules["keras"] = tf.keras
 
-import matplotlib
-matplotlib.use('Agg')
-
-__version__ = '2.3'
+__version__ = '3.0'
diff --git a/bin/batch_conversion.py b/bin/batch_conversion.py
@@ -1,4 +1,4 @@
-#!/bin/env python
+#!/bin/env python3
 
 import os
 import logging
diff --git a/bin/check_conversion.py b/bin/check_conversion.py
@@ -1,4 +1,4 @@
-#! /bin/env python
+#! /bin/env python3
 
 from argparse import ArgumentParser
 from pdb import set_trace
diff --git a/bin/convertDCtoNumpy.py b/bin/convertDCtoNumpy.py
@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
 # encoding: utf-8
 
 from argparse import ArgumentParser
diff --git a/bin/convertFromSource.py b/bin/convertFromSource.py
@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
 # encoding: utf-8
 '''
 
diff --git a/bin/convertPre2.0DCtoDC.py b/bin/convertPre2.0DCtoDC.py
@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
 # encoding: utf-8
 '''
 
diff --git a/bin/convertPre2.0TDtoTD.py b/bin/convertPre2.0TDtoTD.py
@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
 # encoding: utf-8
 '''
 
diff --git a/bin/convertToPB.py b/bin/convertToPB.py
@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
 
 
 #script that takes model in .h5 format as input as spits out the graph format used in CMSSW
diff --git a/bin/convertToTF.py b/bin/convertToTF.py
@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
 
 import imp
 try:
diff --git a/bin/createDataCollectionFromTD.py b/bin/createDataCollectionFromTD.py
@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
 # encoding: utf-8
 
 
diff --git a/bin/createSubpackage.py b/bin/createSubpackage.py
@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
 # encoding: utf-8
 
 
diff --git a/bin/extractDjcCppInterface.py b/bin/extractDjcCppInterface.py
@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
 
 
 from argparse import ArgumentParser
diff --git a/bin/mergeDataCollections.py b/bin/mergeDataCollections.py
@@ -1,4 +1,4 @@
-#! /bin/env python
+#! /bin/env python3
 
 from argparse import ArgumentParser
 from DeepJetCore.DataCollection import DataCollection
diff --git a/bin/mergeOrSplitFiles.py b/bin/mergeOrSplitFiles.py
@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
 # encoding: utf-8
 '''
 
diff --git a/bin/plotLoss.py b/bin/plotLoss.py
@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
     
 
 
diff --git a/bin/predict.py b/bin/predict.py
@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
 
 from argparse import ArgumentParser
 
diff --git a/bin/prepare_for_deployment.py b/bin/prepare_for_deployment.py
@@ -1,4 +1,4 @@
-#! /bin/env python
+#! /bin/env python3
 
 from argparse import ArgumentParser
 
diff --git a/bin/testEnvironment.py b/bin/testEnvironment.py
@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
 
 print('importing tensorflow...')
 
@@ -29,7 +29,7 @@
 b = keras.layers.Dense(10)(a)
 model = keras.models.Model(inputs=a, outputs=b)
 model.compile(optimizer='adam',loss='mse')
-model.fit(x=features, y=truth, batch_size=100, epochs=20)
+model.fit(x=features, y=truth, batch_size=100, epochs=3)
 
 print('loading DeepJetCore compiled library...')
 
@@ -55,7 +55,7 @@
 cd {djc_base}/testing
 rm -rf batchExplode
 export PYTHONPATH=`pwd`:$PYTHONPATH
-python batch_explosion.py batchDC/dataCollection.djcdc batchExplode
+python3 batch_explosion.py batchDC/dataCollection.djcdc batchExplode
 '''.format(djc_base=djc_base)
 os.system(script)
 
diff --git a/bin/validateData.py b/bin/validateData.py
@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
 
 
 
diff --git a/compiled/Makefile b/compiled/Makefile
@@ -1,7 +1,7 @@
 
 # location of the Boost Python include files and library
-PYTHON_INCLUDE = `python-config --includes`
-PYTHON_LIB=`python-config --libs`
+PYTHON_INCLUDE = `python3-config --includes`
+PYTHON_LIB=`python3-config --libs`
 
 
 LINUXADD= #-Wl --export-dynamic
@@ -28,7 +28,7 @@ classdict.so: classdict.cxx
 
 
 %: to_bin/%.cpp libdeepjetcorehelpers.so classdict.cxx
-	g++ $(CFLAGS) -I./interface $(LINUXADD) $(PYTHON_INCLUDE) $< -L. -ldeepjetcorehelpers -lquicklz  $(PYTHON_LIB) -lboost_python -lboost_numpy $(ROOTCFLAGS) $(ROOTSTUFF)   -o  $@  
+	g++ $(CFLAGS) -I./interface $(LINUXADD) $(PYTHON_INCLUDE) $< -L. -ldeepjetcorehelpers -lquicklz  $(PYTHON_LIB) -lboost_python3 -lboost_numpy3 $(ROOTCFLAGS) $(ROOTSTUFF)   -o  $@  
 	mv $@ ../bin/
 
 #helpers
@@ -40,10 +40,10 @@ obj/%.o: src/%.cpp
 
 #pack helpers in lib
 libdeepjetcorehelpers.so: $(OBJ_FILES) 
-	g++ -o $@ -shared -fPIC  $(LINUXADD) -fPIC  obj/*.o $(ROOTSTUFF) $(PYTHON_LIB) -lboost_python -lboost_numpy 
+	g++ -o $@ -shared -fPIC  $(LINUXADD) -fPIC  obj/*.o $(ROOTSTUFF) $(PYTHON_LIB) -lboost_python3 -lboost_numpy3 
 
 %.so: %.o libdeepjetcorehelpers.so libquicklz.so
-	g++  -o $(@) -shared -fPIC  $(LINUXADD) $<   $(ROOTSTUFF) -L./ -lquicklz  $(PYTHON_LIB) -lboost_python -lboost_numpy -L./ -ldeepjetcorehelpers 
+	g++  -o $(@) -shared -fPIC  $(LINUXADD) $<   $(ROOTSTUFF) -L./ -lquicklz  $(PYTHON_LIB) -lboost_python3 -lboost_numpy3 -L./ -ldeepjetcorehelpers 
 
 %.o: src/%.C 
 	g++   $(ROOTCFLAGS) -O2 -I./interface $(PYTHON_INCLUDE) -fPIC -c -o $(@) $<
diff --git a/conversion/conversion.py b/conversion/conversion.py
@@ -1,4 +1,5 @@
 try:
+    import datastructures
     from datastructures import *
 except ImportError:
     print('datastructure modules not found. Please define a DeepJetCore submodule')
diff --git a/docker/Dockerfile b/docker/Dockerfile
@@ -1,20 +1,23 @@
 
-FROM cernml4reco/djcbase:py2_latest
+FROM cernml4reco/djcbase:devpy3
+
+
+
+################################################################################
+# Tensorflow
 
 RUN export DEBIAN_FRONTEND=noninteractive && \
-    pip --no-cache-dir install tensorflow-gpu==2.1 && \
-    cd /usr/local/lib/python2.7/dist-packages/tensorflow_core/include/third_party && \
-    mkdir gpus && \
-    cd gpus && \
-    ln -s /usr/local/cuda cuda
-    
-## torch/jupyter stuff 
-RUN pip install torch future gensim jupyter prompt-toolkit seaborn
+    pip3 --no-cache-dir install tensorflow-gpu==2.2.0-rc1 
     
-EXPOSE 8888
-
 
     
+#RUN cd /usr/local/lib/python3.6/dist-packages/tensorflow_core/include/third_party && \
+#    mkdir gpus && \
+#    cd gpus && \
+#    ln -s /usr/local/cuda cuda
+    
+    
+    
 ################################################################################
 # the actual DeepJetCore. Make sure this gets built every time
 
diff --git a/docker/Dockerfile_base b/docker/Dockerfile_base
@@ -4,14 +4,28 @@ FROM nvidia/cuda:10.1-cudnn7-devel-ubuntu18.04
 
 SHELL ["/bin/bash", "-c"]
 
-RUN apt-get update
-RUN apt-get install -y python2.7-dev python-pip nano emacs
-RUN apt-get install -y wget
-RUN apt-get install -y libboost-all-dev
-RUN apt-get install -y git
+RUN sed -i "s,# deb http://archive.canonical.com/ubuntu,deb http://archive.canonical.com/ubuntu,g" /etc/apt/sources.list &&  apt-get update --fix-missing 
+
+
+#base
+RUN apt-get install -y python3.6-dev python3-pip
+RUN pip3 install --upgrade pip
+RUN ldconfig
+
+#basic user tools
+RUN export DEBIAN_FRONTEND=noninteractive && \
+    apt-get install -y git wget nano emacs \
+    evince eog ffmpeg unzip zsh python3-tk locales \
+    htop
+RUN apt-get update --fix-missing
 
+RUN export DEBIAN_FRONTEND=noninteractive && \
+    apt-get install -y krb5-user && \
+    sed -i 's/default_realm = ATHENA.MIT.EDU/default_realm = CERN.CH/g' /etc/krb5.conf 
+RUN apt-get update --fix-missing
 
-## root part
+
+############  root part
 RUN apt-get install -y libx11-dev libxpm-dev libxft-dev libxext-dev
 RUN apt-get install -y cmake
 
@@ -32,78 +46,23 @@ RUN cd /temproot/build && \
 
 #ensure import ROOT
 ENV PYTHONPATH="/usr/local/lib:${PYTHONPATH}"
+RUN ldconfig
 
+############  END root part
 
-RUN export DEBIAN_FRONTEND=noninteractive && \
-    apt-get install -y python-tk
-RUN apt-get install -y locales
-
-RUN pip install --upgrade pip
-
-RUN pip install numpy
-RUN pip install root_numpy
-RUN pip install scikit-learn
-RUN pip install scikit-image
-RUN pip install h5py
-RUN pip install uproot Pillow scipy seaborn
-RUN pip install scikit-learn
-RUN pip install scikit-image
-RUN pip install matplotlib
-
-#may be useful
-RUN pip install opencv-python
-RUN pip install easydict tqdm
-RUN pip install Cython
-RUN pip install numba
-
-#already in the container from nvidia
-#RUN wget https://developer.download.nvidia.com/compute/cuda/repos/ubuntu1804/x86_64/cuda-repo-ubuntu1804_10.1.243-1_amd64.deb &&\
-#    dpkg -i cuda-repo-ubuntu1804_10.1.243-1_amd64.deb &&\
-#    apt-key adv --fetch-keys https://developer.download.nvidia.com/compute/cuda/repos/ubuntu1804/x86_64/7fa2af80.pub &&\
-#    apt-get update &&\
-#    wget http://developer.download.nvidia.com/compute/machine-learning/repos/ubuntu1804/x86_64/nvidia-machine-learning-repo-ubuntu1804_1.0.0-1_amd64.deb &&\
-#    apt install -y ./nvidia-machine-learning-repo-ubuntu1804_1.0.0-1_amd64.deb &&\
-#    apt-get update
-#    
-#RUN export DEBIAN_FRONTEND=noninteractive && \
-#    apt-get install -y  --allow-downgrades --allow-change-held-packages --no-install-recommends \
-#    cuda-10-1 \
-#    libcudnn7=7.6.4.38-1+cuda10.1  \
-#    libcudnn7-dev=7.6.4.38-1+cuda10.1
-#    
-##will also get new cuda version
-#RUN export DEBIAN_FRONTEND=noninteractive && \
-#    apt-get install -y --no-install-recommends libnvinfer6=6.0.1-1+cuda10.1 \
-#    libnvinfer-dev=6.0.1-1+cuda10.1 \
-#    libnvinfer-plugin6=6.0.1-1+cuda10.1
-
-
-RUN export DEBIAN_FRONTEND=noninteractive && \
-    apt-get update --fix-missing 
-    
-RUN export DEBIAN_FRONTEND=noninteractive && \
-    apt-get install -y krb5-user && \
-    sed -i 's/default_realm = ATHENA.MIT.EDU/default_realm = CERN.CH/g' /etc/krb5.conf 
-    
-#for me ;)    
-
-RUN sed -i "s,# deb http://archive.canonical.com/ubuntu,deb http://archive.canonical.com/ubuntu,g" /etc/apt/sources.list
-
-RUN apt-get update --fix-missing 
-RUN apt-get install -y zsh 
-
-RUN apt-get install -y ffmpeg
-#not yet RUN apt-get install -y nvtop
+### boost
+RUN apt-get install -y libboost-all-dev
 
-##gpu specific stuff
-RUN pip install gpustat 
-RUN pip install setGPU
 
+############  PIP packages
 
-RUN apt-get install -y unzip
+RUN pip3 install --upgrade setuptools
+RUN pip3 install numpy root_numpy scikit-learn scikit-image h5py matplotlib uproot Pillow scipy seaborn 
+RUN pip3 install opencv-python easydict tqdm Cython numba
 
-RUN apt-get install -y evince eog
+RUN pip3 install gpustat setGPU
 
+## torch/jupyter stuff 
+RUN pip3 install torch future gensim jupyter prompt-toolkit 
+EXPOSE 8888
 
-RUN sed -i "s,# deb http://archive.canonical.com/ubuntu,deb http://archive.canonical.com/ubuntu,g" /etc/apt/sources.list
-RUN apt update
diff --git a/evaluation/__init__.py b/evaluation/__init__.py
@@ -4,7 +4,7 @@
 from pdb import set_trace
 
 #gather all the files here
-modules = [basename(i.replace('.py','')) for i in glob('%s/[A-Za-z]*.py' % dirname(__file__))]
+modules = ["DeepJetCore.evaluation."+basename(i.replace('.py','')) for i in glob('%s/[A-Za-z]*.py' % dirname(__file__))]
 __all__ = []
 structure_list=[]
 for module_name in modules:
diff --git a/preprocessing/__init__.py b/preprocessing/__init__.py
@@ -4,7 +4,7 @@
 from pdb import set_trace
 
 #gather all the files here
-modules = [basename(i.replace('.py','')) for i in glob('%s/[A-Za-z]*.py' % dirname(__file__))]
+modules = ["DeepJetCore.preprocessing."+basename(i.replace('.py','')) for i in glob('%s/[A-Za-z]*.py' % dirname(__file__))]
 __all__ = []
 structure_list=[]
 for module_name in modules:
diff --git a/testing/batch_explosion.py b/testing/batch_explosion.py
diff --git a/testing/datastructures/__init__.py b/testing/datastructures/__init__.py
diff --git a/testing/files/filelist.txt b/testing/files/filelist.txt
diff --git a/training/DeepJet_callbacks.py b/training/DeepJet_callbacks.py
diff --git a/training/tokenTools.py b/training/tokenTools.py
diff --git a/training/training_base.py b/training/training_base.py

Original file line number	Diff line number	Diff line change
`@@ -1,4 +1,4 @@`
`1`		`-#!/bin/env python`
	`1`	`+#!/bin/env python3`
`2`	`2`
`3`	`3`	`import os`
`4`	`4`	`import logging`
Original file line number	Diff line number	Diff line change
`@@ -1,4 +1,4 @@`
`1`		`-#! /bin/env python`
	`1`	`+#! /bin/env python3`
`2`	`2`
`3`	`3`	`from argparse import ArgumentParser`
`4`	`4`	`from pdb import set_trace`
Original file line number	Diff line number	Diff line change
`@@ -1,4 +1,4 @@`
`1`		`-#!/usr/bin/env python`
	`1`	`+#!/usr/bin/env python3`
`2`	`2`	`# encoding: utf-8`
`3`	`3`
`4`	`4`	`from argparse import ArgumentParser`
Original file line number	Diff line number	Diff line change
`@@ -1,4 +1,5 @@`
`1`	`1`	`try:`
	`2`	`+ import datastructures`
`2`	`3`	`from datastructures import *`
`3`	`4`	`except ImportError:`
`4`	`5`	`print('datastructure modules not found. Please define a DeepJetCore submodule')`