fix install in venv

README.md

@@ -282,6 +282,7 @@ ImportError: /home//lib/python3.6/site-packages/mgeconvert/backend/ir_to_tflite/
 | matrix mul               | ✓<br/>✓ | ✓<br/>✓ | ✓<br/>✓ |
 | max pool2d               | ✓<br/>✓ | ✓<br/>✓ | ✓<br/>✓ |
 | mul                      | ✓<br/>✓ | ✓<br/>✓ | ✓<br/>✓ |
+| pad                      | ✓<br/>× | ×<br/>× | ×<br/>× |
 | pow                      | ✓<br/>✓ | ✓<br/>✓ | ✓<br/>✓ |
 | reduce max               | ✓<br/>✓ | ✓<br/>✓ | ✓<br/>✓ |
 | reduce min               | ✓<br/>✓ | ✓<br/>✓ | ✓<br/>✓ |
@@ -314,6 +315,7 @@ ImportError: /home//lib/python3.6/site-packages/mgeconvert/backend/ir_to_tflite/
 | Concat                   | ✓<br/>✓ |
 | Dropout                  | ✓<br/>✓ |
 | Flatten                  | ✓<br/>✓ |
+| Gather                   | ✓<br/>✓ |
 | Gemm                     | ✓<br/>✓ |
 | GlobalAveragePool        | ✓<br/>✓ |
 | GlobalMaxPool            | ✓<br/>✓ |

ci/pytest_caffe_and_onnx.sh

@@ -8,7 +8,7 @@ python3 -m pip install --no-binary=protobuf protobuf==3.11.1
 
 apt install -y protobuf-compiler
 
-./mgeconvert/backend/ir_to_caffe/init.sh
+./mgeconvert/backend/ir_to_caffe/init.sh False /usr/bin/python3
 
 pip3 install scikit-image==0.17.2 onnx onnxruntime
 

ci/pytest_caffe_and_onnx_complete.sh

@@ -8,7 +8,7 @@ python3 -m pip install --no-binary=protobuf protobuf==3.11.1
 
 apt install -y protobuf-compiler
 
-./mgeconvert/backend/ir_to_caffe/init.sh
+./mgeconvert/backend/ir_to_caffe/init.sh False /usr/bin/python3
 
 pip3 install scikit-image==0.17.2 onnx onnxruntime
 

ci/pytest_tflite.sh

@@ -3,7 +3,7 @@
 set -e
 
 ./mgeconvert/backend/ir_to_tflite/build_flatbuffer.sh
-./mgeconvert/backend/ir_to_tflite/init.sh
+./mgeconvert/backend/ir_to_tflite/init.sh False /usr/bin/python3
 
 sudo python3 -m pip uninstall flatbuffers -y
 sudo python3 -m pip install tensorflow==2.5.0

ci/pytest_tflite_complete.sh

@@ -3,7 +3,7 @@
 set -e
 
 ./mgeconvert/backend/ir_to_tflite/build_flatbuffer.sh
-./mgeconvert/backend/ir_to_tflite/init.sh
+./mgeconvert/backend/ir_to_tflite/init.sh False /usr/bin/python3
 
 # try to find libflatbuffers.so
 export LD_LIBRARY_PATH=$HOME/.local/lib:$LD_LIBRARY_PATH:

mgeconvert/backend/ir_to_caffe/init.sh

@@ -1,6 +1,13 @@
 #!/bin/bash -e
 
-python3 -m pip install --no-binary=protobuf "protobuf>=3.11.1" --user
+ADD_USER=""
+if [[ $1 == "False" ]]; then
+    ADD_USER="--user"
+fi
+
+PYTHON3=$2
+
+$PYTHON3 -m pip install --no-binary=protobuf "protobuf>=3.11.1" $ADD_USER
 
 hash wget || (echo "please install wget package" && exit -1)
 hash protoc || (echo "please install protobuf-compiler package" && exit -1)

mgeconvert/backend/ir_to_onnx/init.sh

@@ -1,5 +1,12 @@
 #!/bin/bash -e
 
-python3 -m pip install "onnx>=1.7.0" --user
-python3 -m pip install onnx-simplifier --user
-python3 -m pip install protobuf --user
+ADD_USER=""
+if [[ $1 == "False" ]]; then
+    ADD_USER="--user"
+fi
+
+PYTHON3=$2
+
+$PYTHON3 -m pip install "onnx>=1.7.0,<1.12.0" $ADD_USER
+$PYTHON3 -m pip install onnx-simplifier $ADD_USER
+$PYTHON3 -m pip install protobuf $ADD_USER

mgeconvert/backend/ir_to_tflite/init.sh

@@ -1,19 +1,25 @@
 #!/bin/bash -e
 basepath=$(cd `dirname $0`; pwd)
 
+ADD_USER=""
+if [[ $1 == "False" ]]; then
+    ADD_USER="--user"
+fi
+
+PYTHON3=$2
 
-if python3 -c "import flatbuffers">/dev/null 2>&1; then
-    FLAT_BUFFER_VERSION="$(python3 -m pip show flatbuffers | grep Version)"
+if $PYTHON3 -c "import flatbuffers">/dev/null 2>&1; then
+    FLAT_BUFFER_VERSION="$($PYTHON3 -m pip show flatbuffers | grep Version)"
 else
     FLAT_BUFFER_VERSION=""
 fi
 echo ${FLAT_BUFFER_VERSION}
 
 # install flatbuffers
 if [[ "$FLAT_BUFFER_VERSION" != "Version: 1.12" ]]; then
-    python3 -m pip uninstall flatbuffers -y
+    $PYTHON3 -m pip uninstall flatbuffers -y
     echo "install flatbuffers..."
-    python3 -m pip install flatbuffers==1.12 --user
+    $PYTHON3 -m pip install flatbuffers==1.12 $ADD_USER
 fi
 
 if [ ! -d /tmp/mgeconvert ]; then
@@ -26,8 +32,8 @@ export PATH=$PATH:$HOME/.local/bin
 echo "building tflite schema..."
 cd /tmp/mgeconvert
 rm -f schema.fbs
-tf_version=$1
-if [ ! -n "$1" ] ;then
+tf_version=$3
+if [ ! -n "$3" ] ;then
     tf_version="r2.3"
 fi
 echo "Use TFLite $tf_version!"
@@ -37,13 +43,13 @@ chmod 777 /tmp/mgeconvert/tflite
 cp -r /tmp/mgeconvert/tflite $basepath
 
 
-python3 -m pip install pybind11==2.6.2 --user
+$PYTHON3 -m pip install pybind11==2.6.2 $ADD_USER
 
 # using pyflexbuffers
 cd $basepath/pyflexbuffers
-PYBIND11_HEADER=$(python3 -c "import pybind11; print(pybind11.get_include())")
-PYTHON_INCLUDE=$(python3 -c "import sysconfig; print(sysconfig.get_paths()['include'])")
-PYTHON_STDLIB=$(python3 -c "import sysconfig; print(sysconfig.get_paths()['stdlib'])")
+PYBIND11_HEADER=$($PYTHON3 -c "import pybind11; print(pybind11.get_include())")
+PYTHON_INCLUDE=$($PYTHON3 -c "import sysconfig; print(sysconfig.get_paths()['include'])")
+PYTHON_STDLIB=$($PYTHON3 -c "import sysconfig; print(sysconfig.get_paths()['stdlib'])")
 
 g++ fbconverter.cc --std=c++14 -fPIC --shared -I$basepath/pyflexbuffers/include -I${PYBIND11_HEADER} -I${PYTHON_INCLUDE} -L${PYTHON_STDLIB} -L$basepath/pyflexbuffers/lib  -lflatbuffers -o fbconverter.so
 

setup.py

@@ -15,6 +15,9 @@
 
 targets = []
 tfversion = None
+IS_VENV = hasattr(sys, "real_prefix") or (
+    hasattr(sys, "base_prefix") and sys.base_prefix != sys.prefix
+)
 
 
 def write_init(targets, tflite_schema_version=None):
@@ -97,9 +100,11 @@ def find_extension(self, name):
     def build_all(self, ext):
         if ext.script:
             if ext.name == "tflite" and tfversion is not None:
-                subprocess.check_call([ext.script, tfversion])
+                subprocess.check_call(
+                    [ext.script, str(IS_VENV), sys.executable, tfversion]
+                )
             else:
-                subprocess.check_call(ext.script)
+                subprocess.check_call([ext.script, str(IS_VENV), sys.executable])
         if ext.artifacts is not None:
             self.copy_tree(ext.artifacts, os.path.join(self.build_lib, ext.artifacts))
 

test/mge/test_tflite.py

@@ -32,7 +32,7 @@
 from mgeconvert.converters.mge_to_tflite import mge_to_tflite
 from tensorflow.lite.python import interpreter  # pylint: disable=import-error
 
-max_error = 1e-6
+max_error = 1e-4
 tmp_file = "test_model"
 
 

test/traced_module/test_qat_tflite.py

@@ -23,6 +23,11 @@
 
 from .tm_utils import get_traced_module
 
+if mge.__version__ > "1.7.0":
+    from megengine.traced_module.tm_config import disable_default_checker
+
+    disable_default_checker()
+
 max_error = 1e-4
 tmp_file = "test_model"
 

test/traced_module/test_tflite.py

@@ -40,7 +40,7 @@
 from mgeconvert.converters.tm_to_tflite import tracedmodule_to_tflite
 from tensorflow.lite.python import interpreter  # pylint: disable=import-error
 
-max_error = 1e-6
+max_error = 1e-4
 tmp_file = "test_model"