Ncnn (#6)

* Support ncnn benchmark

* Import ncnn and available_benchmark to the init scope

edgebenchmark/__init__.py

@@ -1 +1,3 @@
 from edgebenchmark.tflite_benchmark import TFLiteBenchmark
+from edgebenchmark.ncnn_benchmark import NcnnBenchmark
+from edgebenchmark.settings import available_benchmarks

edgebenchmark/edgebenchmark.py

@@ -14,6 +14,7 @@
 # ==============================================================================
 import sys
 import json
+import logging
 from abc import ABC
 from pathlib import Path
 from typing import List

edgebenchmark/edgebenchmark_cli.py

@@ -39,11 +39,14 @@
 from edgebenchmark.tflite_benchmark import TFLiteBenchmark_2_3_0
 from edgebenchmark.tflite_benchmark import TFLiteBenchmark_2_4_0
 
+from edgebenchmark.ncnn_benchmark import NcnnBenchmark_20210124
+
 
 @click.group()
 def cli_configure():
     pass
 
+
 @cli_configure.command()
 def configure():
     try:
@@ -77,8 +80,6 @@ def cli_devices():
 
 @cli_devices.command()
 def devices():
-    import requests
-
     token = load_token_from_file()
 
     response = get_devices(
@@ -394,19 +395,34 @@ def tflite_2_4_1(model_path, device, features, **benchmark_args):
     )
 
 
-# @main.group()
-# def ncnn():
-#     pass
+def ncnn_options(NcnnBenchmark_class):
+    def wrapper(fn):
+        for name, type in NcnnBenchmark_class.parameters().items():
+            fn = click.option(f"--{name}", type=type)(fn)
+        return fn
+
+    return wrapper
 
 
-# @ncnn.command("1.0.0")
-# def ncnn_1_0_0():
-#     print("1.0.0")
+@click.group()
+def ncnn():
+    pass
 
 
-# @ncnn.command("2.0.0")
-# def ncnn_2_0_0():
-#     print("2.0.0")
+@ncnn.command("20210124")
+@common_benchmark_options
+@ncnn_options(NcnnBenchmark_20210124)
+def ncnn_20210124(model_path, device, features, **benchmark_args):
+    benchmark_args = filter_dict(benchmark_args)
+
+    benchmark(
+        model_path,
+        device,
+        features,
+        available_benchmarks.ncnn_basic,
+        "20210124",
+        benchmark_args,
+    )
 
 
 def benchmark(
@@ -438,7 +454,7 @@ def benchmark(
         print("Model was successfuly sent for benchmarking. Please check the benchmarking result through https://edgebenchmark.com/app website")
 
 
-cli = click.CommandCollection(sources=[cli_configure, cli_tflite, cli_devices])
+cli = click.CommandCollection(sources=[cli_configure, cli_devices, cli_tflite, cli_ncnn])
 
 
 if __name__ == "__main__":

edgebenchmark/ncnn_benchmark.py

@@ -0,0 +1,32 @@
+# Copyright 2021 Bisonai Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+from edgebenchmark.settings import settings
+from edgebenchmark.settings import available_benchmarks
+from edgebenchmark.ncnn_benchmark_20210124 import NcnnBenchmark_20210124
+
+
+def NcnnBenchmark(version: int):
+    """
+    """
+    assert version in settings._NCNN_VERSIONS
+
+    if version in ("20210124"):
+        b = NcnnBenchmark_20210124
+    else:
+        raise ValueError(f"Invalid Ncnn version number. Choose one of {settings._NCNN_VERSIONS}")
+
+    benchmark = b(version)
+    benchmark.benchmark_type = available_benchmarks.ncnn_basic
+    return benchmark

edgebenchmark/ncnn_benchmark_20210124.py

@@ -0,0 +1,114 @@
+# Copyright 2021 Bisonai Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+from edgebenchmark.edgebenchmark import EdgeBenchmark
+
+
+class NcnnBenchmark_20210124(EdgeBenchmark):
+    @staticmethod
+    def default():
+        return {
+            "loop_count": 4,
+            "num_threads": -1,
+            "powersave": 0,
+            "gpu_device": -1,
+            "cooling_down": 1,
+
+            "height": 227,
+            "width": 227,
+            "channels": 3,
+        }
+
+    @staticmethod
+    def parameters():
+        return {
+            "loop_count": int,
+            "num_threads": int,
+            "powersave": int,
+            "gpu_device": int,
+            "cooling_down": int,
+            "height": int,
+            "width": int,
+            "channels": int,
+        }
+
+    @property
+    def loop_count(self):
+        """
+        """
+        return self.get_parameter("loop_count")
+
+    @loop_count.setter
+    def loop_count(self, value: int):
+        if not isinstance(value, int):
+            raise ValueError("loop_count must be type of int")
+        self.set_parameter("loop_count", value)
+
+    @property
+    def num_threads(self):
+        """
+        -1=max_cpu_count
+        """
+        return self.get_parameter("num_threads")
+
+    @num_threads.setter
+    def num_threads(self, value: int):
+        if not isinstance(value, int):
+            raise ValueError("num_threads must be type of int")
+        self.set_parameter("num_threads", value)
+
+    @property
+    def powersave(self):
+        """
+        0=all cores
+        1=little cores only
+        2=big cores only
+        """
+        return self.get_parameter("powersave")
+
+    @powersave.setter
+    def powersave(self, value: int):
+        if not isinstance(value, int):
+            raise ValueError("powersave must be type of int")
+        self.set_parameter("powersave", value)
+
+    @property
+    def gpu_device(self):
+        """
+        -1=cpu-only
+        0=gpu0
+        1=gpu1
+        ...
+        """
+        return self.get_parameter("gpu_device")
+
+    @gpu_device.setter
+    def gpu_device(self, value: int):
+        if not isinstance(value, int):
+            raise ValueError("gpu_device must be type of int")
+        self.set_parameter("gpu_device", value)
+
+    @property
+    def cooling_down(self):
+        """
+        0=disable
+        1=enable
+        """
+        return self.get_parameter("cooling_down")
+
+    @cooling_down.setter
+    def cooling_down(self, value: int):
+        if not isinstance(value, int):
+            raise ValueError("cooling_down must be type of int")
+        self.set_parameter("cooling_down", value)

edgebenchmark/settings.py

@@ -28,6 +28,10 @@ class settings:
     _MODEL_ENDPOINT = f"{_WEB_SERVER_URL}/api/model"
     _DEVICE_ENDPOINT = f"{_WEB_SERVER_URL}/api/devices"
 
+    _NCNN_VERSIONS = [
+        "20210124",
+    ]
+
     _TFLITE_VERSIONS = [
         "1.14.0",
         "1.15.0",
@@ -53,4 +57,5 @@ class settings:
 class available_benchmarks(Enum):
     tflite_basic = 0
     tflite_profiling = 1
-    ncnn = 2
+    ncnn_basic = 2
+    ncnn_profiling = 3

edgebenchmark/tflite_benchmark.py

@@ -14,7 +14,6 @@
 # ==============================================================================
 from edgebenchmark.settings import settings
 from edgebenchmark.settings import available_benchmarks
-
 from edgebenchmark.tflite_benchmark_1_14_0 import TFLiteBenchmark_1_14_0
 from edgebenchmark.tflite_benchmark_1_15_0 import TFLiteBenchmark_1_15_0
 from edgebenchmark.tflite_benchmark_2_0_0 import TFLiteBenchmark_2_0_0

setup.py

@@ -5,7 +5,7 @@
 
 setuptools.setup(
     name="edgebenchmark",
-    version="0.0.8",
+    version="0.0.9",
     author="Bisonai",
     author_email="contact@bisonai.com",
     description="Tool to benchmark speed of machine learning models on real mobile devices",