move numpy serialization from examples to API level (#62)

examples/flask/client.py

@@ -45,8 +45,11 @@
 
 import requests
 
-from deepsparse.utils import generate_random_inputs
-from utils_flask import bytes_to_tensors, tensors_to_bytes
+from deepsparse.utils import (
+    arrays_to_bytes,
+    bytes_to_arrays,
+    generate_random_inputs,
+)
 
 
 def parse_args():
@@ -103,11 +106,11 @@ def main():
 
     start = time.time()
     # Encode inputs
-    data = tensors_to_bytes(inputs)
+    data = arrays_to_bytes(inputs)
     # Send data to server for inference
     response = requests.post(prediction_url, data=data)
     # Decode outputs
-    outputs = bytes_to_tensors(response.content)
+    outputs = bytes_to_arrays(response.content)
     end = time.time()
     elapsed_time = end - start
 

examples/flask/server.py

@@ -49,7 +49,7 @@
 from flask_cors import CORS
 
 from deepsparse import compile_model
-from utils_flask import bytes_to_tensors, tensors_to_bytes
+from deepsparse.utils import arrays_to_bytes, bytes_to_arrays
 
 
 def parse_args():
@@ -114,15 +114,15 @@ def create_model_inference_app(
     def predict():
         data = flask.request.get_data()
 
-        inputs = bytes_to_tensors(data)
+        inputs = bytes_to_arrays(data)
         print(f"Received {len(inputs)} inputs from client")
 
         print("Executing model")
         outputs, elapsed_time = engine.timed_run(inputs)
 
         print(f"Inference time took {elapsed_time * 1000.0:.4f} milliseconds")
         print(f"Produced {len(outputs)} output tensors")
-        return tensors_to_bytes(outputs)
+        return arrays_to_bytes(outputs)
 
     @app.route("/info", methods=["GET"])
     def info():

src/deepsparse/utils/data.py

@@ -20,11 +20,54 @@
 from deepsparse.utils.log import log_init
 
 
-__all__ = ["verify_outputs"]
+__all__ = [
+    "arrays_to_bytes",
+    "bytes_to_arrays",
+    "verify_outputs",
+]
 
 log = log_init(os.path.basename(__file__))
 
 
+def arrays_to_bytes(arrays: List[numpy.array]) -> bytearray:
+    """
+    :param arrays: List of numpy arrays to serialize as bytes
+    :return: bytearray representation of list of numpy arrays
+    """
+    to_return = bytearray()
+    for arr in arrays:
+        arr_dtype = bytearray(str(arr.dtype), "utf-8")
+        arr_shape = bytearray(",".join([str(a) for a in arr.shape]), "utf-8")
+        sep = bytearray("|", "utf-8")
+        arr_bytes = arr.ravel().tobytes()
+        to_return += arr_dtype + sep + arr_shape + sep + arr_bytes
+    return to_return
+
+
+def bytes_to_arrays(serialized_arr: bytearray) -> List[numpy.array]:
+    """
+    :param serialized_arr: bytearray representation of list of numpy arrays
+    :return: List of numpy arrays decoded from input
+    """
+    sep = "|".encode("utf-8")
+    arrays = []
+    i_start = 0
+    while i_start < len(serialized_arr) - 1:
+        i_0 = serialized_arr.find(sep, i_start)
+        i_1 = serialized_arr.find(sep, i_0 + 1)
+        arr_dtype = numpy.dtype(serialized_arr[i_start:i_0].decode("utf-8"))
+        arr_shape = tuple(
+            [int(a) for a in serialized_arr[i_0 + 1 : i_1].decode("utf-8").split(",")]
+        )
+        arr_num_bytes = numpy.prod(arr_shape) * arr_dtype.itemsize
+        arr_str = serialized_arr[i_1 + 1 : arr_num_bytes + (i_1 + 1)]
+        arr = numpy.frombuffer(arr_str, dtype=arr_dtype).reshape(arr_shape)
+        arrays.append(arr.copy())
+
+        i_start = i_1 + arr_num_bytes + 1
+    return arrays
+
+
 def verify_outputs(
     outputs: List[numpy.array],
     gt_outputs: List[numpy.array],

tests/utils/test_data.py

@@ -0,0 +1,44 @@
+# Copyright (c) 2021 - present / Neuralmagic, Inc. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import numpy
+
+import pytest
+from deepsparse.utils import arrays_to_bytes, bytes_to_arrays
+
+
+@pytest.mark.parametrize(
+    "arrays",
+    (
+        [
+            numpy.random.randint(255, size=(3, 244, 244), dtype=numpy.uint8)
+            for _ in range(10)
+        ],
+        [numpy.random.randn(3, 224, 224).astype(numpy.float32) for _ in range(10)],
+        [numpy.random.randn(i * 5, i * 20) for i in range(5)],
+        [numpy.random.randint(255, size=(3, 244, 244), dtype=numpy.uint8)],
+    ),
+)
+def test_arrays_bytes_conversion(arrays):
+    serialized_arrays = arrays_to_bytes(arrays)
+    assert isinstance(serialized_arrays, bytearray)
+
+    deserialized_arrays = bytes_to_arrays(serialized_arrays)
+    assert isinstance(deserialized_arrays, list)
+    assert len(deserialized_arrays) == len(arrays)
+
+    for array, deserialized_array in zip(arrays, deserialized_arrays):
+        assert isinstance(deserialized_array, numpy.ndarray)
+        assert array.shape == deserialized_array.shape
+        assert numpy.all(array == deserialized_array)