From bf026551a16d12d4b4e75949933d080f39a85eef Mon Sep 17 00:00:00 2001 From: Lester Solbakken Date: Mon, 22 Nov 2021 15:41:12 +0100 Subject: [PATCH 1/3] Only evaluate ONNX models once in stateless model eval --- .../cfg/application/stateless_eval/mul.onnx | 17 ++- .../cfg/application/stateless_eval/mul.py | 20 ++- .../container/ml/ModelsEvaluatorTest.java | 10 +- model-evaluation/abi-spec.json | 16 +++ .../models/evaluation/FunctionEvaluator.java | 10 +- .../ai/vespa/models/evaluation/Model.java | 18 ++- .../models/evaluation/ModelsEvaluator.java | 11 ++ .../evaluation/MultiFunctionEvaluator.java | 120 ++++++++++++++++++ .../ai/vespa/models/evaluation/OnnxModel.java | 4 + .../models/evaluation/OnnxEvaluatorTest.java | 13 ++ 10 files changed, 221 insertions(+), 18 deletions(-) create mode 100644 model-evaluation/src/main/java/ai/vespa/models/evaluation/MultiFunctionEvaluator.java diff --git a/config-model/src/test/cfg/application/stateless_eval/mul.onnx b/config-model/src/test/cfg/application/stateless_eval/mul.onnx index 087e2c3427f1..26411c96986c 100644 --- a/config-model/src/test/cfg/application/stateless_eval/mul.onnx +++ b/config-model/src/test/cfg/application/stateless_eval/mul.onnx @@ -1,7 +1,10 @@ -mul.py:f - +mul.py:Ÿ + input1 -input2output"MulmulZ +input2output1"Mul + +input1 +input2output2"AddmulZ input1  @@ -9,8 +12,12 @@ input2  -b -output +b +output1 + + +b +output2  B \ No newline at end of file diff --git a/config-model/src/test/cfg/application/stateless_eval/mul.py b/config-model/src/test/cfg/application/stateless_eval/mul.py index 9fcb8612af9a..6bbc4e232007 100755 --- a/config-model/src/test/cfg/application/stateless_eval/mul.py +++ b/config-model/src/test/cfg/application/stateless_eval/mul.py @@ -2,25 +2,31 @@ import onnx from onnx import helper, TensorProto -INPUT_1 = helper.make_tensor_value_info('input1', TensorProto.FLOAT, [1]) -INPUT_2 = helper.make_tensor_value_info('input2', TensorProto.FLOAT, [1]) -OUTPUT = helper.make_tensor_value_info('output', TensorProto.FLOAT, [1]) +INPUT1 = helper.make_tensor_value_info('input1', TensorProto.FLOAT, [1]) +INPUT2 = helper.make_tensor_value_info('input2', TensorProto.FLOAT, [1]) +OUTPUT1 = helper.make_tensor_value_info('output1', TensorProto.FLOAT, [1]) +OUTPUT2 = helper.make_tensor_value_info('output2', TensorProto.FLOAT, [1]) nodes = [ helper.make_node( 'Mul', ['input1', 'input2'], - ['output'], + ['output1'], + ), + helper.make_node( + 'Add', + ['input1', 'input2'], + ['output2'], ), ] graph_def = helper.make_graph( nodes, 'mul', [ - INPUT_1, - INPUT_2 + INPUT1, + INPUT2 ], - [OUTPUT], + [OUTPUT1, OUTPUT2], ) model_def = helper.make_model(graph_def, producer_name='mul.py', opset_imports=[onnx.OperatorSetIdProto(version=12)]) onnx.save(model_def, 'mul.onnx') diff --git a/config-model/src/test/java/com/yahoo/vespa/model/container/ml/ModelsEvaluatorTest.java b/config-model/src/test/java/com/yahoo/vespa/model/container/ml/ModelsEvaluatorTest.java index 5630d3cc186b..70c4cb942bc7 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/container/ml/ModelsEvaluatorTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/container/ml/ModelsEvaluatorTest.java @@ -3,9 +3,12 @@ import ai.vespa.models.evaluation.FunctionEvaluator; import ai.vespa.models.evaluation.ModelsEvaluator; +import ai.vespa.models.evaluation.MultiFunctionEvaluator; import com.yahoo.tensor.Tensor; import org.junit.Test; +import java.util.Map; + import static org.junit.Assert.assertEquals; /** @@ -21,12 +24,17 @@ public void testModelsEvaluatorTester() { assertEquals(3, modelsEvaluator.models().size()); // ONNX model evaluation - FunctionEvaluator mul = modelsEvaluator.evaluatorOf("mul"); + FunctionEvaluator mul = modelsEvaluator.evaluatorOf("mul", "output1"); Tensor input1 = Tensor.from("tensor(d0[1]):[2]"); Tensor input2 = Tensor.from("tensor(d0[1]):[3]"); Tensor output = mul.bind("input1", input1).bind("input2", input2).evaluate(); assertEquals(6.0, output.sum().asDouble(), 1e-9); + MultiFunctionEvaluator eval = modelsEvaluator.multiEvaluatorOf("mul"); + Map out = eval.bind("input1", input1).bind("input2", input2).evaluate(); + assertEquals(6.0, out.get("output1").sum().asDouble(), 1e-9); + assertEquals(5.0, out.get("output2").sum().asDouble(), 1e-9); + // LightGBM model evaluation FunctionEvaluator lgbm = modelsEvaluator.evaluatorOf("lightgbm_regression"); lgbm.bind("numerical_1", 0.1).bind("numerical_2", 0.2).bind("categorical_1", "a").bind("categorical_2", "i"); diff --git a/model-evaluation/abi-spec.json b/model-evaluation/abi-spec.json index 6728d5cd9b44..3f23e7456ad4 100644 --- a/model-evaluation/abi-spec.json +++ b/model-evaluation/abi-spec.json @@ -56,6 +56,7 @@ "public java.lang.String name()", "public java.util.List functions()", "public varargs ai.vespa.models.evaluation.FunctionEvaluator evaluatorOf(java.lang.String[])", + "public varargs ai.vespa.models.evaluation.MultiFunctionEvaluator multiEvaluatorOf(java.lang.String[])", "public java.lang.String toString()" ], "fields": [] @@ -72,10 +73,25 @@ "public void (java.util.Map)", "public java.util.Map models()", "public varargs ai.vespa.models.evaluation.FunctionEvaluator evaluatorOf(java.lang.String, java.lang.String[])", + "public varargs ai.vespa.models.evaluation.MultiFunctionEvaluator multiEvaluatorOf(java.lang.String, java.lang.String[])", "public ai.vespa.models.evaluation.Model requireModel(java.lang.String)" ], "fields": [] }, + "ai.vespa.models.evaluation.MultiFunctionEvaluator": { + "superClass": "java.lang.Object", + "interfaces": [], + "attributes": [ + "public" + ], + "methods": [ + "public ai.vespa.models.evaluation.MultiFunctionEvaluator bind(java.lang.String, com.yahoo.tensor.Tensor)", + "public ai.vespa.models.evaluation.MultiFunctionEvaluator bind(java.lang.String, double)", + "public java.util.Map evaluate()", + "public java.util.List functions()" + ], + "fields": [] + }, "ai.vespa.models.evaluation.RankProfilesConfigImporter": { "superClass": "java.lang.Object", "interfaces": [], diff --git a/model-evaluation/src/main/java/ai/vespa/models/evaluation/FunctionEvaluator.java b/model-evaluation/src/main/java/ai/vespa/models/evaluation/FunctionEvaluator.java index 6af33e29e62c..aa13cb968451 100644 --- a/model-evaluation/src/main/java/ai/vespa/models/evaluation/FunctionEvaluator.java +++ b/model-evaluation/src/main/java/ai/vespa/models/evaluation/FunctionEvaluator.java @@ -101,14 +101,18 @@ public FunctionEvaluator setMissingValue(double value) { } public Tensor evaluate() { - for (Map.Entry argument : function.argumentTypes().entrySet()) { - checkArgument(argument.getKey(), argument.getValue()); - } + checkArguments(); evaluated = true; evaluateOnnxModels(); return function.getBody().evaluate(context).asTensor(); } + void checkArguments() { + for (Map.Entry argument : function.argumentTypes().entrySet()) { + checkArgument(argument.getKey(), argument.getValue()); + } + } + private void checkArgument(String name, TensorType type) { if (context.isMissing(name)) throw new IllegalStateException("Missing argument '" + name + "': Must be bound to a value of type " + type); diff --git a/model-evaluation/src/main/java/ai/vespa/models/evaluation/Model.java b/model-evaluation/src/main/java/ai/vespa/models/evaluation/Model.java index 8af5f7bc4996..84ab6e818407 100644 --- a/model-evaluation/src/main/java/ai/vespa/models/evaluation/Model.java +++ b/model-evaluation/src/main/java/ai/vespa/models/evaluation/Model.java @@ -6,9 +6,7 @@ import com.google.common.collect.ImmutableMap; import com.yahoo.searchlib.rankingexpression.ExpressionFunction; import com.yahoo.searchlib.rankingexpression.evaluation.ContextIndex; -import com.yahoo.searchlib.rankingexpression.evaluation.DoubleValue; import com.yahoo.searchlib.rankingexpression.evaluation.ExpressionOptimizer; -import com.yahoo.searchlib.rankingexpression.evaluation.Value; import com.yahoo.tensor.TensorType; import java.util.Arrays; @@ -232,6 +230,22 @@ private FunctionEvaluator evaluatorOf(ExpressionFunction function) { return new FunctionEvaluator(function, requireContextPrototype(function.getName()).copy()); } + /** + * Returns an evaluator which can be used to evaluate the given model in a single thread once. + * + * @param names The names identifying the outputs. If none are passed, evaluates all outputs. + * @throws IllegalArgumentException if the function is not present. + */ + public MultiFunctionEvaluator multiEvaluatorOf(String ... names) { + List evaluators; + if (names.length == 0) { + evaluators = functions.stream().map(this::evaluatorOf).collect(Collectors.toList()); + } else { + evaluators = Arrays.stream(names).map(this::evaluatorOf).collect(Collectors.toList()); + } + return new MultiFunctionEvaluator(evaluators); + } + private void throwUndeterminedFunction(String message) { throw new IllegalArgumentException(message + ". Available functions: " + functions.stream().map(f -> f.getName()).collect(Collectors.joining(", "))); diff --git a/model-evaluation/src/main/java/ai/vespa/models/evaluation/ModelsEvaluator.java b/model-evaluation/src/main/java/ai/vespa/models/evaluation/ModelsEvaluator.java index 01427ca811af..bd00f5510c6c 100644 --- a/model-evaluation/src/main/java/ai/vespa/models/evaluation/ModelsEvaluator.java +++ b/model-evaluation/src/main/java/ai/vespa/models/evaluation/ModelsEvaluator.java @@ -60,6 +60,17 @@ public FunctionEvaluator evaluatorOf(String modelName, String ... names) { return requireModel(modelName).evaluatorOf(names); } + /** + * Returns a model evaluator which can be used to evaluate multiple functions in a model + * + * @param modelName the name of the model + * @param names the names of the outputs to evaluate, or none if all should be evaluated + * @throws IllegalArgumentException if the function or model is not present + */ + public MultiFunctionEvaluator multiEvaluatorOf(String modelName, String ... names) { + return requireModel(modelName).multiEvaluatorOf(names); + } + /** Returns the given model, or throws a IllegalArgumentException if it does not exist */ public Model requireModel(String name) { Model model = models.get(name); diff --git a/model-evaluation/src/main/java/ai/vespa/models/evaluation/MultiFunctionEvaluator.java b/model-evaluation/src/main/java/ai/vespa/models/evaluation/MultiFunctionEvaluator.java new file mode 100644 index 000000000000..53d470ecc194 --- /dev/null +++ b/model-evaluation/src/main/java/ai/vespa/models/evaluation/MultiFunctionEvaluator.java @@ -0,0 +1,120 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package ai.vespa.models.evaluation; + +import com.yahoo.searchlib.rankingexpression.evaluation.TensorValue; +import com.yahoo.tensor.Tensor; +import com.yahoo.tensor.TensorType; + +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; + +/** + * An evaluator which can be used to evaluate a model with multiple outputs. + * This will ensure that ONNX models are only evaluated once. + * + * @author lesters + */ +public class MultiFunctionEvaluator { + + private final List functions; + private boolean evaluated = false; + + MultiFunctionEvaluator(List functions) { + this.functions = functions; + } + + /** + * Binds the given variable referred in this expression to the given value. + * + * @param name the variable to bind + * @param value the value this becomes bound to + * @return this for chaining + */ + public MultiFunctionEvaluator bind(String name, Tensor value) { + if (evaluated) + throw new IllegalStateException("Cannot bind a new value in a used evaluator"); + for (FunctionEvaluator function : functions) { + if (function.function().argumentTypes().containsKey(name)) { + function.bind(name, value); // only bind input to the functions that need them + } + } + return this; + } + + /** + * Binds the given variable referred in this expression to the given value. + * This is equivalent to bind(name, Tensor.Builder.of(TensorType.empty).cell(value).build()) + * + * @param name the variable to bind + * @param value the value this becomes bound to + * @return this for chaining + */ + public MultiFunctionEvaluator bind(String name, double value) { + return bind(name, Tensor.Builder.of(TensorType.empty).cell(value).build()); + } + + public Map evaluate() { + for (FunctionEvaluator function : functions) { + function.checkArguments(); + } + + evaluateOnnxModels(); // evaluate each ONNX model only once + + Map results = new HashMap<>(); + for (FunctionEvaluator function : functions) { + results.put(function.function().getName(), function.evaluate()); + } + evaluated = true; + return results; + } + + /** + * Evaluate all ONNX models across all functions once and add the result + * back to the functions' context. + */ + private void evaluateOnnxModels() { + Set onnxModels = new HashSet<>(); + for (FunctionEvaluator function : functions) { + onnxModels.addAll(function.context().onnxModels().values()); + } + + for (OnnxModel onnxModel : onnxModels) { + + // Gather inputs from all functions. Inputs with the same name must have the same value. + Map inputs = new HashMap<>(); + for (FunctionEvaluator function : functions) { + for (OnnxModel functionModel : function.context().onnxModels().values()) { + if (functionModel.name().equals(onnxModel.name())) { + for (String inputName: onnxModel.inputs().keySet()) { + inputs.put(inputName, function.context().get(inputName).asTensor()); + } + } + } + } + + // Evaluate model once. + Map outputs = onnxModel.evaluate(inputs); + + // Add outputs back to the context of the functions that need them; they won't be recalculated. + for (FunctionEvaluator function : functions) { + for (Map.Entry entry : function.context().onnxModels().entrySet()) { + String onnxFeature = entry.getKey(); + OnnxModel functionModel = entry.getValue(); + if (functionModel.name().equals(onnxModel.name())) { + Tensor result = outputs.get(function.function().getName()); // Function name is output of model + function.context().put(onnxFeature, new TensorValue(result)); + } + } + } + + } + } + + public List functions() { + return functions; + } + +} diff --git a/model-evaluation/src/main/java/ai/vespa/models/evaluation/OnnxModel.java b/model-evaluation/src/main/java/ai/vespa/models/evaluation/OnnxModel.java index 19a9a1dccd5f..06045b07f7c9 100644 --- a/model-evaluation/src/main/java/ai/vespa/models/evaluation/OnnxModel.java +++ b/model-evaluation/src/main/java/ai/vespa/models/evaluation/OnnxModel.java @@ -50,6 +50,10 @@ public Tensor evaluate(Map inputs, String output) { return evaluator().evaluate(inputs, output); } + public Map evaluate(Map inputs) { + return evaluator().evaluate(inputs); + } + private OnnxEvaluator evaluator() { if (evaluator == null) { throw new IllegalStateException("ONNX model has not been loaded."); diff --git a/model-evaluation/src/test/java/ai/vespa/models/evaluation/OnnxEvaluatorTest.java b/model-evaluation/src/test/java/ai/vespa/models/evaluation/OnnxEvaluatorTest.java index a15c35fe8548..59ab378e43a1 100644 --- a/model-evaluation/src/test/java/ai/vespa/models/evaluation/OnnxEvaluatorTest.java +++ b/model-evaluation/src/test/java/ai/vespa/models/evaluation/OnnxEvaluatorTest.java @@ -18,6 +18,7 @@ import java.util.Map; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; /** @@ -44,6 +45,18 @@ public void testOnnxEvaluation() { function.bind("input2", Tensor.from("tensor(d0[1]):[3]")); assertEquals(5.0, function.evaluate().sum().asDouble(), delta); + MultiFunctionEvaluator evaluator = models.multiEvaluatorOf("add_mul"); + Tensor input1 = Tensor.from("tensor(d0[1]):[2]"); + Tensor input2 = Tensor.from("tensor(d0[1]):[3]"); + Map result = evaluator.bind("input1", input1).bind("input2", input2).evaluate(); + assertEquals(6.0, result.get("output1").sum().asDouble(), delta); + assertEquals(5.0, result.get("output2").sum().asDouble(), delta); + + evaluator = models.multiEvaluatorOf("add_mul", "output1"); + result = evaluator.bind("input1", input1).bind("input2", input2).evaluate(); + assertTrue("Result does not contain requested output", result.containsKey("output1")); + assertFalse("Result contains output that was not requested", result.containsKey("output2")); + function = models.evaluatorOf("one_layer"); function.bind("input", Tensor.from("tensor(d0[2],d1[3]):[[0.1, 0.2, 0.3],[0.4,0.5,0.6]]")); assertEquals(function.evaluate(), Tensor.from("tensor(d0[2],d1[1]):[0.63931,0.67574]")); From a40eea2df1d64d8586768f1122da90a5756bef10 Mon Sep 17 00:00:00 2001 From: Lester Solbakken Date: Wed, 24 Nov 2021 12:47:23 +0100 Subject: [PATCH 2/3] Remove MultiFunctionEvaluator --- .../container/ml/ModelsEvaluatorTest.java | 12 +- model-evaluation/abi-spec.json | 20 +-- .../models/evaluation/FunctionEvaluator.java | 126 +++++++++++++----- .../models/evaluation/LazyArrayContext.java | 2 +- .../ai/vespa/models/evaluation/Model.java | 24 ++-- .../models/evaluation/ModelsEvaluator.java | 11 -- .../evaluation/MultiFunctionEvaluator.java | 120 ----------------- .../evaluation/ModelsEvaluatorTest.java | 4 +- .../models/evaluation/OnnxEvaluatorTest.java | 31 ++--- 9 files changed, 125 insertions(+), 225 deletions(-) delete mode 100644 model-evaluation/src/main/java/ai/vespa/models/evaluation/MultiFunctionEvaluator.java diff --git a/config-model/src/test/java/com/yahoo/vespa/model/container/ml/ModelsEvaluatorTest.java b/config-model/src/test/java/com/yahoo/vespa/model/container/ml/ModelsEvaluatorTest.java index 70c4cb942bc7..8ed229b2ff57 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/container/ml/ModelsEvaluatorTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/container/ml/ModelsEvaluatorTest.java @@ -3,12 +3,9 @@ import ai.vespa.models.evaluation.FunctionEvaluator; import ai.vespa.models.evaluation.ModelsEvaluator; -import ai.vespa.models.evaluation.MultiFunctionEvaluator; import com.yahoo.tensor.Tensor; import org.junit.Test; -import java.util.Map; - import static org.junit.Assert.assertEquals; /** @@ -30,10 +27,11 @@ public void testModelsEvaluatorTester() { Tensor output = mul.bind("input1", input1).bind("input2", input2).evaluate(); assertEquals(6.0, output.sum().asDouble(), 1e-9); - MultiFunctionEvaluator eval = modelsEvaluator.multiEvaluatorOf("mul"); - Map out = eval.bind("input1", input1).bind("input2", input2).evaluate(); - assertEquals(6.0, out.get("output1").sum().asDouble(), 1e-9); - assertEquals(5.0, out.get("output2").sum().asDouble(), 1e-9); + FunctionEvaluator eval = modelsEvaluator.evaluatorOf("mul"); + output = eval.bind("input1", input1).bind("input2", input2).evaluate(); + assertEquals(6.0, output.sum().asDouble(), 1e-9); + assertEquals(6.0, eval.result("output1").sum().asDouble(), 1e-9); + assertEquals(5.0, eval.result("output2").sum().asDouble(), 1e-9); // LightGBM model evaluation FunctionEvaluator lgbm = modelsEvaluator.evaluatorOf("lightgbm_regression"); diff --git a/model-evaluation/abi-spec.json b/model-evaluation/abi-spec.json index 3f23e7456ad4..71dd7ffc2ebb 100644 --- a/model-evaluation/abi-spec.json +++ b/model-evaluation/abi-spec.json @@ -6,6 +6,7 @@ "public" ], "methods": [ + "public com.yahoo.tensor.Tensor result(java.lang.String)", "public ai.vespa.models.evaluation.FunctionEvaluator bind(java.lang.String, com.yahoo.tensor.Tensor)", "public ai.vespa.models.evaluation.FunctionEvaluator bind(java.lang.String, double)", "public ai.vespa.models.evaluation.FunctionEvaluator bind(java.lang.String, java.lang.String)", @@ -13,7 +14,8 @@ "public ai.vespa.models.evaluation.FunctionEvaluator setMissingValue(double)", "public com.yahoo.tensor.Tensor evaluate()", "public com.yahoo.searchlib.rankingexpression.ExpressionFunction function()", - "public ai.vespa.models.evaluation.LazyArrayContext context()" + "public ai.vespa.models.evaluation.LazyArrayContext context()", + "public java.util.List outputs()" ], "fields": [] }, @@ -56,7 +58,6 @@ "public java.lang.String name()", "public java.util.List functions()", "public varargs ai.vespa.models.evaluation.FunctionEvaluator evaluatorOf(java.lang.String[])", - "public varargs ai.vespa.models.evaluation.MultiFunctionEvaluator multiEvaluatorOf(java.lang.String[])", "public java.lang.String toString()" ], "fields": [] @@ -73,25 +74,10 @@ "public void (java.util.Map)", "public java.util.Map models()", "public varargs ai.vespa.models.evaluation.FunctionEvaluator evaluatorOf(java.lang.String, java.lang.String[])", - "public varargs ai.vespa.models.evaluation.MultiFunctionEvaluator multiEvaluatorOf(java.lang.String, java.lang.String[])", "public ai.vespa.models.evaluation.Model requireModel(java.lang.String)" ], "fields": [] }, - "ai.vespa.models.evaluation.MultiFunctionEvaluator": { - "superClass": "java.lang.Object", - "interfaces": [], - "attributes": [ - "public" - ], - "methods": [ - "public ai.vespa.models.evaluation.MultiFunctionEvaluator bind(java.lang.String, com.yahoo.tensor.Tensor)", - "public ai.vespa.models.evaluation.MultiFunctionEvaluator bind(java.lang.String, double)", - "public java.util.Map evaluate()", - "public java.util.List functions()" - ], - "fields": [] - }, "ai.vespa.models.evaluation.RankProfilesConfigImporter": { "superClass": "java.lang.Object", "interfaces": [], diff --git a/model-evaluation/src/main/java/ai/vespa/models/evaluation/FunctionEvaluator.java b/model-evaluation/src/main/java/ai/vespa/models/evaluation/FunctionEvaluator.java index aa13cb968451..7a992cb7aa90 100644 --- a/model-evaluation/src/main/java/ai/vespa/models/evaluation/FunctionEvaluator.java +++ b/model-evaluation/src/main/java/ai/vespa/models/evaluation/FunctionEvaluator.java @@ -8,24 +8,37 @@ import com.yahoo.tensor.TensorType; import java.util.HashMap; +import java.util.HashSet; +import java.util.List; import java.util.Map; +import java.util.Set; import java.util.stream.Collectors; /** - * An evaluator which can be used to evaluate a single function once. + * An evaluator which can be used to evaluate a function once. * * @author bratseth */ // This wraps all access to the context and the ranking expression to avoid incorrect usage public class FunctionEvaluator { - private final ExpressionFunction function; - private final LazyArrayContext context; + private final List functions; + private final Map contexts; + private final Map results; private boolean evaluated = false; FunctionEvaluator(ExpressionFunction function, LazyArrayContext context) { - this.function = function; - this.context = context; + this(List.of(function), Map.of(function.getName(), context)); + } + + FunctionEvaluator(List functions, Map contexts) { + this.functions = List.copyOf(functions); + this.contexts = Map.copyOf(contexts); + this.results = new HashMap<>(); + } + + public Tensor result(String name) { + return results.get(name); } /** @@ -38,15 +51,14 @@ public class FunctionEvaluator { public FunctionEvaluator bind(String name, Tensor value) { if (evaluated) throw new IllegalStateException("Cannot bind a new value in a used evaluator"); - TensorType requiredType = function.argumentTypes().get(name); - if (requiredType == null) - throw new IllegalArgumentException("'" + name + "' is not a valid argument in " + function + - ". Expected arguments: " + function.argumentTypes().entrySet().stream() - .map(e -> e.getKey() + ": " + e.getValue()) - .collect(Collectors.joining(", "))); - if ( ! value.type().isAssignableTo(requiredType)) - throw new IllegalArgumentException("'" + name + "' must be of type " + requiredType + ", not " + value.type()); - context.put(name, new TensorValue(value)); + for (ExpressionFunction function : functions) { + if (function.argumentTypes().containsKey(name)) { + TensorType requiredType = function.argumentTypes().get(name); + if ( ! value.type().isAssignableTo(requiredType)) + throw new IllegalArgumentException("'" + name + "' must be of type " + requiredType + ", not " + value.type()); + contexts.get(function.getName()).put(name, new TensorValue(value)); + } + } return this; } @@ -73,7 +85,11 @@ public FunctionEvaluator bind(String name, double value) { public FunctionEvaluator bind(String name, String value) { if (evaluated) throw new IllegalStateException("Cannot bind a new value in a used evaluator"); - context.put(name, new StringValue(value)); + for (ExpressionFunction function : functions) { + if (function.argumentTypes().containsKey(name)) { + contexts.get(function.getName()).put(name, new StringValue(value)); + } + } return this; } @@ -86,7 +102,9 @@ public FunctionEvaluator bind(String name, String value) { public FunctionEvaluator setMissingValue(Tensor value) { if (evaluated) throw new IllegalStateException("Cannot change the missing value in a used evaluator"); - context.setMissingValue(value); + for (LazyArrayContext context : contexts.values()) { + context.setMissingValue(value); + } return this; } @@ -102,18 +120,31 @@ public FunctionEvaluator setMissingValue(double value) { public Tensor evaluate() { checkArguments(); - evaluated = true; evaluateOnnxModels(); - return function.getBody().evaluate(context).asTensor(); + + Tensor defaultResult = null; + for (ExpressionFunction function: functions) { + LazyArrayContext context = contexts.get(function.getName()); + Tensor result = function.getBody().evaluate(context).asTensor(); + results.put(function.getName(), function.getBody().evaluate(context).asTensor()); + if (defaultResult == null) { + defaultResult = result; + } + } + evaluated = true; + return defaultResult; } void checkArguments() { - for (Map.Entry argument : function.argumentTypes().entrySet()) { - checkArgument(argument.getKey(), argument.getValue()); + for (ExpressionFunction function : functions) { + LazyArrayContext context = contexts.get(function.getName()); + for (Map.Entry argument : function.argumentTypes().entrySet()) { + checkArgument(argument.getKey(), argument.getValue(), context); + } } } - private void checkArgument(String name, TensorType type) { + private void checkArgument(String name, TensorType type, LazyArrayContext context) { if (context.isMissing(name)) throw new IllegalStateException("Missing argument '" + name + "': Must be bound to a value of type " + type); if (! context.get(name).type().isAssignableTo(type)) @@ -124,23 +155,52 @@ private void checkArgument(String name, TensorType type) { * Evaluate ONNX models (if not already evaluated) and add the result back to the context. */ private void evaluateOnnxModels() { - for (Map.Entry entry : context().onnxModels().entrySet()) { - String onnxFeature = entry.getKey(); - OnnxModel onnxModel = entry.getValue(); - if (context.get(onnxFeature).equals(context.defaultValue())) { - Map inputs = new HashMap<>(); - for (Map.Entry input: onnxModel.inputs().entrySet()) { - inputs.put(input.getKey(), context.get(input.getKey()).asTensor()); + Set onnxModels = new HashSet<>(); + for (LazyArrayContext context : contexts.values()) { + onnxModels.addAll(context.onnxModels().values()); + } + + for (OnnxModel onnxModel : onnxModels) { + + // Gather inputs from all functions. Inputs with the same name must have the same value. + Map inputs = new HashMap<>(); + for (LazyArrayContext context : contexts.values()) { + for (OnnxModel functionModel : context.onnxModels().values()) { + if (functionModel.name().equals(onnxModel.name())) { + for (String inputName: onnxModel.inputs().keySet()) { + inputs.put(inputName, context.get(inputName).asTensor()); + } + } } - Tensor result = onnxModel.evaluate(inputs, function.getName()); // Function name is output of model - context.put(onnxFeature, new TensorValue(result)); } + + // Evaluate model once. + Map outputs = onnxModel.evaluate(inputs); + + // Add outputs back to the context of the functions that need them; they won't be recalculated. + for (ExpressionFunction function : functions) { + LazyArrayContext context = contexts.get(function.getName()); + for (Map.Entry entry : context.onnxModels().entrySet()) { + String onnxFeature = entry.getKey(); + OnnxModel functionModel = entry.getValue(); + if (functionModel.name().equals(onnxModel.name())) { + Tensor result = outputs.get(function.getName()); // Function name is output of model + context.put(onnxFeature, new TensorValue(result)); + } + } + } + } } - /** Returns the function evaluated by this */ - public ExpressionFunction function() { return function; } + /** Returns the default function evaluated by this */ + public ExpressionFunction function() { return functions.get(0); } - public LazyArrayContext context() { return context; } + public LazyArrayContext context() { return contexts.get(function().getName()); } + + /** Returns the names of the outputs of this function */ + public List outputs() { + return functions.stream().map(ExpressionFunction::getName).collect(Collectors.toList()); + } } diff --git a/model-evaluation/src/main/java/ai/vespa/models/evaluation/LazyArrayContext.java b/model-evaluation/src/main/java/ai/vespa/models/evaluation/LazyArrayContext.java index d97235d11d25..cc53f38f800a 100644 --- a/model-evaluation/src/main/java/ai/vespa/models/evaluation/LazyArrayContext.java +++ b/model-evaluation/src/main/java/ai/vespa/models/evaluation/LazyArrayContext.java @@ -153,7 +153,7 @@ private static class IndexedBindings { /** The mapping from variable name to index */ private final ImmutableMap nameToIndex; - /** The names which needs to be bound externally when invoking this (i.e not constant or invocation */ + /** The names which needs to be bound externally when invoking this (i.e. not constant or invocation) */ private final ImmutableSet arguments; /** The current values set */ diff --git a/model-evaluation/src/main/java/ai/vespa/models/evaluation/Model.java b/model-evaluation/src/main/java/ai/vespa/models/evaluation/Model.java index 84ab6e818407..ab24986e542b 100644 --- a/model-evaluation/src/main/java/ai/vespa/models/evaluation/Model.java +++ b/model-evaluation/src/main/java/ai/vespa/models/evaluation/Model.java @@ -12,6 +12,7 @@ import java.util.Arrays; import java.util.Collection; import java.util.Collections; +import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.stream.Collectors; @@ -181,9 +182,7 @@ ExpressionFunction requireReferencedFunction(FunctionReference reference) { */ public FunctionEvaluator evaluatorOf(String ... names) { // TODO: Parameter overloading? if (names.length == 0) { - if (functions.size() > 1) - throwUndeterminedFunction("More than one function is available in " + this + ", but no name is given"); - return evaluatorOf(functions.get(0)); + return evaluatorOf(functions); } else if (names.length == 1) { String name = names[0]; @@ -230,20 +229,13 @@ private FunctionEvaluator evaluatorOf(ExpressionFunction function) { return new FunctionEvaluator(function, requireContextPrototype(function.getName()).copy()); } - /** - * Returns an evaluator which can be used to evaluate the given model in a single thread once. - * - * @param names The names identifying the outputs. If none are passed, evaluates all outputs. - * @throws IllegalArgumentException if the function is not present. - */ - public MultiFunctionEvaluator multiEvaluatorOf(String ... names) { - List evaluators; - if (names.length == 0) { - evaluators = functions.stream().map(this::evaluatorOf).collect(Collectors.toList()); - } else { - evaluators = Arrays.stream(names).map(this::evaluatorOf).collect(Collectors.toList()); + /** Returns a single-use evaluator of a function */ + private FunctionEvaluator evaluatorOf(List functions) { + Map contexts = new HashMap<>(); + for (ExpressionFunction function : functions) { + contexts.put(function.getName(), requireContextPrototype(function.getName()).copy()); } - return new MultiFunctionEvaluator(evaluators); + return new FunctionEvaluator(functions, contexts); } private void throwUndeterminedFunction(String message) { diff --git a/model-evaluation/src/main/java/ai/vespa/models/evaluation/ModelsEvaluator.java b/model-evaluation/src/main/java/ai/vespa/models/evaluation/ModelsEvaluator.java index bd00f5510c6c..01427ca811af 100644 --- a/model-evaluation/src/main/java/ai/vespa/models/evaluation/ModelsEvaluator.java +++ b/model-evaluation/src/main/java/ai/vespa/models/evaluation/ModelsEvaluator.java @@ -60,17 +60,6 @@ public FunctionEvaluator evaluatorOf(String modelName, String ... names) { return requireModel(modelName).evaluatorOf(names); } - /** - * Returns a model evaluator which can be used to evaluate multiple functions in a model - * - * @param modelName the name of the model - * @param names the names of the outputs to evaluate, or none if all should be evaluated - * @throws IllegalArgumentException if the function or model is not present - */ - public MultiFunctionEvaluator multiEvaluatorOf(String modelName, String ... names) { - return requireModel(modelName).multiEvaluatorOf(names); - } - /** Returns the given model, or throws a IllegalArgumentException if it does not exist */ public Model requireModel(String name) { Model model = models.get(name); diff --git a/model-evaluation/src/main/java/ai/vespa/models/evaluation/MultiFunctionEvaluator.java b/model-evaluation/src/main/java/ai/vespa/models/evaluation/MultiFunctionEvaluator.java deleted file mode 100644 index 53d470ecc194..000000000000 --- a/model-evaluation/src/main/java/ai/vespa/models/evaluation/MultiFunctionEvaluator.java +++ /dev/null @@ -1,120 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package ai.vespa.models.evaluation; - -import com.yahoo.searchlib.rankingexpression.evaluation.TensorValue; -import com.yahoo.tensor.Tensor; -import com.yahoo.tensor.TensorType; - -import java.util.HashMap; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Set; - -/** - * An evaluator which can be used to evaluate a model with multiple outputs. - * This will ensure that ONNX models are only evaluated once. - * - * @author lesters - */ -public class MultiFunctionEvaluator { - - private final List functions; - private boolean evaluated = false; - - MultiFunctionEvaluator(List functions) { - this.functions = functions; - } - - /** - * Binds the given variable referred in this expression to the given value. - * - * @param name the variable to bind - * @param value the value this becomes bound to - * @return this for chaining - */ - public MultiFunctionEvaluator bind(String name, Tensor value) { - if (evaluated) - throw new IllegalStateException("Cannot bind a new value in a used evaluator"); - for (FunctionEvaluator function : functions) { - if (function.function().argumentTypes().containsKey(name)) { - function.bind(name, value); // only bind input to the functions that need them - } - } - return this; - } - - /** - * Binds the given variable referred in this expression to the given value. - * This is equivalent to bind(name, Tensor.Builder.of(TensorType.empty).cell(value).build()) - * - * @param name the variable to bind - * @param value the value this becomes bound to - * @return this for chaining - */ - public MultiFunctionEvaluator bind(String name, double value) { - return bind(name, Tensor.Builder.of(TensorType.empty).cell(value).build()); - } - - public Map evaluate() { - for (FunctionEvaluator function : functions) { - function.checkArguments(); - } - - evaluateOnnxModels(); // evaluate each ONNX model only once - - Map results = new HashMap<>(); - for (FunctionEvaluator function : functions) { - results.put(function.function().getName(), function.evaluate()); - } - evaluated = true; - return results; - } - - /** - * Evaluate all ONNX models across all functions once and add the result - * back to the functions' context. - */ - private void evaluateOnnxModels() { - Set onnxModels = new HashSet<>(); - for (FunctionEvaluator function : functions) { - onnxModels.addAll(function.context().onnxModels().values()); - } - - for (OnnxModel onnxModel : onnxModels) { - - // Gather inputs from all functions. Inputs with the same name must have the same value. - Map inputs = new HashMap<>(); - for (FunctionEvaluator function : functions) { - for (OnnxModel functionModel : function.context().onnxModels().values()) { - if (functionModel.name().equals(onnxModel.name())) { - for (String inputName: onnxModel.inputs().keySet()) { - inputs.put(inputName, function.context().get(inputName).asTensor()); - } - } - } - } - - // Evaluate model once. - Map outputs = onnxModel.evaluate(inputs); - - // Add outputs back to the context of the functions that need them; they won't be recalculated. - for (FunctionEvaluator function : functions) { - for (Map.Entry entry : function.context().onnxModels().entrySet()) { - String onnxFeature = entry.getKey(); - OnnxModel functionModel = entry.getValue(); - if (functionModel.name().equals(onnxModel.name())) { - Tensor result = outputs.get(function.function().getName()); // Function name is output of model - function.context().put(onnxFeature, new TensorValue(result)); - } - } - } - - } - } - - public List functions() { - return functions; - } - -} diff --git a/model-evaluation/src/test/java/ai/vespa/models/evaluation/ModelsEvaluatorTest.java b/model-evaluation/src/test/java/ai/vespa/models/evaluation/ModelsEvaluatorTest.java index 4cb52216137a..3e065d25ad20 100644 --- a/model-evaluation/src/test/java/ai/vespa/models/evaluation/ModelsEvaluatorTest.java +++ b/model-evaluation/src/test/java/ai/vespa/models/evaluation/ModelsEvaluatorTest.java @@ -95,8 +95,8 @@ public void testBindingValidation() { evaluator.bind("argNone", Tensor.from(TensorType.fromSpec("tensor(d1{})"), "{{d1:foo}:0.1}")); evaluator.evaluate(); } - catch (IllegalArgumentException e) { - assertEquals("'argNone' is not a valid argument in function 'test'. Expected arguments: arg2: tensor(d1{}), arg1: tensor(d0[1])", + catch (IllegalStateException e) { + assertEquals("Argument 'arg2' must be bound to a value of type tensor(d1{})", Exceptions.toMessageString(e)); } diff --git a/model-evaluation/src/test/java/ai/vespa/models/evaluation/OnnxEvaluatorTest.java b/model-evaluation/src/test/java/ai/vespa/models/evaluation/OnnxEvaluatorTest.java index 59ab378e43a1..ae77af264a1d 100644 --- a/model-evaluation/src/test/java/ai/vespa/models/evaluation/OnnxEvaluatorTest.java +++ b/model-evaluation/src/test/java/ai/vespa/models/evaluation/OnnxEvaluatorTest.java @@ -35,27 +35,22 @@ public void testOnnxEvaluation() { assertTrue(models.models().containsKey("add_mul")); assertTrue(models.models().containsKey("one_layer")); + Tensor input1 = Tensor.from("tensor(d0[1]):[2]"); + Tensor input2 = Tensor.from("tensor(d0[1]):[3]"); + FunctionEvaluator function = models.evaluatorOf("add_mul", "output1"); - function.bind("input1", Tensor.from("tensor(d0[1]):[2]")); - function.bind("input2", Tensor.from("tensor(d0[1]):[3]")); - assertEquals(6.0, function.evaluate().sum().asDouble(), delta); + Tensor result = function.bind("input1", input1).bind("input2", input2).evaluate(); + assertEquals(6.0, result.sum().asDouble(), delta); function = models.evaluatorOf("add_mul", "output2"); - function.bind("input1", Tensor.from("tensor(d0[1]):[2]")); - function.bind("input2", Tensor.from("tensor(d0[1]):[3]")); - assertEquals(5.0, function.evaluate().sum().asDouble(), delta); - - MultiFunctionEvaluator evaluator = models.multiEvaluatorOf("add_mul"); - Tensor input1 = Tensor.from("tensor(d0[1]):[2]"); - Tensor input2 = Tensor.from("tensor(d0[1]):[3]"); - Map result = evaluator.bind("input1", input1).bind("input2", input2).evaluate(); - assertEquals(6.0, result.get("output1").sum().asDouble(), delta); - assertEquals(5.0, result.get("output2").sum().asDouble(), delta); - - evaluator = models.multiEvaluatorOf("add_mul", "output1"); - result = evaluator.bind("input1", input1).bind("input2", input2).evaluate(); - assertTrue("Result does not contain requested output", result.containsKey("output1")); - assertFalse("Result contains output that was not requested", result.containsKey("output2")); + result = function.bind("input1", input1).bind("input2", input2).evaluate(); + assertEquals(5.0, result.sum().asDouble(), delta); + + function = models.evaluatorOf("add_mul"); // contains two models + result = function.bind("input1", input1).bind("input2", input2).evaluate(); + assertEquals(6.0, result.sum().asDouble(), delta); + assertEquals(6.0, function.result("output1").sum().asDouble(), delta); + assertEquals(5.0, function.result("output2").sum().asDouble(), delta); function = models.evaluatorOf("one_layer"); function.bind("input", Tensor.from("tensor(d0[2],d1[3]):[[0.1, 0.2, 0.3],[0.4,0.5,0.6]]")); From 0fded3444d2214b3280f7119a64395065bd30593 Mon Sep 17 00:00:00 2001 From: Lester Solbakken Date: Wed, 24 Nov 2021 12:48:24 +0100 Subject: [PATCH 3/3] Update stateless eval REST API for changes in multiple output functions --- .../handler/ModelsEvaluationHandler.java | 43 +++++++++++++++---- .../handler/ModelsEvaluationHandlerTest.java | 7 --- .../handler/OnnxEvaluationHandlerTest.java | 17 +++++++- 3 files changed, 49 insertions(+), 18 deletions(-) diff --git a/model-evaluation/src/main/java/ai/vespa/models/handler/ModelsEvaluationHandler.java b/model-evaluation/src/main/java/ai/vespa/models/handler/ModelsEvaluationHandler.java index b0e2be26f8aa..d5ae1bbf5919 100644 --- a/model-evaluation/src/main/java/ai/vespa/models/handler/ModelsEvaluationHandler.java +++ b/model-evaluation/src/main/java/ai/vespa/models/handler/ModelsEvaluationHandler.java @@ -4,6 +4,9 @@ import ai.vespa.models.evaluation.FunctionEvaluator; import ai.vespa.models.evaluation.Model; import ai.vespa.models.evaluation.ModelsEvaluator; +import com.fasterxml.jackson.core.JsonEncoding; +import com.fasterxml.jackson.core.JsonFactory; +import com.fasterxml.jackson.core.JsonGenerator; import com.yahoo.container.jdisc.HttpRequest; import com.yahoo.container.jdisc.HttpResponse; import com.yahoo.container.jdisc.ThreadedHttpRequestHandler; @@ -15,6 +18,7 @@ import com.yahoo.tensor.serialization.JsonFormat; import com.yahoo.yolean.Exceptions; +import java.io.ByteArrayOutputStream; import java.io.IOException; import java.io.OutputStream; import java.net.URI; @@ -65,14 +69,14 @@ public HttpResponse handle(HttpRequest request) { } return listModelInformation(request, model, function); - } catch (IllegalArgumentException e) { + } catch (IllegalArgumentException | IOException e) { return new ErrorResponse(404, Exceptions.toMessageString(e)); } catch (IllegalStateException e) { // On missing bindings return new ErrorResponse(400, Exceptions.toMessageString(e)); } } - private HttpResponse evaluateModel(HttpRequest request, Model model, String[] function) { + private HttpResponse evaluateModel(HttpRequest request, Model model, String[] function) throws IOException { FunctionEvaluator evaluator = model.evaluatorOf(function); property(request, missingValueKey).ifPresent(missingValue -> evaluator.setMissingValue(Tensor.from(missingValue))); @@ -87,16 +91,37 @@ private HttpResponse evaluateModel(HttpRequest request, Model model, String[] fu } } } - Tensor result = evaluator.evaluate(); + String format = property(request, "format.tensors").orElse("default"); + if (evaluator.outputs().size() > 1) { + evaluator.evaluate(); + return new Response(200, encodeMultipleResults(evaluator, format)); + } + return new Response(200, encodeSingleResult(evaluator.evaluate(), format)); + } - Optional format = property(request, "format.tensors"); - if (format.isPresent() && format.get().equalsIgnoreCase("short")) { - return new Response(200, JsonFormat.encodeShortForm(result)); + private byte[] encodeMultipleResults(FunctionEvaluator evaluator, String format) throws IOException { + ByteArrayOutputStream out = new ByteArrayOutputStream(); + JsonGenerator g = new JsonFactory().createGenerator(out, JsonEncoding.UTF8); + g.writeStartObject(); + for (String output : evaluator.outputs()) { + g.writeFieldName(output); + g.writeRawValue(new String(encodeSingleResult(evaluator.result(output), format))); } - else if (format.isPresent() && format.get().equalsIgnoreCase("string")) { - return new Response(200, result.toString().getBytes(StandardCharsets.UTF_8)); + g.writeEndObject(); + g.close(); + return out.toByteArray(); + } + + private byte[] encodeSingleResult(Tensor tensor, String format) { + if (format != null) { + if (format.equalsIgnoreCase("short")) { + return JsonFormat.encodeShortForm(tensor); + } + if (format.equalsIgnoreCase("string")) { + return tensor.toString().getBytes(StandardCharsets.UTF_8); + } } - return new Response(200, JsonFormat.encode(result)); + return JsonFormat.encode(tensor); } private HttpResponse listAllModels(HttpRequest request) { diff --git a/model-evaluation/src/test/java/ai/vespa/models/handler/ModelsEvaluationHandlerTest.java b/model-evaluation/src/test/java/ai/vespa/models/handler/ModelsEvaluationHandlerTest.java index 33e56d5d465e..8c7be4e7be95 100644 --- a/model-evaluation/src/test/java/ai/vespa/models/handler/ModelsEvaluationHandlerTest.java +++ b/model-evaluation/src/test/java/ai/vespa/models/handler/ModelsEvaluationHandlerTest.java @@ -205,13 +205,6 @@ public void testMnistSavedTypeDetails() { handler.assertResponse(url, 200, expected); } - @Test - public void testMnistSavedEvaluateDefaultFunctionShouldFail() { - String url = "http://localhost/model-evaluation/v1/mnist_saved/eval"; - String expected = "{\"error\":\"More than one function is available in model 'mnist_saved', but no name is given. Available functions: imported_ml_function_mnist_saved_dnn_hidden1_add, serving_default.y\"}"; - handler.assertResponse(url, 404, expected); - } - @Test public void testVespaModelShortOutput() { Map properties = new HashMap<>(); diff --git a/model-evaluation/src/test/java/ai/vespa/models/handler/OnnxEvaluationHandlerTest.java b/model-evaluation/src/test/java/ai/vespa/models/handler/OnnxEvaluationHandlerTest.java index 6014bd7c7ef4..74715ad96a24 100644 --- a/model-evaluation/src/test/java/ai/vespa/models/handler/OnnxEvaluationHandlerTest.java +++ b/model-evaluation/src/test/java/ai/vespa/models/handler/OnnxEvaluationHandlerTest.java @@ -61,8 +61,8 @@ public void testModelInfo() { @Test public void testEvaluationWithoutSpecifyingOutput() { String url = "http://localhost/model-evaluation/v1/add_mul/eval"; - String expected = "{\"error\":\"More than one function is available in model 'add_mul', but no name is given. Available functions: output1, output2\"}"; - handler.assertResponse(url, 404, expected); + String expected = "{\"error\":\"Argument 'input1' must be bound to a value of type tensor(d0[1])\"}"; + handler.assertResponse(url, 400, expected); } @Test @@ -92,6 +92,19 @@ public void testEvaluationOutput2() { handler.assertResponse(url, properties, 200, expected); } + @Test + public void testEvaluateAllOutputs() { + Map properties = new HashMap<>(); + properties.put("input1", "tensor(d0[1]):[2]"); + properties.put("input2", "tensor(d0[1]):[3]"); + String url = "http://localhost/model-evaluation/v1/add_mul/eval"; // remember to add to discovery! + String expected = "{" + + "\"output1\":{\"cells\":[{\"address\":{\"d0\":\"0\"},\"value\":6.0}]}," + // output1 is a mul + "\"output2\":{\"cells\":[{\"address\":{\"d0\":\"0\"},\"value\":5.0}]}" + // output1 is an add + "}"; + handler.assertResponse(url, properties, 200, expected); + } + @Test public void testBatchDimensionModelInfo() { String url = "http://localhost/model-evaluation/v1/one_layer";