diff --git a/src/deepsparse/evaluation/evaluator.py b/src/deepsparse/evaluation/evaluator.py
index 3d18f8489f..6d24c8f6da 100644
--- a/src/deepsparse/evaluation/evaluator.py
+++ b/src/deepsparse/evaluation/evaluator.py
@@ -56,9 +56,10 @@ def evaluate(
 
     # if target is a string, turn it into an appropriate pipeline
     # otherwise assume it is a pipeline
-    pipeline = (
-        create_pipeline(model, engine_type) if isinstance(model, (Path, str)) else model
-    )
+    if isinstance(model, (Path, str)):
+        pipeline, kwargs = create_pipeline(model, engine_type, **kwargs)
+    else:
+        pipeline = model
 
     eval_integration = EvaluationRegistry.resolve(pipeline, datasets, integration)
 
diff --git a/src/deepsparse/evaluation/utils.py b/src/deepsparse/evaluation/utils.py
index 15503f0553..111fda9591 100644
--- a/src/deepsparse/evaluation/utils.py
+++ b/src/deepsparse/evaluation/utils.py
@@ -202,11 +202,13 @@ def create_pipeline(
     :return: The initialized pipeline
     """
     engine_type = engine_type or DEEPSPARSE_ENGINE
-    return Pipeline.create(
-        task=kwargs.pop("task", "text-generation"),
-        model_path=model_path,
-        sequence_length=kwargs.pop("sequence_length", 2048),
-        engine_type=engine_type,
-        batch_size=kwargs.pop("batch_size", 1),
-        **kwargs,
+    return (
+        Pipeline.create(
+            task=kwargs.pop("task", "text-generation"),
+            model_path=model_path,
+            sequence_length=kwargs.pop("sequence_length", 2048),
+            engine_type=engine_type,
+            batch_size=kwargs.pop("batch_size", 1),
+        ),
+        kwargs,
     )