Add converted_end_to_end to check if model is e2e (#678)

tests/conftest.py

@@ -181,6 +181,7 @@ def compile_and_run(device, reset_torch_dynamo, request):
                 "success": True,
                 "run_time": round(run_time, 2),
                 "run_time_first_iter": round(first_iter_runtime, 2),
+                "has_aten": None,
             }
             logging.info(f"Compilation and run successful in {comp_runtime_metrics['run_time']} ms.")
 
@@ -198,6 +199,12 @@ def compile_and_run(device, reset_torch_dynamo, request):
                 option.metrics_path,
                 "compiled-schema_list",
             )
+            comp_runtime_metrics["has_aten"] = False
+            for g in option._out_fx_graphs:
+                nodes = list(g.nodes)
+                if any(["aten." in str(node.target) for node in nodes]):
+                    comp_runtime_metrics["has_aten"] = True
+                    break
         except Exception as e:
             logging.error("Compilation failed.", exc_info=True)
             comp_runtime_metrics = {
@@ -237,6 +244,9 @@ def compile_and_run(device, reset_torch_dynamo, request):
                 pickle.dump(comp_runtime_metrics, f)
             logging.info(f"Compiled runtime metrics saved to {compiled_metrics_path}.")
 
+        if request.node.get_closest_marker("converted_end_to_end") and comp_runtime_metrics.get("has_aten"):
+            raise TypeError(f"{model_name} - Marked as converted end-to-end but still contains aten ops.")
+
 
 def run_model(model, inputs):
     if isinstance(inputs, collections.Mapping):

tests/models/MobileNetV2/test_MobileNetV2.py

@@ -31,6 +31,7 @@ def _load_inputs(self):
     "mode",
     ["eval"],
 )
+@pytest.mark.converted_end_to_end
 def test_MobileNetV2(record_property, mode):
     model_name = "MobileNetV2"
     record_property("model_name", model_name)

tests/models/albert/test_albert_masked_lm.py

@@ -33,7 +33,12 @@ def append_fake_loss_function(self, outputs):
 )
 @pytest.mark.parametrize(
     "model_name",
-    ["albert/albert-base-v2", "albert/albert-large-v2", "albert/albert-xlarge-v2", "albert/albert-xxlarge-v2"],
+    [
+        pytest.param("albert/albert-base-v2", marks=pytest.mark.converted_end_to_end),
+        pytest.param("albert/albert-large-v2", marks=pytest.mark.converted_end_to_end),
+        pytest.param("albert/albert-xlarge-v2", marks=pytest.mark.converted_end_to_end),
+        "albert/albert-xxlarge-v2",
+    ],
 )
 def test_albert_masked_lm(record_property, model_name, mode):
     record_property("model_name", model_name)

tests/models/albert/test_albert_question_answering.py

@@ -22,6 +22,7 @@ def _load_inputs(self):
     "mode",
     ["eval"],
 )
+@pytest.mark.converted_end_to_end
 @pytest.mark.parametrize("model_name", ["twmkn9/albert-base-v2-squad2"])
 def test_albert_question_answering(record_property, model_name, mode):
     record_property("model_name", model_name)

tests/models/albert/test_albert_sequence_classification.py

@@ -21,6 +21,7 @@ def _load_inputs(self):
     "mode",
     ["eval"],
 )
+@pytest.mark.converted_end_to_end
 @pytest.mark.parametrize("model_name", ["textattack/albert-base-v2-imdb"])
 def test_albert_sequence_classification(record_property, model_name, mode):
     record_property("model_name", model_name)

tests/models/albert/test_albert_token_classification.py

@@ -21,7 +21,12 @@ def _load_inputs(self):
     "mode",
     ["eval"],
 )
-@pytest.mark.parametrize("model_name", ["albert/albert-base-v2"])
+@pytest.mark.parametrize(
+    "model_name",
+    [
+        pytest.param("albert/albert-base-v2", marks=pytest.mark.converted_end_to_end),
+    ],
+)
 def test_albert_token_classification(record_property, model_name, mode):
     record_property("model_name", f"{model_name}-classification")
     record_property("mode", mode)

tests/models/autoencoder_linear/test_autoencoder_linear.py

@@ -78,7 +78,7 @@ def _load_inputs(self):
 
 @pytest.mark.parametrize(
     "mode",
-    ["train", "eval"],
+    ["train", pytest.param("eval", marks=pytest.mark.converted_end_to_end)],
 )
 def test_autoencoder_linear(record_property, mode):
     model_name = "Autoencoder (linear)"

tests/models/bert/test_bert.py

@@ -34,6 +34,7 @@ def _load_inputs(self):
     "mode",
     ["eval"],
 )
+@pytest.mark.converted_end_to_end
 def test_bert(record_property, mode):
     model_name = "BERT"
     record_property("model_name", model_name)

tests/models/bloom/test_bloom.py

@@ -32,6 +32,7 @@ def _load_inputs(self):
     "mode",
     ["eval"],
 )
+@pytest.mark.converted_end_to_end
 def test_bloom(record_property, mode):
     model_name = "Bloom"
     record_property("model_name", model_name)

tests/models/distilbert/test_distilbert.py

@@ -20,6 +20,7 @@ def _load_inputs(self):
     "mode",
     ["eval"],
 )
+@pytest.mark.converted_end_to_end
 @pytest.mark.parametrize("model_name", ["distilbert-base-uncased"])
 def test_distilbert(record_property, model_name, mode):
     record_property("model_name", model_name)

tests/models/dpr/test_dpr.py

@@ -26,6 +26,7 @@ def _load_inputs(self):
     "mode",
     ["eval"],
 )
+@pytest.mark.converted_end_to_end
 def test_dpr(record_property, mode):
     model_name = "DPR"
     record_property("model_name", model_name)

tests/models/hand_landmark/test_hand_landmark.py

@@ -39,6 +39,7 @@ def set_model_eval(self, model):
     ["eval"],
 )
 @pytest.mark.usefixtures("manage_dependencies")
+@pytest.mark.converted_end_to_end
 def test_hand_landmark(record_property, mode):
     model_name = "Hand Landmark"
     record_property("model_name", model_name)

tests/models/llama/test_llama.py

@@ -35,6 +35,7 @@ def _load_inputs(self):
     "mode",
     ["eval"],
 )
+@pytest.mark.converted_end_to_end
 def test_llama(record_property, mode):
     model_name = "Llama"
     record_property("model_name", model_name)

tests/models/mnist/test_mnist.py

@@ -52,7 +52,7 @@ def _load_inputs(self):
 
 @pytest.mark.parametrize(
     "mode",
-    ["train", "eval"],
+    ["train", pytest.param("eval", marks=pytest.mark.converted_end_to_end)],
 )
 def test_mnist_train(record_property, mode):
     model_name = "Mnist"

tests/models/openpose/test_openpose_v2.py

@@ -42,7 +42,10 @@ def _load_inputs(self):
 
 @pytest.mark.parametrize(
     "mode",
-    ["train", "eval"],
+    [
+        "train",
+        pytest.param("eval", marks=pytest.mark.converted_end_to_end),
+    ],
 )
 def test_openpose_v2(record_property, mode):
     model_name = "OpenPose V2"

tests/models/perceiver_io/test_perceiver_io.py

@@ -28,6 +28,7 @@ def _load_inputs(self):
     "mode",
     ["eval"],
 )
+@pytest.mark.converted_end_to_end
 def test_perceiver_io(record_property, mode):
     model_name = "Perceiver IO"
     record_property("model_name", model_name)

tests/models/resnet/test_resnet.py

@@ -18,7 +18,10 @@ def _load_inputs(self):
 
 @pytest.mark.parametrize(
     "mode",
-    ["train", "eval"],
+    [
+        "train",
+        pytest.param("eval", marks=pytest.mark.converted_end_to_end),
+    ],
 )
 def test_resnet(record_property, mode):
     model_name = "ResNet18"

tests/models/resnet50/test_resnet50.py

@@ -31,7 +31,10 @@ def _load_inputs(self):
 
 @pytest.mark.parametrize(
     "mode",
-    ["train", "eval"],
+    [
+        "train",
+        pytest.param("eval", marks=pytest.mark.converted_end_to_end),
+    ],
 )
 def test_resnet(record_property, mode):
     model_name = "ResNet50"

tests/models/roberta/test_roberta.py

@@ -21,6 +21,7 @@ def _load_inputs(self):
     "mode",
     ["eval"],
 )
+@pytest.mark.converted_end_to_end
 def test_roberta(record_property, mode):
     model_name = "RoBERTa"
     record_property("model_name", model_name)

tests/models/squeeze_bert/test_squeeze_bert.py

@@ -23,6 +23,7 @@ def _load_inputs(self):
     "mode",
     ["eval"],
 )
+@pytest.mark.converted_end_to_end
 def test_squeeze_bert(record_property, mode):
     model_name = "SqueezeBERT"
     record_property("model_name", model_name)

tests/models/timm/test_timm_image_classification.py

@@ -49,19 +49,19 @@ def _load_inputs(self):
     ["xception71.tf_in1k", "train"],
     ["dla34.in1k", "train"],
     pytest.param(["hrnet_w18.ms_aug_in1k", "train"], marks=pytest.mark.compilation_xfail),
-    ["tf_efficientnet_lite0.in1k", "eval"],
-    ["tf_efficientnet_lite1.in1k", "eval"],
-    ["tf_efficientnet_lite2.in1k", "eval"],
+    pytest.param(["tf_efficientnet_lite0.in1k", "eval"], marks=pytest.mark.converted_end_to_end),
+    pytest.param(["tf_efficientnet_lite1.in1k", "eval"], marks=pytest.mark.converted_end_to_end),
+    pytest.param(["tf_efficientnet_lite2.in1k", "eval"], marks=pytest.mark.converted_end_to_end),
     ["tf_efficientnet_lite3.in1k", "eval"],
     ["tf_efficientnet_lite4.in1k", "eval"],
-    ["ghostnet_100.in1k", "eval"],
+    pytest.param(["ghostnet_100.in1k", "eval"], marks=pytest.mark.converted_end_to_end),
     ["ghostnetv2_100.in1k", "eval"],
     ["inception_v4.tf_in1k", "eval"],
     ["mixer_b16_224.goog_in21k", "eval"],
-    ["mobilenetv1_100.ra4_e3600_r224_in1k", "eval"],
+    pytest.param(["mobilenetv1_100.ra4_e3600_r224_in1k", "eval"], marks=pytest.mark.converted_end_to_end),
     ["ese_vovnet19b_dw.ra_in1k", "eval"],
-    ["xception71.tf_in1k", "eval"],
-    ["dla34.in1k", "eval"],
+    pytest.param(["xception71.tf_in1k", "eval"], marks=pytest.mark.converted_end_to_end),
+    pytest.param(["dla34.in1k", "eval"], marks=pytest.mark.converted_end_to_end),
     ["hrnet_w18.ms_aug_in1k", "eval"],
 ]
 

tests/models/torchvision/test_torchvision_image_classification.py

@@ -38,47 +38,51 @@ def _load_inputs(self):
     [("densenet161", "DenseNet161_Weights"), "eval"],
     [("densenet169", "DenseNet169_Weights"), "eval"],
     [("densenet201", "DenseNet201_Weights"), "eval"],
-    [("mobilenet_v2", "MobileNet_V2_Weights"), "eval"],
-    [("mobilenet_v3_small", "MobileNet_V3_Small_Weights"), "eval"],
-    [("mobilenet_v3_large", "MobileNet_V3_Large_Weights"), "eval"],
-    [("resnet18", "ResNet18_Weights"), "eval"],
-    [("resnet34", "ResNet34_Weights"), "eval"],
-    [("resnet50", "ResNet50_Weights"), "eval"],
-    [("resnet101", "ResNet101_Weights"), "eval"],
-    [("resnet152", "ResNet152_Weights"), "eval"],
-    [("resnext50_32x4d", "ResNeXt50_32X4D_Weights"), "eval"],
-    [("resnext101_32x8d", "ResNeXt101_32X8D_Weights"), "eval"],
-    [("resnext101_64x4d", "ResNeXt101_64X4D_Weights"), "eval"],
-    [("vgg11", "VGG11_Weights"), "eval"],
-    [("vgg11_bn", "VGG11_BN_Weights"), "eval"],
-    [("vgg13", "VGG13_Weights"), "eval"],
-    [("vgg13_bn", "VGG13_BN_Weights"), "eval"],
-    [("vgg16", "VGG16_Weights"), "eval"],
-    [("vgg16_bn", "VGG16_BN_Weights"), "eval"],
-    [("vgg19", "VGG19_Weights"), "eval"],
-    [("vgg19_bn", "VGG19_BN_Weights"), "eval"],
+    pytest.param([("mobilenet_v2", "MobileNet_V2_Weights"), "eval"], marks=pytest.mark.converted_end_to_end),
+    pytest.param(
+        [("mobilenet_v3_small", "MobileNet_V3_Small_Weights"), "eval"], marks=pytest.mark.converted_end_to_end
+    ),
+    pytest.param(
+        [("mobilenet_v3_large", "MobileNet_V3_Large_Weights"), "eval"], marks=pytest.mark.converted_end_to_end
+    ),
+    pytest.param([("resnet18", "ResNet18_Weights"), "eval"], marks=pytest.mark.converted_end_to_end),
+    pytest.param([("resnet34", "ResNet34_Weights"), "eval"], marks=pytest.mark.converted_end_to_end),
+    pytest.param([("resnet50", "ResNet50_Weights"), "eval"], marks=pytest.mark.converted_end_to_end),
+    pytest.param([("resnet101", "ResNet101_Weights"), "eval"], marks=pytest.mark.converted_end_to_end),
+    pytest.param([("resnet152", "ResNet152_Weights"), "eval"], marks=pytest.mark.converted_end_to_end),
+    pytest.param([("resnext50_32x4d", "ResNeXt50_32X4D_Weights"), "eval"], marks=pytest.mark.converted_end_to_end),
+    pytest.param([("resnext101_32x8d", "ResNeXt101_32X8D_Weights"), "eval"], marks=pytest.mark.converted_end_to_end),
+    pytest.param([("resnext101_64x4d", "ResNeXt101_64X4D_Weights"), "eval"], marks=pytest.mark.converted_end_to_end),
+    pytest.param([("vgg11", "VGG11_Weights"), "eval"], marks=pytest.mark.converted_end_to_end),
+    pytest.param([("vgg11_bn", "VGG11_BN_Weights"), "eval"], marks=pytest.mark.converted_end_to_end),
+    pytest.param([("vgg13", "VGG13_Weights"), "eval"], marks=pytest.mark.converted_end_to_end),
+    pytest.param([("vgg13_bn", "VGG13_BN_Weights"), "eval"], marks=pytest.mark.converted_end_to_end),
+    pytest.param([("vgg16", "VGG16_Weights"), "eval"], marks=pytest.mark.converted_end_to_end),
+    pytest.param([("vgg16_bn", "VGG16_BN_Weights"), "eval"], marks=pytest.mark.converted_end_to_end),
+    pytest.param([("vgg19", "VGG19_Weights"), "eval"], marks=pytest.mark.converted_end_to_end),
+    pytest.param([("vgg19_bn", "VGG19_BN_Weights"), "eval"], marks=pytest.mark.converted_end_to_end),
     [("vit_b_16", "ViT_B_16_Weights"), "eval"],
     [("vit_b_32", "ViT_B_32_Weights"), "eval"],
     [("vit_l_16", "ViT_L_16_Weights"), "eval"],
     [("vit_l_32", "ViT_L_32_Weights"), "eval"],
     [("vit_h_14", "ViT_H_14_Weights"), "eval"],
-    [("wide_resnet50_2", "Wide_ResNet50_2_Weights"), "eval"],
-    [("wide_resnet101_2", "Wide_ResNet101_2_Weights"), "eval"],
-    [("regnet_y_400mf", "RegNet_Y_400MF_Weights"), "eval"],
-    [("regnet_y_800mf", "RegNet_Y_800MF_Weights"), "eval"],
-    [("regnet_y_1_6gf", "RegNet_Y_1_6GF_Weights"), "eval"],
-    [("regnet_y_3_2gf", "RegNet_Y_3_2GF_Weights"), "eval"],
-    [("regnet_y_8gf", "RegNet_Y_8GF_Weights"), "eval"],
+    pytest.param([("wide_resnet50_2", "Wide_ResNet50_2_Weights"), "eval"], marks=pytest.mark.converted_end_to_end),
+    pytest.param([("wide_resnet101_2", "Wide_ResNet101_2_Weights"), "eval"], marks=pytest.mark.converted_end_to_end),
+    pytest.param([("regnet_y_400mf", "RegNet_Y_400MF_Weights"), "eval"], marks=pytest.mark.converted_end_to_end),
+    pytest.param([("regnet_y_800mf", "RegNet_Y_800MF_Weights"), "eval"], marks=pytest.mark.converted_end_to_end),
+    pytest.param([("regnet_y_1_6gf", "RegNet_Y_1_6GF_Weights"), "eval"], marks=pytest.mark.converted_end_to_end),
+    pytest.param([("regnet_y_3_2gf", "RegNet_Y_3_2GF_Weights"), "eval"], marks=pytest.mark.converted_end_to_end),
+    pytest.param([("regnet_y_8gf", "RegNet_Y_8GF_Weights"), "eval"], marks=pytest.mark.converted_end_to_end),
     [("regnet_y_16gf", "RegNet_Y_16GF_Weights"), "eval"],
-    [("regnet_y_32gf", "RegNet_Y_32GF_Weights"), "eval"],
+    pytest.param([("regnet_y_32gf", "RegNet_Y_32GF_Weights"), "eval"], marks=pytest.mark.converted_end_to_end),
     [("regnet_y_128gf", "RegNet_Y_128GF_Weights"), "eval"],
-    [("regnet_x_400mf", "RegNet_X_400MF_Weights"), "eval"],
-    [("regnet_x_800mf", "RegNet_X_800MF_Weights"), "eval"],
-    [("regnet_x_1_6gf", "RegNet_X_1_6GF_Weights"), "eval"],
-    [("regnet_x_3_2gf", "RegNet_X_3_2GF_Weights"), "eval"],
-    [("regnet_x_8gf", "RegNet_X_8GF_Weights"), "eval"],
-    [("regnet_x_16gf", "RegNet_X_16GF_Weights"), "eval"],
-    [("regnet_x_32gf", "RegNet_X_32GF_Weights"), "eval"],
+    pytest.param([("regnet_x_400mf", "RegNet_X_400MF_Weights"), "eval"], marks=pytest.mark.converted_end_to_end),
+    pytest.param([("regnet_x_800mf", "RegNet_X_800MF_Weights"), "eval"], marks=pytest.mark.converted_end_to_end),
+    pytest.param([("regnet_x_1_6gf", "RegNet_X_1_6GF_Weights"), "eval"], marks=pytest.mark.converted_end_to_end),
+    pytest.param([("regnet_x_3_2gf", "RegNet_X_3_2GF_Weights"), "eval"], marks=pytest.mark.converted_end_to_end),
+    pytest.param([("regnet_x_8gf", "RegNet_X_8GF_Weights"), "eval"], marks=pytest.mark.converted_end_to_end),
+    pytest.param([("regnet_x_16gf", "RegNet_X_16GF_Weights"), "eval"], marks=pytest.mark.converted_end_to_end),
+    pytest.param([("regnet_x_32gf", "RegNet_X_32GF_Weights"), "eval"], marks=pytest.mark.converted_end_to_end),
     [("swin_t", "Swin_T_Weights"), "eval"],
     [("swin_s", "Swin_S_Weights"), "eval"],
     [("swin_b", "Swin_B_Weights"), "eval"],

tests/models/unet/test_unet.py

@@ -40,7 +40,10 @@ def _load_inputs(self):
 
 @pytest.mark.parametrize(
     "mode",
-    ["train", "eval"],
+    [
+        "train",
+        pytest.param("eval", marks=pytest.mark.converted_end_to_end),
+    ],
 )
 def test_unet(record_property, mode):
     model_name = "U-Net"

tests/models/unet_brain/test_unet_brain.py

@@ -50,7 +50,10 @@ def _load_inputs(self):
 
 @pytest.mark.parametrize(
     "mode",
-    ["train", "eval"],
+    [
+        "train",
+        pytest.param("eval", marks=pytest.mark.converted_end_to_end),
+    ],
 )
 def test_unet_brain(record_property, mode):
     model_name = "Unet-brain"

tests/models/unet_carvana/test_unet_carvana.py

@@ -26,7 +26,10 @@ def _load_inputs(self):
 
 @pytest.mark.parametrize(
     "mode",
-    ["train", "eval"],
+    [
+        "train",
+        pytest.param("eval", marks=pytest.mark.converted_end_to_end),
+    ],
 )
 def test_unet_carvana(record_property, mode):
     model_name = "Unet-carvana"

tests/models/yolov5/test_yolov5.py

@@ -96,6 +96,7 @@ def teardown_module(module):
     "mode",
     ["eval"],
 )
+@pytest.mark.converted_end_to_end
 @pytest.mark.usefixtures("manage_dependencies")
 def test_yolov5(record_property, mode):
     model_name = "YOLOv5"