[ONNX] Skip flaky dynamic tests before ORT==1.15 in fx exporter (pytorch#98856)

Disable all flaky dynamic tests
From pytorch#98626 (comment)

Rerun all test cases and update skip reasons. The cases failing on both static and dynamic shapes are unittest.skipped. If it only fails on dynamic, it's skipped by skip_dynamic_test. There are a few skipped with skip_ort_min_version, since ORT is not supporting dynamic fx exporter until next version.
Pull Request resolved: pytorch#98856
Approved by: https://github.com/BowenBao

Author: titaiwangms

test/onnx/pytorch_test_common.py

@@ -158,14 +158,19 @@ def wrapper(self, *args, **kwargs):
     return skip_dec
 
 
-def skip_min_ort_version(reason: str, version: str):
+# TODO(titaiwang): dynamic_only is specific to the situation that dynamic fx exporter
+# is not yet supported by ORT until 1.15.0. Remove dynamic_only once ORT 1.15.0 is released.
+def skip_min_ort_version(reason: str, version: str, dynamic_only: bool = False):
     def skip_dec(func):
         @functools.wraps(func)
         def wrapper(self, *args, **kwargs):
             if (
                 packaging.version.parse(self.ort_version).release
                 < packaging.version.parse(version).release
             ):
+                if dynamic_only and not self.dynamic_shapes:
+                    return func(self, *args, **kwargs)
+
                 raise unittest.SkipTest(
                     f"ONNX Runtime version: {version} is older than required version {version}. "
                     f"Reason: {reason}."
@@ -194,6 +199,7 @@ def wrapper(self, *args, **kwargs):
                 raise unittest.SkipTest(
                     f"Skip verify dynamic shapes test for FX. {reason}"
                 )
+            return func(self, *args, **kwargs)
 
         return wrapper
 

test/onnx/test_fx_to_onnx_with_onnxruntime.py

@@ -7,6 +7,7 @@
 import itertools
 import os
 import tempfile
+import unittest
 import warnings
 
 from typing import Any, Callable, Generator, Optional, Sequence, Tuple, Union
@@ -34,6 +35,16 @@
 _InputArgsType = Union[torch.Tensor, Tuple[Any, ...]]
 _OutputsType = Sequence[_NumericType]
 
+try:
+    import torchvision
+
+    HAS_TORCHVISION = True
+except ImportError:
+    HAS_TORCHVISION = False
+except RuntimeError:
+    HAS_TORCHVISION = False
+skip_if_no_torchvision = unittest.skipIf(not HAS_TORCHVISION, "no torchvision")
+
 
 @_beartype.beartype
 def _run_ort(
@@ -179,6 +190,11 @@ def tearDown(self):
         )
         super().tearDown()
 
+    @pytorch_test_common.skip_min_ort_version(
+        reason="ORT doesn't support dynamic fx exporter yet making SegFault flaky test",
+        version="1.15",
+        dynamic_only=True,
+    )
     def test_simple_function(self):
         def func(x):
             # TODO(justinchuby): Replicate torch's type casting policy
@@ -191,6 +207,11 @@ def func(x):
 
         _run_test_with_fx_to_onnx_exporter_and_onnx_runtime(self, func, (tensor_x,))
 
+    @pytorch_test_common.skip_min_ort_version(
+        reason="ORT doesn't support dynamic fx exporter yet making SegFault flaky test",
+        version="1.15",
+        dynamic_only=True,
+    )
     def test_func_with_args_and_kwargs(self):
         # Non-tensor optional kwargs are always folded into constant and
         # removed from input list in Dynamo-traced graph, so we can't
@@ -223,7 +244,11 @@ def func(x, b=torch.tensor(1.0)):
             self, func, (tensor_x,), b=torch.tensor(5.0)
         )
 
-    @pytorch_test_common.skip_min_ort_version(reason="SegFault", version="1.15")
+    @pytorch_test_common.skip_min_ort_version(
+        reason="ORT doesn't support dynamic fx exporter yet making SegFault flaky test",
+        version="1.15",
+        dynamic_only=True,
+    )
     def test_mnist(self):
         class MNISTModel(nn.Module):
             def __init__(self):
@@ -249,6 +274,11 @@ def forward(self, tensor_x: torch.Tensor):
             self, MNISTModel(), (tensor_x,)
         )
 
+    @pytorch_test_common.skip_min_ort_version(
+        reason="ORT doesn't support dynamic fx exporter yet making SegFault flaky test",
+        version="1.15",
+        dynamic_only=True,
+    )
     # test single op with no kwargs
     def test_sigmoid(self):
         x = torch.randn(1, 4, 2, 3)
@@ -263,13 +293,15 @@ def forward(self, x):
 
         _run_test_with_fx_to_onnx_exporter_and_onnx_runtime(self, SigmoidModel(), (x,))
 
-    @pytorch_test_common.skip_dynamic_fx_test(
-        "_aten_convolution_onnx: _add_attribute_to_torchscript_node()"
-        " parameter value=[None, None] violates type hint"
-        "typing.Union[float, int, str, bytes, typing.Sequence[float],"
-        " typing.Sequence[int], torch.Tensor], as [None, None]:"
+    @unittest.skip(
+        "RuntimeError: false INTERNAL ASSERT FAILED at "
+        "'/home/titaiwang/pytorch/build/aten/src/ATen/RegisterFunctionalization_0.cpp':3725,"
+        " please report a bug to PyTorch. mutating a non-functional tensor with a "
+        "functional tensor is not allowed. Please ensure that all of your inputs are "
+        "wrapped inside of a functionalize() call."
     )
-    def test_shufflenet_v2_dynamic_axes(self):
+    @skip_if_no_torchvision
+    def test_shufflenet_v2(self):
         model = torchvision.models.shufflenet_v2_x0_5(pretrained=False)
         dummy_input = torch.randn(1, 3, 224, 224, requires_grad=True)
         test_inputs = torch.randn(3, 3, 224, 224, requires_grad=True)
@@ -283,6 +315,11 @@ def test_shufflenet_v2_dynamic_axes(self):
             atol=1e-5,
         )
 
+    @pytorch_test_common.skip_min_ort_version(
+        reason="ORT doesn't support dynamic fx exporter yet making SegFault flaky test",
+        version="1.15",
+        dynamic_only=True,
+    )
     def test_add(self):
         class DynamicAdd(torch.nn.Module):
             def forward(self, x, y):
@@ -297,6 +334,11 @@ def forward(self, x, y):
             self, DynamicAdd(), (x, y), additional_test_inputs=[(another_x, another_y)]
         )
 
+    @pytorch_test_common.skip_min_ort_version(
+        reason="ORT doesn't support dynamic fx exporter yet making SegFault flaky test",
+        version="1.15",
+        dynamic_only=True,
+    )
     def test_sigmoid_add(self):
         class DynamicAdd(torch.nn.Module):
             def __init__(self, *args, **kwargs) -> None:
@@ -318,7 +360,11 @@ def forward(self, x, y):
             self, DynamicAdd(), (x, y), additional_test_inputs=[(input_x, input_y)]
         )
 
-    @pytorch_test_common.skip_min_ort_version(reason="SegFault", version="1.15")
+    @pytorch_test_common.skip_min_ort_version(
+        reason="ORT doesn't support dynamic fx exporter yet making SegFault flaky test",
+        version="1.15",
+        dynamic_only=True,
+    )
     def test_matmul(self):
         class DynamicMatMul(torch.nn.Module):
             def forward(self, x, y):
@@ -333,9 +379,8 @@ def forward(self, x, y):
             self, DynamicMatMul(), (x, y), additional_test_inputs=[(input_x, input_y)]
         )
 
-    @pytorch_test_common.skip_dynamic_fx_test(
-        "fx.graph: doesn't handle scalar like normal tensor, so this is not yet "
-        "supported! TypeError: forward() takes 1 positional argument but 2 were given"
+    @unittest.skip(
+        "RuntimeError: The two modules have different number of arguments. module: 1, reference_module: 0"
     )
     def test_scalar_tensor(self):
         class test(torch.nn.Module):
@@ -353,12 +398,6 @@ def forward(self, x):
             additional_test_inputs=[(y,)],
         )
 
-    @pytorch_test_common.skip_dynamic_fx_test(
-        "_aten_convolution_onnx: _add_attribute_to_torchscript_node()"
-        " parameter value=[None, None] violates type hint"
-        "typing.Union[float, int, str, bytes, typing.Sequence[float],"
-        " typing.Sequence[int], torch.Tensor], as [None, None]:"
-    )
     def test_transpose_infer_shape(self):
         class TransposeModule(torch.nn.Module):
             def __init__(self):
@@ -378,7 +417,7 @@ def forward(self, x):
             additional_test_inputs=[(y,)],
         )
 
-    @pytorch_test_common.skip_dynamic_fx_test("torch._dynamo.exc.TorchRuntimeError")
+    @unittest.skip("torch._dynamo.exc.TorchRuntimeError")
     def test_squeeze_runtime_dim(self):
         class Squeeze(torch.nn.Module):
             def forward(self, d1, d2):
@@ -417,7 +456,13 @@ def forward(self, x):
             additional_test_inputs=[(y,)],
         )
 
-    @pytorch_test_common.skip_min_ort_version(reason="SegFault", version="1.15")
+    # TODO(titaiwang): This is also detected flaky in static shape:
+    # https://github.com/pytorch/pytorch/issues/98622
+    @pytorch_test_common.skip_min_ort_version(
+        reason="ORT doesn't support dynamic fx exporter yet making SegFault flaky test",
+        version="1.15",
+        dynamic_only=False,
+    )
     def test_mutation(self):
         class MutationModel(torch.nn.Module):
             def forward(self, x):
@@ -428,9 +473,8 @@ def forward(self, x):
             self, MutationModel(), (torch.randn(12),), input_mutation=True
         )
 
-    @pytorch_test_common.skip_dynamic_fx_test(
-        "fx.graph: doesn't handle scalar like normal tensor, so this is not yet"
-        "supported! TypeError: forward() takes 1 positional argument but 2 were given"
+    @unittest.skip(
+        "RuntimeError: The two modules have different number of arguments. module: 1, reference_module: 0"
     )
     def test_arange(self):
         class ArangeModel(torch.nn.Module):
@@ -450,7 +494,7 @@ def forward(self, input):
             additional_test_inputs=[(y,)],
         )
 
-    @pytorch_test_common.skip_dynamic_fx_test(
+    @unittest.skip(
         "fx.graph: torch._subclasses.fake_tensor.DataDependentOutputException: "
         "aten._local_scalar_dense.default"
     )
@@ -469,10 +513,7 @@ def forward(self, x):
             additional_test_inputs=[(x2,)],
         )
 
-    @pytorch_test_common.skip_dynamic_fx_test(
-        "ATenLib: INVALID_ARGUMENT : Failed to load model with error: "
-        "ONNX Schema aten_copy: failed validating the check: !(it.GetName().empty())"
-    )
+    @unittest.skip("RuntimeError: Unknown call_function target: aten.copy.default")
     def test_expand_as_fill_tensor(self):
         class Model(torch.nn.Module):
             def forward(self, x):
@@ -488,6 +529,11 @@ def forward(self, x):
             additional_test_inputs=[(x2,)],
         )
 
+    @pytorch_test_common.skip_min_ort_version(
+        reason="ORT doesn't support dynamic fx exporter yet making SegFault flaky test",
+        version="1.15",
+        dynamic_only=True,
+    )
     def test_expand_as_fill_seperate_tensor(self):
         class Model(torch.nn.Module):
             def forward(self, x):
@@ -503,6 +549,11 @@ def forward(self, x):
             additional_test_inputs=[(x2,)],
         )
 
+    @pytorch_test_common.skip_min_ort_version(
+        reason="ORT doesn't support dynamic fx exporter yet making SegFault flaky test",
+        version="1.15",
+        dynamic_only=True,
+    )
     def test_view_dynamic_zero_dim(self):
         class ViewModel(torch.nn.Module):
             def forward(self, input):
@@ -518,6 +569,11 @@ def forward(self, input):
             additional_test_inputs=[(another_x,)],
         )
 
+    @pytorch_test_common.skip_min_ort_version(
+        reason="ORT doesn't support dynamic fx exporter yet making SegFault flaky test",
+        version="1.15",
+        dynamic_only=True,
+    )
     def test_flatten_dynamic_axes(self):
         class MyModule(torch.nn.Module):
             def forward(self, x):
@@ -531,7 +587,11 @@ def forward(self, x):
             self, model, (x,), additional_test_inputs=[(y,)]
         )
 
-    @pytorch_test_common.skip_min_ort_version(reason="SegFault", version="1.15")
+    @pytorch_test_common.skip_min_ort_version(
+        reason="ORT doesn't support dynamic fx exporter yet making SegFault flaky test",
+        version="1.15",
+        dynamic_only=True,
+    )
     def test_gpt2_tiny(self):
         model_name = "sshleifer/tiny-gpt2"
         # Download pytorch model