Fix deadline/flaky + suppress GPU-only opcheck SKIPPING (OMH round 2) (#5932)

fbgemm_gpu/test/permute/permute_pooled_embedding_test.py

@@ -58,7 +58,7 @@ def setUp(self) -> None:
         # registration (flaky). T191384137
         self.device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
 
-    @settings(deadline=10000)
+    @settings(deadline=None)
     @given(fwd_only=st.booleans())
     def test_permutation(self, fwd_only: bool) -> None:
         net = Net(fwd_only=fwd_only).to(self.device)

fbgemm_gpu/test/quantize/bfloat16_test.py

@@ -27,7 +27,7 @@ class SparseNNOperatorsGPUTest(unittest.TestCase):
         k=st.integers(min_value=2, max_value=2),
         n=st.integers(min_value=2, max_value=2),
     )
-    @settings(deadline=10000, suppress_health_check=[HealthCheck.filter_too_much])
+    @settings(deadline=None, suppress_health_check=[HealthCheck.filter_too_much])
     def test_dense_mlp_quantize_ops(
         self, precision: str, batch_size: int, k: int, n: int
     ) -> None:
@@ -64,7 +64,7 @@ class TestBfloat16QuantizationConversion(unittest.TestCase):
         nrows=st.integers(min_value=0, max_value=100),
         ncols=st.integers(min_value=0, max_value=100),
     )
-    @settings(deadline=10000, suppress_health_check=[HealthCheck.filter_too_much])
+    @settings(deadline=None, suppress_health_check=[HealthCheck.filter_too_much])
     def test_quantize_op(self, nrows: int, ncols: int) -> None:
         input_data = torch.rand(nrows, ncols).float()
         quantized_data = torch.ops.fbgemm.FloatToBfloat16Quantized(input_data)
@@ -91,7 +91,7 @@ def test_quantize_op(self, nrows: int, ncols: int) -> None:
         nrows=st.integers(min_value=0, max_value=100),
         ncols=st.integers(min_value=0, max_value=100),
     )
-    @settings(deadline=10000, suppress_health_check=[HealthCheck.filter_too_much])
+    @settings(deadline=None, suppress_health_check=[HealthCheck.filter_too_much])
     def test_quantize_and_dequantize_op(self, nrows: int, ncols: int) -> None:
         input_data = torch.rand(nrows, ncols).float()
         quantized_data = torch.ops.fbgemm.FloatToBfloat16Quantized(input_data)
@@ -129,7 +129,7 @@ def test_quantize_and_dequantize_op(self, nrows: int, ncols: int) -> None:
     @given(
         ncols_nrows=st.sampled_from([(65540, 256), (256, 65540)]),
     )
-    @settings(deadline=10000, suppress_health_check=[HealthCheck.filter_too_much])
+    @settings(deadline=None, suppress_health_check=[HealthCheck.filter_too_much])
     def test_quantize_and_dequantize_op_cuda_large_nrows_bf16(
         self, ncols_nrows: tuple[int, int]
     ) -> None:

fbgemm_gpu/test/quantize/fused_8bit_rowwise_test.py

@@ -50,7 +50,7 @@ class TestFused8BitRowwiseQuantizationConversion(unittest.TestCase):
         is_half=st.booleans(),
         test_float_or_half_op=st.booleans(),
     )
-    @settings(deadline=10000, suppress_health_check=[HealthCheck.filter_too_much])
+    @settings(deadline=None, suppress_health_check=[HealthCheck.filter_too_much])
     def test_quantize_op(
         self,
         nrows: int,
@@ -336,7 +336,7 @@ def quantize_and_dequantize_op_test_helper(  # noqa: C901
         ),
         test_generic_op=st.booleans(),
     )
-    @settings(deadline=10000, suppress_health_check=[HealthCheck.filter_too_much])
+    @settings(deadline=None, suppress_health_check=[HealthCheck.filter_too_much])
     def test_quantize_and_dequantize_op_cpu(  # noqa: C901
         self,
         nrows: int,
@@ -361,7 +361,7 @@ def test_quantize_and_dequantize_op_cpu(  # noqa: C901
         ),
         test_generic_op=st.booleans(),
     )
-    @settings(deadline=10000, suppress_health_check=[HealthCheck.filter_too_much])
+    @settings(deadline=None, suppress_health_check=[HealthCheck.filter_too_much])
     def test_quantize_and_dequantize_op_cuda(  # noqa: C901
         self,
         nrows: int,

fbgemm_gpu/test/quantize/fused_nbit_rowwise_test.py

@@ -49,7 +49,7 @@ class TestFusedNBitRowwiseQuantizationConversion(unittest.TestCase):
         is_half=st.booleans(),
         test_float_or_half_op=st.booleans(),
     )
-    @settings(deadline=10000, suppress_health_check=[HealthCheck.filter_too_much])
+    @settings(deadline=None, suppress_health_check=[HealthCheck.filter_too_much])
     def test_quantize_op(
         self,
         nrows: int,
@@ -151,7 +151,7 @@ def test_quantize_op(
         test_meta=st.booleans(),
         test_cuda=st.booleans(),
     )
-    @settings(deadline=10000, suppress_health_check=[HealthCheck.filter_too_much])
+    @settings(deadline=None, suppress_health_check=[HealthCheck.filter_too_much])
     def test_quantize_and_dequantize_op(
         self,
         nrows: int,
@@ -357,7 +357,7 @@ def test_quantize_and_dequantize_op_cuda_large_nrows(self) -> None:
             [SparseType.BF16]
         ),
     )
-    @settings(deadline=10000, suppress_health_check=[HealthCheck.filter_too_much])
+    @settings(deadline=None, suppress_health_check=[HealthCheck.filter_too_much])
     def test_quantize_and_dequantize_op_cpu_and_cuda(
         self,
         nrows: int,

fbgemm_gpu/test/quantize/hfp8_test.py

@@ -75,7 +75,7 @@ def _test_conversion(
         ncols=st.integers(min_value=1, max_value=100),
         exponent_bias=st.integers(min_value=4, max_value=7),
     )
-    @settings(deadline=10000, suppress_health_check=[HealthCheck.filter_too_much])
+    @settings(deadline=None, suppress_health_check=[HealthCheck.filter_too_much])
     def test_quantize_and_dequantize_op(
         self, nrows: int, ncols: int, exponent_bias: int
     ) -> None:

fbgemm_gpu/test/quantize/mixed_dim_int8_test.py

@@ -55,7 +55,7 @@ def test_mixed_dim_8bit_dequantize_op_empty(self) -> None:
         min_dim=st.just(1),
         max_dim=st.just(100),
     )
-    @settings(deadline=10000, suppress_health_check=[HealthCheck.filter_too_much])
+    @settings(deadline=None, suppress_health_check=[HealthCheck.filter_too_much])
     def test_mixed_dim_8bit_dequantize_op(
         self,
         B: int,
@@ -77,7 +77,7 @@ def test_mixed_dim_8bit_dequantize_op(
         min_dim=st.just(100),
         max_dim=st.just(1000),
     )
-    @settings(deadline=10000, suppress_health_check=[HealthCheck.filter_too_much])
+    @settings(deadline=None, suppress_health_check=[HealthCheck.filter_too_much])
     def test_mixed_dim_8bit_dequantize_op_large_dims(
         self,
         B: int,
@@ -99,7 +99,7 @@ def test_mixed_dim_8bit_dequantize_op_large_dims(
         min_dim=st.just(1),
         max_dim=st.just(100),
     )
-    @settings(deadline=10000, suppress_health_check=[HealthCheck.filter_too_much])
+    @settings(deadline=None, suppress_health_check=[HealthCheck.filter_too_much])
     def test_mixed_dim_8bit_dequantize_op_large_rows(
         self,
         B: int,

fbgemm_gpu/test/quantize/msfp_test.py

@@ -33,7 +33,7 @@ class TestMSFPQuantizationConversion(unittest.TestCase):
         nrows=st.integers(min_value=0, max_value=100),
         ncols=st.integers(min_value=0, max_value=100),
     )
-    @settings(deadline=10000, suppress_health_check=[HealthCheck.filter_too_much])
+    @settings(deadline=None, suppress_health_check=[HealthCheck.filter_too_much])
     def test_quantize_op(self, nrows: int, ncols: int) -> None:
         ebits = 8
         mbits = 7

fbgemm_gpu/test/sparse/failures_dict.json

@@ -23,7 +23,12 @@
     "fbgemm::asynchronous_inclusive_cumsum": {},
     "fbgemm::batch_index_select_dim0": {},
     "fbgemm::batch_index_select_dim0_tensor": {},
-    "fbgemm::block_bucketize_sparse_features": {},
+    "fbgemm::block_bucketize_sparse_features": {
+      "BlockBucketizeTest.test_aot_dispatch_dynamic__test_block_bucketize_sparse_features_large": {
+        "comment": "max_examples=32 with my_size up to 1024 (~2048 items) recompiles per hypothesis example under aot_dispatch_dynamic and exceeds the 600s tpx timeout (flaky ~10%). Eager + faketensor variants still cover the op. T191384137",
+        "status": "skip"
+      }
+    },
     "fbgemm::block_bucketize_sparse_features_2d_weights": {},
     "fbgemm::block_bucketize_sparse_features_inference": {},
     "fbgemm::bottom_k_per_row": {

fbgemm_gpu/test/sparse/histogram_binning_calibration_test.py

@@ -20,10 +20,10 @@
 
 if open_source:
     # pyre-ignore[21]
-    from test_utils import gpu_unavailable
+    from test_utils import gpu_unavailable, optests
 else:
     import fbgemm_gpu.sparse_ops  # noqa: F401, E402
-    from fbgemm_gpu.test.test_utils import gpu_unavailable
+    from fbgemm_gpu.test.test_utils import gpu_unavailable, optests
 
 
 class HistogramBinningCalibrationTest(unittest.TestCase):
@@ -327,6 +327,9 @@ def test_generic_histogram_binning_calibration_by_feature(
         data_type=st.sampled_from([torch.bfloat16, torch.half, torch.float32]),
     )
     @settings(verbosity=Verbosity.verbose, deadline=None)
+    @optests.dontGenerateOpCheckTests(
+        "GPU-only CPU/GPU parity test; op covered by test_generic_histogram_binning_calibration_by_feature (T191384137)"
+    )
     def test_generic_histogram_binning_calibration_by_feature_cpu_gpu(
         self,
         data_type: torch.dtype,

fbgemm_gpu/test/sparse/index_select_test.py

@@ -26,10 +26,10 @@
 
 if open_source:
     # pyre-ignore[21]
-    from test_utils import gpu_available
+    from test_utils import gpu_available, optests
 else:
     import fbgemm_gpu.sparse_ops  # noqa: F401, E402
-    from fbgemm_gpu.test.test_utils import gpu_available
+    from fbgemm_gpu.test.test_utils import gpu_available, optests
 
 
 class IndexSelectTest(unittest.TestCase):
@@ -251,6 +251,9 @@ def compare_tensor_groups(
         )
 
     @unittest.skipIf(not gpu_available, "Skip when CUDA is not available")
+    @optests.dontGenerateOpCheckTests(
+        "GPU-only test; opcheck variants only skip on CPU samples and add no op coverage (T191384137)"
+    )
     def test_group_index_select_dim0_mixed_input_dtype(self) -> None:
         # lint-fixme: TorchDeviceCuda, TorchFunctionCallCudaDevice
         # CUDA specifically required: testing GPU-only FBGEMM sparse op validation
@@ -268,6 +271,9 @@ def test_group_index_select_dim0_mixed_input_dtype(self) -> None:
             )
 
     @unittest.skipIf(not gpu_available, "Skip when CUDA is not available")
+    @optests.dontGenerateOpCheckTests(
+        "GPU-only test; opcheck variants only skip on CPU samples and add no op coverage (T191384137)"
+    )
     def test_group_index_select_dim0_mixed_indices_dtype(self) -> None:
         # lint-fixme: TorchDeviceCuda, TorchFunctionCallCudaDevice
         # CUDA specifically required: testing GPU-only FBGEMM sparse op validation
@@ -499,6 +505,9 @@ def test_batch_index_select_dim0(
         max_examples=20,
         deadline=None,
     )
+    @optests.dontGenerateOpCheckTests(
+        "GPU-only test; opcheck variants only skip on CPU samples and add no op coverage (T191384137)"
+    )
     def test_batch_index_select_dim0_long_runs(
         self,
         input_rows_count: int,

fbgemm_gpu/test/sparse/pack_segments_test.py

@@ -23,12 +23,13 @@
 
 if open_source:
     # pyre-ignore[21]
-    from test_utils import gpu_available, gpu_memory_lt_gb, gpu_unavailable
+    from test_utils import gpu_available, gpu_memory_lt_gb, gpu_unavailable, optests
 else:
     from fbgemm_gpu.test.test_utils import (
         gpu_available,
         gpu_memory_lt_gb,
         gpu_unavailable,
+        optests,
     )
 
 
@@ -423,6 +424,9 @@ def test_pack_segments_meta_backend(
         use_cpu=st.booleans(),
     )
     @settings(deadline=None)
+    @optests.dontGenerateOpCheckTests(
+        "GPU-only test; opcheck variants only skip on CPU samples; op covered by test_pack_segments (T191384137)"
+    )
     def test_pack_segments_noncontig(
         self,
         n: int,
@@ -569,6 +573,9 @@ def test_pack_segments_noncontig(
         ),
     )
     @settings(deadline=None)
+    @optests.dontGenerateOpCheckTests(
+        "GPU-only test; opcheck variants only skip on CPU samples; op covered by test_pack_segments (T191384137)"
+    )
     def test_pack_segments_backward_truncated(self, dtype: torch.dtype) -> None:
         """
         Regression test: when lengths[seq] > max_length, the backward kernel
@@ -634,6 +641,9 @@ def test_pack_segments_backward_truncated(self, dtype: torch.dtype) -> None:
     # output of shape (num_seq, max_length) at fp16, ~8 GiB at the chosen
     # max_length.
     @unittest.skipIf(*gpu_memory_lt_gb(12))
+    @optests.dontGenerateOpCheckTests(
+        "large-grid GPU-memory-gated stress repro; opcheck variants add no coverage (T191384137)"
+    )
     def test_pack_segments_large_grid(self) -> None:
         """
         Reproduces the HIP grid-overflow bug in pack_segments_cuda{,_v2}

fbgemm_gpu/test/sparse/permute_indices_test.py

@@ -182,6 +182,9 @@ def test_permute_indices(
     )
     @settings(verbosity=Verbosity.verbose, max_examples=20, deadline=None)
     @unittest.skipIf(*gpu_unavailable)
+    @optests.dontGenerateOpCheckTests(
+        "GPU-only test; opcheck variants only skip on CPU samples and add no op coverage (T191384137)"
+    )
     def test_permute_indices_non_contiguous(
         self,
         B: int,
@@ -246,6 +249,9 @@ def create_non_contiguous(x: torch.Tensor) -> torch.Tensor:
     # models returned. So we need to add a unittest to ensure the op return
     # real None, not an undefined tensor.
     @unittest.skipIf(*gpu_unavailable)
+    @optests.dontGenerateOpCheckTests(
+        "GPU-only test; opcheck variants only skip on CPU samples and add no op coverage (T191384137)"
+    )
     def test_permute_indices_scripted_with_none_weights(
         self,
     ) -> None:
@@ -362,6 +368,9 @@ def test_permute_indices_with_repeats(
     )
     @settings(verbosity=Verbosity.verbose, max_examples=100, deadline=None)
     @unittest.skipIf(*gpu_unavailable)
+    @optests.dontGenerateOpCheckTests(
+        "GPU-only test; opcheck variants only skip on CPU samples and add no op coverage (T191384137)"
+    )
     def test_permute_1D_sparse_data_vec(
         self,
         num_segments: int,
@@ -511,6 +520,9 @@ def test_permute_1D_sparse_data_vec(
     )
     @settings(verbosity=Verbosity.verbose, max_examples=50, deadline=None)
     @unittest.skipIf(*gpu_unavailable)
+    @optests.dontGenerateOpCheckTests(
+        "GPU-only test; opcheck variants only skip on CPU samples and add no op coverage (T191384137)"
+    )
     def test_permute_1D_sparse_data_vec_2d_weights(
         self,
         num_segments: int,
@@ -650,6 +662,9 @@ def permute_1d_ref(
     )
     @settings(verbosity=Verbosity.verbose, max_examples=10, deadline=None)
     @unittest.skipIf(*gpu_unavailable)
+    @optests.dontGenerateOpCheckTests(
+        "GPU-only test; opcheck variants only skip on CPU samples and add no op coverage (T191384137)"
+    )
     def test_permute_2D_indices_vec_remainder(
         self,
         long_index: bool,
@@ -726,6 +741,9 @@ def test_permute_2D_indices_vec_remainder(
     )
     @settings(verbosity=Verbosity.verbose, max_examples=10, deadline=None)
     @unittest.skipIf(*gpu_unavailable)
+    @optests.dontGenerateOpCheckTests(
+        "GPU-only test; opcheck variants only skip on CPU samples and add no op coverage (T191384137)"
+    )
     def test_permute_2D_indices_large_segments(
         self,
         index_dtype: torch.dtype,

fbgemm_gpu/test/sparse/reorder_batched_test.py

@@ -20,10 +20,10 @@
 
 if open_source:
     # pyre-ignore[21]
-    from test_utils import gpu_unavailable, skipIfRocm
+    from test_utils import gpu_unavailable, optests, skipIfRocm
 else:
     import fbgemm_gpu.sparse_ops  # noqa: F401, E402
-    from fbgemm_gpu.test.test_utils import gpu_unavailable, skipIfRocm
+    from fbgemm_gpu.test.test_utils import gpu_unavailable, optests, skipIfRocm
 
 
 class ReorderBatchedTest(unittest.TestCase):
@@ -37,6 +37,9 @@ class ReorderBatchedTest(unittest.TestCase):
         broadcast_lengths=st.booleans(),
     )
     @settings(verbosity=Verbosity.verbose, max_examples=20, deadline=None)
+    @optests.dontGenerateOpCheckTests(
+        "GPU-only test; opcheck variants only skip on CPU samples; op covered by *_cpu twin (T191384137)"
+    )
     def test_reorder_batched_ad_lengths(
         self,
         B: int,
@@ -135,6 +138,9 @@ def test_reorder_batched_ad_lengths_cpu(
         broadcast_indices=st.booleans(),
     )
     @settings(verbosity=Verbosity.verbose, max_examples=20, deadline=None)
+    @optests.dontGenerateOpCheckTests(
+        "GPU-only test; opcheck variants only skip on CPU samples; op covered by *_cpu twin (T191384137)"
+    )
     def test_reorder_batched_ad_indices(
         self,
         B: int,
@@ -511,6 +517,9 @@ def test_reorder_batched_sequence_embeddings_cpu(
         ),
     )
     @settings(verbosity=Verbosity.verbose, max_examples=40, deadline=None)
+    @optests.dontGenerateOpCheckTests(
+        "GPU-only test; opcheck variants only skip on CPU samples; op covered by *_cpu twin (T191384137)"
+    )
     def test_reorder_batched_sequence_embeddings(
         self,
         B: int,