Remove the eager import

Signed-off-by: Jingyu Xin <jingyux@nvidia.com>

Author: jingyu-ml

modelopt/torch/sparsity/attention_sparsity/methods/flash_skip_softmax.py

@@ -390,13 +390,6 @@ def sparse_softmax(input, dim=-1, *args, **kwargs):
         set_skip_softmax_context(True)
         stack.callback(set_skip_softmax_context, False)
 
-        try:
-            from ..kernels.diffusers_eager_attention import get_skip_softmax_attention_backend
-
-            stack.enter_context(get_skip_softmax_attention_backend())
-        except (ImportError, RuntimeError):
-            pass
-
         stack.enter_context(replace_function(torch.nn.functional, "softmax", sparse_softmax))
         return stack