feat: add mamba kernels for benchmark

:

Signed-off-by: Mehant Kammakomati <mehant.kammakomati2@ibm.com>

Author: kmehant

scripts/benchmarks/scenarios-moe.yaml

@@ -76,7 +76,7 @@ scenarios:
             learning_rate: 5e-5
             torch_dtype: bfloat16
             gradient_accumulation_steps: null
-            per_device_train_batch_size: 2
+            per_device_train_batch_size: 8
             logging_steps: 1
             packing: False
             adam_epsilon: 1e-8

tox.ini

@@ -34,6 +34,9 @@ commands =
     # some models need this for tokenizers
     pip install protobuf
 
+    # for mamba based models
+    pip install --no-build-isolation mamba_ssm[causal-conv1d]>=2.0.0
+
     # install the plugins for test
     # NOTE: when there are more plugins install here
     python -m fms_acceleration.cli install -e {toxinidir}/plugins/accelerated-peft