Merge remote-tracking branch 'upstream/main' into vision-model

Author: Abhishek-TAMU

build/Dockerfile

@@ -154,11 +154,13 @@ RUN --mount=type=cache,target=/home/${USER}/.cache/pip,uid=${USER_UID} \
 # fms_acceleration_peft = PEFT-training, e.g., 4bit QLoRA
 # fms_acceleration_foak = Fused LoRA and triton kernels
 # fms_acceleration_aadp = Padding-Free Flash Attention Computation
+# fms_acceleration_moe = Parallelized Mixture of Experts
 RUN if [[ "${ENABLE_FMS_ACCELERATION}" == "true" ]]; then \
         python -m pip install --user "$(head bdist_name)[fms-accel]"; \
         python -m fms_acceleration.cli install fms_acceleration_peft; \
         python -m fms_acceleration.cli install fms_acceleration_foak; \
         python -m fms_acceleration.cli install fms_acceleration_aadp; \
+        python -m fms_acceleration.cli install fms_acceleration_moe; \
     fi
 
 RUN if [[ "${ENABLE_AIM}" == "true" ]]; then \

tuning/config/configs.py

@@ -35,7 +35,7 @@
 
 @dataclass
 class ModelArguments:
-    model_name_or_path: Optional[str] = field(default="facebook/opt-125m")
+    model_name_or_path: Optional[str] = field(default=None)
     use_flash_attn: bool = field(
         default=True,
         metadata={"help": "Use Flash attention v2 from transformers, default is True"},

tuning/sft_trainer.py

@@ -134,6 +134,14 @@ def train(
     train_args, logger = set_log_level(train_args, "sft_trainer_train")
 
     # Validate parameters
+    if (not isinstance(model_args.model_name_or_path, str)) or (
+        model_args.model_name_or_path == ""
+    ):
+        raise ValueError(
+            "model_name_or_path has to be a string containing a valid"
+            + " HuggingFace Hub model name or the path to a checkpoint folder"
+        )
+
     if (not isinstance(train_args.num_train_epochs, (float, int))) or (
         train_args.num_train_epochs <= 0
     ):