Raise on quantization_config + already-instantiated model in DPOTrainer/ KTOTrainer (#6312)

Author: qgallouedec

trl/trainer/dpo_trainer.py

@@ -573,9 +573,10 @@ def __init__(
                     "The `model_init_kwargs` will be ignored."
                 )
             if quantization_config is not None:
-                logger.warning(
-                    "You passed `quantization_config` to the trainer, but your model is already instantiated. The "
-                    "`quantization_config` will be ignored."
+                raise ValueError(
+                    "You passed `quantization_config` to the trainer, but your model is already instantiated. "
+                    "Quantization can only be applied when the model is loaded from a model identifier (`str`). "
+                    "Either pass the model as a model identifier, or omit `quantization_config`."
                 )
         # Non-quantized models do not have the `is_loaded_in_{8,4}bit` attributes, whereas quantized models do
         _is_quantized_model = getattr(model, "is_loaded_in_4bit", False) or getattr(model, "is_loaded_in_8bit", False)

trl/trainer/kto_trainer.py

@@ -624,9 +624,10 @@ def __init__(
                     "The `model_init_kwargs` will be ignored."
                 )
             if quantization_config is not None:
-                logger.warning(
-                    "You passed `quantization_config` to the trainer, but your model is already instantiated. The "
-                    "`quantization_config` will be ignored."
+                raise ValueError(
+                    "You passed `quantization_config` to the trainer, but your model is already instantiated. "
+                    "Quantization can only be applied when the model is loaded from a model identifier (`str`). "
+                    "Either pass the model as a model identifier, or omit `quantization_config`."
                 )
         # Non-quantized models do not have the `is_loaded_in_{8,4}bit` attributes, whereas quantized models do
         _is_quantized_model = getattr(model, "is_loaded_in_4bit", False) or getattr(model, "is_loaded_in_8bit", False)