feat: upgrade to transformers v5 (#659)

* accommodate renaming of AutoModelForVision2Seq to AutoModelForImageTextToText

Signed-off-by: Yash Mehan <yashmehan@gmail.com>

* accomodate renaming of include_tokens_per_second to include_num_input_tokens_seen

Signed-off-by: Yash Mehan <yashmehan@gmail.com>

* accommodate mllama model.language_model issue and gptneoxtokenizer and llama tokenizer

Signed-off-by: Yash Mehan <yashmehan@gmail.com>

* replace addition_special_tokens with extra_special_tokens

Signed-off-by: Yash Mehan <yashmehan@gmail.com>

* changed library versions in pyproject.toml

Signed-off-by: Yash Mehan <yashmehan@gmail.com>

* fix pyproject toml after the rebase

Signed-off-by: Yash Mehan <yashmehan@gmail.com>

* add more handing for the filenotfound exception handling in multi rank case

Signed-off-by: Yash Mehan <yashmehan@gmail.com>

* merge existing extra_special_tokens with new ones to prevent replacement

Signed-off-by: Yash Mehan <yashmehan@gmail.com>

* remove the old commented lines, only letting the new renamed lines stay, adding justification for what was renamed

Signed-off-by: Yash Mehan <yashmehan@gmail.com>

* remove the old commented lines

Signed-off-by: Yash Mehan <yashmehan@gmail.com>

* remove the old commented lines

Signed-off-by: Yash Mehan <yashmehan@gmail.com>

* complete housekeeping and removing old comments and repetitive comments

Signed-off-by: Yash Mehan <yashmehan@gmail.com>

* put upperbounds on dependencies in pyproject

Signed-off-by: Yash Mehan <yashmehan@gmail.com>

* fix input_ids

Signed-off-by: Yash Mehan <yashmehan@gmail.com>

* linting and formatting

Signed-off-by: Yash Mehan <yashmehan@gmail.com>

* remove multiple excepts

Signed-off-by: Yash Mehan <yashmehan@gmail.com>

* add messages control to remove in pylintrc

Signed-off-by: Yash Mehan <yashmehan@gmail.com>

* upgrade support to trl 0.27+

Signed-off-by: Yash Mehan <yashmehan@gmail.com>

---------

Signed-off-by: Yash Mehan <yashmehan@gmail.com>
Signed-off-by: Yash Mehan <71321431+yash4242@users.noreply.github.com>
Co-authored-by: Dushyant Behl <dushyantbehl@users.noreply.github.com>

Author: yash4242

.pylintrc

@@ -447,7 +447,10 @@ disable=raw-checker-failed,
         duplicate-code,
         unbalanced-tuple-unpacking,
         unspecified-encoding,
-        too-many-lines
+        too-many-lines,
+        no-name-in-module,
+        unexpected-keyword-arg,
+        unused-argument
 
 # Enable the message, report, category or checker with the given id(s). You can
 # either give multiple identifier separated by comma (,) or put this option

build/accelerate_launch.py

@@ -110,14 +110,19 @@ def main():
         # message to termination log.
         logging.error(traceback.format_exc())
         # The exit code that sft_trainer.py threw is captured in e.returncode
-
         return_code = e.returncode
         if return_code not in [INTERNAL_ERROR_EXIT_CODE, USER_ERROR_EXIT_CODE]:
             return_code = INTERNAL_ERROR_EXIT_CODE
             write_termination_log(f"Unhandled exception during training. {e}")
         sys.exit(return_code)
     except Exception as e:  # pylint: disable=broad-except
         logging.error(traceback.format_exc())
+        # v5: torch.distributed raises ChildFailedError with per-rank exit codes
+        # Check if the root cause was a user error
+        if hasattr(e, "failures"):
+            root_codes = [f.exitcode for f in e.failures.values()]
+            if any(c == USER_ERROR_EXIT_CODE for c in root_codes):
+                sys.exit(USER_ERROR_EXIT_CODE)
         write_termination_log(f"Unhandled exception during training. {e}")
         sys.exit(INTERNAL_ERROR_EXIT_CODE)
 

pyproject.toml

@@ -28,18 +28,19 @@ classifiers=[
 dependencies = [
 "numpy>=1.26.4,<2.2.0",
 "accelerate>=1.9.0,<2.0.0",
-"transformers>=4.55.0,<=4.55.4",
-"torch>2.7.0,<2.9.0",
-"torchvision<0.24",
+"transformers>=5.2.0,<5.3.0",
+"torch>2.7.0,<=2.9.0",
+"torchvision<=0.24.0",
 "sentencepiece>=0.1.99,<0.3",
-"tokenizers<=0.22",
+"tokenizers<=0.23.0",
 "tqdm>=4.66.2,<5.0",
-"trl>=0.19.1,<0.20.0",
-"peft>=0.18.0,< 0.19.0",
+"trl>=0.27.0,<0.29.0",
+"peft>=0.18.1,<0.19.0",
 "datasets>=4.0.0,<5.0.0",
 "simpleeval>=0.9.13,<2.0",
 "pillow>=12.1.1",
-"kernels<=0.9.0",
+"kernels>=0.12.1,<0.13.0",
+"huggingface_hub>=1.3.0,<1.4.0",
 ]
 
 [project.optional-dependencies]

tests/build/test_launch_script.py

@@ -51,7 +51,7 @@
     "warmup_ratio": 0.03,
     "lr_scheduler_type": "cosine",
     "logging_steps": 1,
-    "include_tokens_per_second": True,
+    "include_num_input_tokens_seen": True,
     "packing": False,
     "response_template": "\n### Label:",
     "dataset_text_field": "output",

tests/data/test_data_preprocessing.py

@@ -22,7 +22,6 @@
 from datasets import Dataset, DatasetDict, IterableDataset
 from PIL import Image
 from transformers import AutoProcessor, AutoTokenizer, DataCollatorForSeq2Seq
-from trl import DataCollatorForCompletionOnlyLM
 import datasets
 import numpy as np
 import pyarrow
@@ -69,7 +68,7 @@
 # Local
 from tuning.config import configs
 from tuning.config.acceleration_configs import AttentionAndDistributedPackingConfig
-from tuning.data.collators import VisionDataCollator
+from tuning.data.collators import DataCollatorForCompletionOnlyLM, VisionDataCollator
 from tuning.data.data_config import (
     DataHandlerConfig,
     DataPreProcessorConfig,

tests/test_sft_trainer.py

@@ -124,7 +124,7 @@
     warmup_ratio=0.03,
     lr_scheduler_type="cosine",
     logging_steps=1,
-    include_tokens_per_second=True,
+    include_num_input_tokens_seen=True,
     packing=False,
     max_seq_length=4096,
     save_strategy="epoch",
@@ -140,7 +140,7 @@
     warmup_ratio=0.03,
     lr_scheduler_type="cosine",
     logging_steps=1,
-    include_tokens_per_second=True,
+    include_num_input_tokens_seen=True,
     packing=False,
     max_seq_length=4096,
     save_strategy="epoch",

tests/utils/test_embedding_resize.py

@@ -20,11 +20,9 @@
 
 # Third Party
 from transformers import (
-    AutoModelForCausalLM,
-    AutoModelForVision2Seq,
-    AutoProcessor,
-    AutoTokenizer,
+    AutoModelForImageTextToText,  # AutoModelForVision2Seq was renamed to this in transformers v5
 )
+from transformers import AutoModelForCausalLM, AutoProcessor, AutoTokenizer
 import torch
 
 # First Party
@@ -128,16 +126,17 @@ def test_special_tokens_before_and_after():
     model = AutoModelForCausalLM.from_pretrained(MODEL_NAME)
 
     input_tokenizer_len = len(tokenizer.get_vocab())
-    addn_spl_tokens_before = tokenizer.special_tokens_map.get(
-        "additional_special_tokens"
-    )
+    addn_spl_tokens_before = list(tokenizer.extra_special_tokens)
     assert (
         len(addn_spl_tokens_before) > 0
     ), "this test needs tokenizer special tokens to not be empty before testing"
 
     special_tokens_dict = {"sep_token": "<SEP>", "pad_token": "<PAD>"}
     addn_spl_tokens_added = ["<NotSeenTokenA>", "<NotSeenTokenB>", "<NotSeenTokenC>"]
-    special_tokens_dict["additional_special_tokens"] = addn_spl_tokens_added
+    # for transformers v5: merge existing extra_special_tokens with new ones to prevent replacement
+    special_tokens_dict["additional_special_tokens"] = (
+        list(tokenizer.extra_special_tokens) + addn_spl_tokens_added
+    )
 
     resize_result = tokenizer_and_embedding_resize(
         special_tokens_dict=special_tokens_dict,
@@ -150,9 +149,7 @@ def test_special_tokens_before_and_after():
     addn_spl_tokens_before.extend(addn_spl_tokens_added)
     expected_addn_special_tokens = addn_spl_tokens_before
     expected_embedding_size = input_tokenizer_len + len(addn_spl_tokens_added) + 2
-    addn_spl_tokens_after = tokenizer.special_tokens_map.get(
-        "additional_special_tokens"
-    )
+    addn_spl_tokens_after = list(tokenizer.extra_special_tokens)
 
     assert "<SEP>" in tokenizer.get_vocab()
     assert "<PAD>" in tokenizer.get_vocab()
@@ -212,7 +209,9 @@ def test_resize_with_multiple_of():
 
 
 def test_resize_llama_vision_model():
-    model = AutoModelForVision2Seq.from_pretrained(TINY_LLAMA_VISION_MODEL_NAME)
+    model = AutoModelForImageTextToText.from_pretrained(
+        TINY_LLAMA_VISION_MODEL_NAME
+    )  # AutoModelForVision2Seq was renamed to AutoModelForImageTextToText in transformers v5
     processor = AutoProcessor.from_pretrained(TINY_LLAMA_VISION_MODEL_NAME)
     tokenizer = processor.tokenizer