feat: upgrade to transformers v5

build/accelerate_launch.py

@@ -116,6 +116,16 @@ def main():
             return_code = INTERNAL_ERROR_EXIT_CODE
             write_termination_log(f"Unhandled exception during training. {e}")
         sys.exit(return_code)
+    except Exception as e:  # pylint: disable=broad-except
+        logging.error(traceback.format_exc())
+        # v5: torch.distributed raises ChildFailedError with per-rank exit codes
+        # Check if the root cause was a user error
+        if hasattr(e, 'failures'):
+            root_codes = [f.exitcode for f in e.failures.values()]
+            if any(c == USER_ERROR_EXIT_CODE for c in root_codes):
+                sys.exit(USER_ERROR_EXIT_CODE)
+        write_termination_log(f"Unhandled exception during training. {e}")
+        sys.exit(INTERNAL_ERROR_EXIT_CODE)
     except Exception as e:  # pylint: disable=broad-except
         logging.error(traceback.format_exc())
         write_termination_log(f"Unhandled exception during training. {e}")

pyproject.toml

@@ -28,18 +28,19 @@ classifiers=[
 dependencies = [
 "numpy>=1.26.4,<2.2.0",
 "accelerate>=1.9.0,<2.0.0",
-"transformers>=4.55.0,<=4.55.4",
-"torch>2.7.0,<2.9.0",
-"torchvision<0.24",
+"transformers==5.2.0",
+"torch==2.10.0",
+"torchvision>=0.25.0",
 "sentencepiece>=0.1.99,<0.3",
-"tokenizers<=0.22",
+"tokenizers==0.22.2",
 "tqdm>=4.66.2,<5.0",
 "trl>=0.19.1,<0.20.0",
-"peft>=0.18.0,< 0.19.0",
+"peft>=0.18.1,<0.19.0",
 "datasets>=4.0.0,<5.0.0",
 "simpleeval>=0.9.13,<2.0",
 "pillow>=12.1.1",
-"kernels<=0.9.0",
+"kernels==0.12.1",
+"huggingface_hub>=1.3.0"
 ]
 
 [project.optional-dependencies]

tests/build/test_launch_script.py

@@ -51,7 +51,7 @@
     "warmup_ratio": 0.03,
     "lr_scheduler_type": "cosine",
     "logging_steps": 1,
-    "include_tokens_per_second": True,
+    "include_num_input_tokens_seen": True,
     "packing": False,
     "response_template": "\n### Label:",
     "dataset_text_field": "output",

tests/test_sft_trainer.py

@@ -124,7 +124,8 @@
     warmup_ratio=0.03,
     lr_scheduler_type="cosine",
     logging_steps=1,
-    include_tokens_per_second=True,
+    # include_tokens_per_second=True,
+    include_num_input_tokens_seen=True,
     packing=False,
     max_seq_length=4096,
     save_strategy="epoch",
@@ -140,7 +141,8 @@
     warmup_ratio=0.03,
     lr_scheduler_type="cosine",
     logging_steps=1,
-    include_tokens_per_second=True,
+    # include_tokens_per_second=True,
+    include_num_input_tokens_seen=True,
     packing=False,
     max_seq_length=4096,
     save_strategy="epoch",

tests/utils/test_embedding_resize.py

@@ -21,7 +21,7 @@
 # Third Party
 from transformers import (
     AutoModelForCausalLM,
-    AutoModelForVision2Seq,
+    AutoModelForImageTextToText, #AutoModelForVision2Seq was renamed to this in transformers v5
     AutoProcessor,
     AutoTokenizer,
 )
@@ -128,16 +128,16 @@ def test_special_tokens_before_and_after():
     model = AutoModelForCausalLM.from_pretrained(MODEL_NAME)
 
     input_tokenizer_len = len(tokenizer.get_vocab())
-    addn_spl_tokens_before = tokenizer.special_tokens_map.get(
-        "additional_special_tokens"
-    )
+    addn_spl_tokens_before = list(tokenizer.extra_special_tokens) # "additional_special_tokens" was renamed to extra_special_tokens in transformers v5
     assert (
         len(addn_spl_tokens_before) > 0
     ), "this test needs tokenizer special tokens to not be empty before testing"
 
     special_tokens_dict = {"sep_token": "<SEP>", "pad_token": "<PAD>"}
     addn_spl_tokens_added = ["<NotSeenTokenA>", "<NotSeenTokenB>", "<NotSeenTokenC>"]
-    special_tokens_dict["additional_special_tokens"] = addn_spl_tokens_added
+    # for transformers v5: merge existing extra_special_tokens with new ones to prevent replacement
+    special_tokens_dict["additional_special_tokens"] = list(tokenizer.extra_special_tokens) + addn_spl_tokens_added # "additional_special_tokens" was renamed to extra_special_tokens in transformers v5
+
 
     resize_result = tokenizer_and_embedding_resize(
         special_tokens_dict=special_tokens_dict,
@@ -150,9 +150,7 @@ def test_special_tokens_before_and_after():
     addn_spl_tokens_before.extend(addn_spl_tokens_added)
     expected_addn_special_tokens = addn_spl_tokens_before
     expected_embedding_size = input_tokenizer_len + len(addn_spl_tokens_added) + 2
-    addn_spl_tokens_after = tokenizer.special_tokens_map.get(
-        "additional_special_tokens"
-    )
+    addn_spl_tokens_after = list(tokenizer.extra_special_tokens) # "additional_special_tokens" was renamed to extra_special_tokens in transformers v5
 
     assert "<SEP>" in tokenizer.get_vocab()
     assert "<PAD>" in tokenizer.get_vocab()
@@ -212,7 +210,7 @@ def test_resize_with_multiple_of():
 
 
 def test_resize_llama_vision_model():
-    model = AutoModelForVision2Seq.from_pretrained(TINY_LLAMA_VISION_MODEL_NAME)
+    model = AutoModelForImageTextToText.from_pretrained(TINY_LLAMA_VISION_MODEL_NAME) # AutoModelForVision2Seq was renamed to AutoModelForImageTextToText in transformers v5
     processor = AutoProcessor.from_pretrained(TINY_LLAMA_VISION_MODEL_NAME)
     tokenizer = processor.tokenizer
 

tuning/data/tokenizer_utils.py

@@ -44,21 +44,42 @@ def get_special_tokens_dict(
 
     special_tokens_dict = {}
     if not tokenizer_name_or_path:
-        # TODO: understand if we need to hardcode these here or just use defaults in model
-        if isinstance(
-            tokenizer, (transformers.LlamaTokenizer, transformers.LlamaTokenizerFast)
-        ):
+        # # TODO: understand if we need to hardcode these here or just use defaults in model
+        # if isinstance(
+        #     tokenizer, (transformers.LlamaTokenizer, transformers.LlamaTokenizerFast)
+        # ):
+        llama_classes = tuple(
+            cls for cls in [
+                getattr(transformers, "LlamaTokenizer", None),
+                getattr(transformers, "LlamaTokenizerFast", None),
+            ] if cls is not None
+        )
+        is_llama_tokenizer = (
+            (bool(llama_classes) and isinstance(tokenizer, llama_classes))
+            or "llama" in (getattr(tokenizer, "name_or_path", "") or "").lower()
+        )
+
+        gpt_neox_classes = tuple(
+            cls for cls in [
+                getattr(transformers, "GPTNeoXTokenizerFast", None),
+                getattr(transformers, "GPTNeoXTokenizer", None),
+            ] if cls is not None
+        )
+
+        if is_llama_tokenizer:
             special_tokens_dict["bos_token"] = "<s>"
             special_tokens_dict["eos_token"] = "</s>"
             special_tokens_dict["unk_token"] = "<unk>"
             special_tokens_dict["pad_token"] = "<pad>"
         elif isinstance(
-            tokenizer, (transformers.GPT2Tokenizer, transformers.GPTNeoXTokenizerFast)
+            # tokenizer, (transformers.GPT2Tokenizer, transformers.GPTNeoXTokenizerFast)
+            tokenizer, (transformers.GPT2Tokenizer, *gpt_neox_classes)
         ):
             special_tokens_dict["pad_token"] = "<pad>"
 
         # Add special tokens only when a custom tokenizer is not passed
-        if tokenizer.pad_token is None:
+        # if tokenizer.pad_token is None:
+        if tokenizer.pad_token is None or "pad_token" in special_tokens_dict:
             logger.warning("PAD token set to default, missing in tokenizer")
             special_tokens_dict["pad_token"] = configs.DEFAULT_PAD_TOKEN
         if tokenizer.eos_token is None:
@@ -102,7 +123,8 @@ def tokenizer_and_embedding_resize(
         dict: Metadata on number of added tokens.
     """
     num_new_tokens = tokenizer.add_special_tokens(
-        special_tokens_dict=special_tokens_dict, replace_additional_special_tokens=False
+        special_tokens_dict=special_tokens_dict, 
+        # replace_additional_special_tokens=False
     )
     embedding_size = int(multiple_of * math.ceil(len(tokenizer) / multiple_of))
     num_new_tokens = num_new_tokens + embedding_size - len(tokenizer)
@@ -119,8 +141,11 @@ def tokenizer_and_embedding_resize(
         model.set_input_embeddings(resized_input_embeddings)
 
         # Resize vocab size when embeddings updated for Mllama models
-        if model.language_model.vocab_size != embedding_size:
-            model.language_model.vocab_size = embedding_size
+        # if model.language_model.vocab_size != embedding_size:
+        #     model.language_model.vocab_size = embedding_size
+        if model.model.vocab_size != embedding_size:
+            model.model.vocab_size = embedding_size
+
     else:
         model.resize_token_embeddings(embedding_size)
 

tuning/sft_trainer.py

@@ -30,7 +30,7 @@
 from torch.cuda import OutOfMemoryError
 from transformers import (
     AutoModelForCausalLM,
-    AutoModelForVision2Seq,
+    AutoModelForImageTextToText, # AutoModelForVision2Seq was renamed in transformers v5
     AutoProcessor,
     AutoTokenizer,
     TrainerCallback,
@@ -292,7 +292,8 @@ def train(
                 )
             )
             # try to load model as a vision model
-            model = AutoModelForVision2Seq.from_pretrained(
+            # in transformers v5, AutoModelForVision2Seq was renamed to AutoModelForImageTextToText
+            model = AutoModelForImageTextToText.from_pretrained(
                 model_args.model_name_or_path, **model_kwargs
             )
             try: