build(deps): upgrade Transformers to 5.12.1 (#4928)

Signed-off-by: Chen Cui <chcui@nvidia.com>

Author: cuichenx

.github/workflows/cache-hf-model.yml

@@ -39,8 +39,23 @@ jobs:
 
       - name: Install dependencies
         run: |
-          TRANSFORMERS_VERSION=$(grep -E 'transformers[[:space:]]*[=><!~]+' ${GITHUB_WORKSPACE}/pyproject.toml | sed -E 's/.*transformers[[:space:]]*([=><!~]+)[[:space:]]*"?([0-9a-zA-Z\.\-\*]+)"?.*/\1\2/')
-          pip install torch "transformers${TRANSFORMERS_VERSION}" "huggingface_hub[cli]"
+          TRANSFORMERS_REQUIREMENT=$(
+            python - "${GITHUB_WORKSPACE}/pyproject.toml" <<'PY'
+          import re
+          import sys
+          import tomllib
+
+          with open(sys.argv[1], "rb") as pyproject:
+              dependencies = tomllib.load(pyproject)["project"]["dependencies"]
+
+          pattern = re.compile(r"^transformers(?:$|[\s\[<>=!~;@])", re.IGNORECASE)
+          requirement = next((item for item in dependencies if pattern.match(item)), None)
+          if requirement is None:
+              raise SystemExit("Transformers requirement not found in pyproject.toml")
+          sys.stdout.write(requirement)
+          PY
+          )
+          pip install torch "${TRANSFORMERS_REQUIREMENT}" "huggingface_hub[cli]"
 
       - name: Log in to HuggingFace
         env:

pyproject.toml

@@ -71,7 +71,7 @@ classifiers = [
     "Topic :: Utilities",
 ]
 dependencies = [
-    "transformers>=5.8.1,<5.9.0",
+    "transformers>=5.8,<=5.12.1",
     "mistral-common>=1.10.0",
     "peft>=0.18.1",
     "datasets>=2.20.0",

src/megatron/bridge/models/qwen_audio/modeling_qwen2_audio.py

@@ -41,19 +41,31 @@
 # Import HuggingFace Qwen2Audio model classes with fallback
 try:
     from transformers import Qwen2AudioForConditionalGeneration
+    from transformers.models.qwen2_audio import modeling_qwen2_audio as hf_modeling_qwen2_audio
     from transformers.models.qwen2_audio.modeling_qwen2_audio import (
         Qwen2AudioEncoder,
         Qwen2AudioMultiModalProjector,
     )
 
+    HFQwen2AudioModel = getattr(hf_modeling_qwen2_audio, "Qwen2AudioModel", None)
     HAS_QWEN2_AUDIO = True
 except ImportError:
     Qwen2AudioForConditionalGeneration = None
     Qwen2AudioEncoder = None
     Qwen2AudioMultiModalProjector = None
+    HFQwen2AudioModel = None
     HAS_QWEN2_AUDIO = False
 
 
+_QWEN2_AUDIO_MERGE_METHOD = None
+if HAS_QWEN2_AUDIO:
+    _QWEN2_AUDIO_MERGE_METHOD = getattr(
+        Qwen2AudioForConditionalGeneration, "_merge_input_ids_with_audio_features", None
+    )
+    if _QWEN2_AUDIO_MERGE_METHOD is None and HFQwen2AudioModel is not None:
+        _QWEN2_AUDIO_MERGE_METHOD = getattr(HFQwen2AudioModel, "_merge_input_ids_with_audio_features", None)
+
+
 class Qwen2AudioModel(MegatronModule):
     """
     Qwen2-Audio Model wrapper for Megatron.
@@ -141,11 +153,9 @@ def __init__(
         self.share_embeddings_and_output_weights = config.share_embeddings_and_output_weights
         self.shared_embedding_or_output_weight = self.language_model.shared_embedding_or_output_weight
 
-        # Monkey-patch methods from HuggingFace Qwen2AudioForConditionalGeneration
-        if HAS_QWEN2_AUDIO and Qwen2AudioForConditionalGeneration is not None:
-            self._merge_input_ids_with_audio_features = types.MethodType(
-                Qwen2AudioForConditionalGeneration._merge_input_ids_with_audio_features, self
-            )
+        # Bind the merge method from the Hugging Face Qwen2-Audio implementation.
+        if _QWEN2_AUDIO_MERGE_METHOD is not None:
+            self._merge_input_ids_with_audio_features = types.MethodType(_QWEN2_AUDIO_MERGE_METHOD, self)
 
         # Store audio token id from config
         self.audio_token_id = getattr(config, "audio_token_id", 151646)

tests/functional_tests/test_groups/models/ministral3/test_ministral3_conversion.py

@@ -18,6 +18,10 @@
 
 import pytest
 import torch
+from tokenizers import Tokenizer
+from tokenizers.models import WordLevel
+from tokenizers.pre_tokenizers import Whitespace
+from transformers import PreTrainedTokenizerFast
 
 
 # Ministral 3 toy model configuration based on typical Ministral 3 structure
@@ -105,19 +109,19 @@ def ministral3_toy_model_path(self, tmp_path_factory):
             print(f"Before save - {name}: {param.dtype}")
             break  # Just check the first parameter
 
-        # Create minimal tokenizer files
-        tokenizer_config = {
-            "tokenizer_class": "LlamaTokenizer",
-            "vocab_size": 32768,
-            "bos_token": "<s>",
-            "eos_token": "</s>",
-            "pad_token": "<pad>",
-            "unk_token": "<unk>",
-        }
-
         model_dir.mkdir(parents=True, exist_ok=True)
-        with open(model_dir / "tokenizer_config.json", "w") as f:
-            json.dump(tokenizer_config, f, indent=2)
+
+        # Save a valid tokenizer artifact so both Transformers 5.8 and 5.12 can reload it.
+        backend_tokenizer = Tokenizer(WordLevel({"<unk>": 0, "<s>": 1, "</s>": 2, "<pad>": 3}, unk_token="<unk>"))
+        backend_tokenizer.pre_tokenizer = Whitespace()
+        tokenizer = PreTrainedTokenizerFast(
+            tokenizer_object=backend_tokenizer,
+            unk_token="<unk>",
+            bos_token="<s>",
+            eos_token="</s>",
+            pad_token="<pad>",
+        )
+        tokenizer.save_pretrained(model_dir)
 
         # Save model and config to directory
         model.save_pretrained(model_dir, safe_serialization=True)

tests/unit_tests/models/qwen_audio/test_qwen2_audio_provider.py

@@ -13,6 +13,7 @@
 # limitations under the License.
 
 from megatron.bridge.models.qwen_audio import Qwen2AudioModelProvider
+from megatron.bridge.models.qwen_audio.modeling_qwen2_audio import _QWEN2_AUDIO_MERGE_METHOD
 
 
 class TestQwen2AudioModelProvider:
@@ -30,6 +31,10 @@ def test_initialization(self):
         assert provider.hidden_size == 4096
         assert provider.num_attention_heads == 32
 
+    def test_hf_audio_merge_method_is_available(self):
+        """Test that the installed Transformers version exposes the audio merge helper."""
+        assert _QWEN2_AUDIO_MERGE_METHOD is not None
+
     def test_audio_specific_defaults(self):
         """Test Qwen2AudioModelProvider audio-specific default configuration."""
         provider = Qwen2AudioModelProvider(