[None][feat] Add AD custom model for Llama 4 family (#237)

Add a lean, prefill-only custom model for the Llama 4 family
(Scout-17B-16E, Maverick-17B-128E) using AutoDeploy canonical ops.
Replaces the previous MoE/vision patches with a self-contained
implementation.

Key features:
- GQA with complex-frequency RoPE (torch_rope_with_complex_freqs)
- NoPE layers with attention temperature tuning
- L2 QK normalization on RoPE layers (mean-based, plain PyTorch)
- MoE with stacked expert weights (bmm) matching HF checkpoint format;
  AD MatchBmmMoePattern transform handles conversion at deployment
- Multimodal wrapper (ForConditionalGeneration) for weight compat
- Fix multimodal processor chat template for text-only prompts

Includes hierarchical unit tests (block, layer, full model, export)
covering RoPE/NoPE layers, MoE/dense layers, and dynamic shapes.

Signed-off-by: Lucas Liebenwein <lliebenwein@nvidia.com>
Signed-off-by: Lucas Liebenwein <11156568+lucaslie@users.noreply.github.com>

Author: lucaslie

tensorrt_llm/_torch/auto_deploy/llm.py

@@ -46,17 +46,25 @@ def __call__(
             # multi_modal_data should not be present in the messages field
             assert "multi_modal_data" not in inputs, f"unexpected multi_modal_data key in {inputs=}"
 
+            # Normalize message content to list-of-dicts format for multimodal
+            # processors (e.g., Llama4) that expect {"type": "text", "text": "..."}
+            # instead of plain strings when tokenize=True.
+            messages = inputs["messages"]
+            for msg in messages:
+                if isinstance(msg.get("content"), str):
+                    msg["content"] = [{"type": "text", "text": msg["content"]}]
+
             # TODO: we don't really need this but it makes for a good sanity check. Consider
             # removing this in the future if we need to speed things up.
             prompt = self.processor.apply_chat_template(
-                inputs["messages"],
+                messages,
                 add_generation_prompt=True,
                 tokenize=False,
             )
             inputs["prompt"] = prompt
 
             all_args = self.processor.apply_chat_template(
-                inputs["messages"],
+                messages,
                 add_generation_prompt=True,
                 tokenize=True,
                 return_dict=True,

tensorrt_llm/_torch/auto_deploy/models/custom/__init__.py

@@ -15,6 +15,7 @@
 from .modeling_internlm3 import InternLM3ForCausalLM
 from .modeling_kimi_k2 import KimiK2ForCausalLM, KimiK25ForConditionalGeneration
 from .modeling_llama3 import Llama3ForCausalLM
+from .modeling_llama4 import Llama4ForCausalLM, Llama4ForConditionalGeneration
 from .modeling_minimax_m2 import MiniMaxM2ForCausalLM
 from .modeling_mistral import MistralForCausalLM
 from .modeling_mistral3 import Mistral3ForConditionalGeneration, Mistral3TextForCausalLM
@@ -49,6 +50,8 @@
     "KimiK2ForCausalLM",
     "KimiK25ForConditionalGeneration",
     "Llama3ForCausalLM",
+    "Llama4ForCausalLM",
+    "Llama4ForConditionalGeneration",
     "MiniMaxM2ForCausalLM",
     "MistralForCausalLM",
     "Mistral3ForConditionalGeneration",