Add support for HRM-Text (#1695)

xenova · web-flow · commit bc9cf7400f4f · 2026-06-04T12:13:26.000-04:00
diff --git a/README.md b/README.md
@@ -317,6 +317,7 @@ To find compatible models on the Hub, select the "transformers.js" library tag i
 1. **[GroupViT](https://huggingface.co/docs/transformers/model_doc/groupvit)** (from UCSD, NVIDIA) released with the paper [GroupViT: Semantic Segmentation Emerges from Text Supervision](https://huggingface.co/papers/2202.11094) by Jiarui Xu, Shalini De Mello, Sifei Liu, Wonmin Byeon, Thomas Breuel, Jan Kautz, Xiaolong Wang.
 1. **[Helium](https://huggingface.co/docs/transformers/main/model_doc/helium)** (from the Kyutai Team) released with the blog post [Announcing Helium-1 Preview](https://kyutai.org/2025/01/13/helium.html) by the Kyutai Team.
 1. **[HerBERT](https://huggingface.co/docs/transformers/model_doc/herbert)** (from Allegro.pl, AGH University of Science and Technology) released with the paper [KLEJ: Comprehensive Benchmark for Polish Language Understanding](https://www.aclweb.org/anthology/2020.acl-main.111.pdf) by Piotr Rybak, Robert Mroczkowski, Janusz Tracz, Ireneusz Gawlik.
+1. **[HRM-Text](https://huggingface.co/docs/transformers/model_doc/hrm_text)** (from Sapient Intelligence) released with the paper [Hierarchical Reasoning Model](https://huggingface.co/papers/2506.21734) by Guan Wang, Jin Li, Yuhao Sun, Xing Chen, Changling Liu, Yue Wu, Meng Lu, Sen Song, Yasin Abbasi Yadkori.
 1. **[Hiera](https://huggingface.co/docs/transformers/model_doc/hiera)** (from Meta) released with the paper [Hiera: A Hierarchical Vision Transformer without the Bells-and-Whistles](https://huggingface.co/papers/2306.00989) by Chaitanya Ryali, Yuan-Ting Hu, Daniel Bolya, Chen Wei, Haoqi Fan, Po-Yao Huang, Vaibhav Aggarwal, Arkabandhu Chowdhury, Omid Poursaeed, Judy Hoffman, Jitendra Malik, Yanghao Li, Christoph Feichtenhofer.
 1. **[Hubert](https://huggingface.co/docs/transformers/model_doc/hubert)** (from Facebook) released with the paper [HuBERT: Self-Supervised Speech Representation Learning by Masked Prediction of Hidden Units](https://huggingface.co/papers/2106.07447) by Wei-Ning Hsu, Benjamin Bolte, Yao-Hung Hubert Tsai, Kushal Lakhotia, Ruslan Salakhutdinov, Abdelrahman Mohamed.
 1. **[HunYuanDenseV1](https://huggingface.co/docs/transformers/model_doc/hunyuan_v1_dense)** (from Tencent) released with the paper [Hunyuan-Large: An Open-Source MoE Model with 52 Billion Activated Parameters by Tencent](https://huggingface.co/papers/2411.02265) by the Tencent team.
diff --git a/packages/transformers/docs/snippets/5_supported-models.snippet b/packages/transformers/docs/snippets/5_supported-models.snippet
@@ -78,6 +78,7 @@
 1. **[GroupViT](https://huggingface.co/docs/transformers/model_doc/groupvit)** (from UCSD, NVIDIA) released with the paper [GroupViT: Semantic Segmentation Emerges from Text Supervision](https://huggingface.co/papers/2202.11094) by Jiarui Xu, Shalini De Mello, Sifei Liu, Wonmin Byeon, Thomas Breuel, Jan Kautz, Xiaolong Wang.
 1. **[Helium](https://huggingface.co/docs/transformers/main/model_doc/helium)** (from the Kyutai Team) released with the blog post [Announcing Helium-1 Preview](https://kyutai.org/2025/01/13/helium.html) by the Kyutai Team.
 1. **[HerBERT](https://huggingface.co/docs/transformers/model_doc/herbert)** (from Allegro.pl, AGH University of Science and Technology) released with the paper [KLEJ: Comprehensive Benchmark for Polish Language Understanding](https://www.aclweb.org/anthology/2020.acl-main.111.pdf) by Piotr Rybak, Robert Mroczkowski, Janusz Tracz, Ireneusz Gawlik.
+1. **[HRM-Text](https://huggingface.co/docs/transformers/model_doc/hrm_text)** (from Sapient Intelligence) released with the paper [Hierarchical Reasoning Model](https://huggingface.co/papers/2506.21734) by Guan Wang, Jin Li, Yuhao Sun, Xing Chen, Changling Liu, Yue Wu, Meng Lu, Sen Song, Yasin Abbasi Yadkori.
 1. **[Hiera](https://huggingface.co/docs/transformers/model_doc/hiera)** (from Meta) released with the paper [Hiera: A Hierarchical Vision Transformer without the Bells-and-Whistles](https://huggingface.co/papers/2306.00989) by Chaitanya Ryali, Yuan-Ting Hu, Daniel Bolya, Chen Wei, Haoqi Fan, Po-Yao Huang, Vaibhav Aggarwal, Arkabandhu Chowdhury, Omid Poursaeed, Judy Hoffman, Jitendra Malik, Yanghao Li, Christoph Feichtenhofer.
 1. **[Hubert](https://huggingface.co/docs/transformers/model_doc/hubert)** (from Facebook) released with the paper [HuBERT: Self-Supervised Speech Representation Learning by Masked Prediction of Hidden Units](https://huggingface.co/papers/2106.07447) by Wei-Ning Hsu, Benjamin Bolte, Yao-Hung Hubert Tsai, Kushal Lakhotia, Ruslan Salakhutdinov, Abdelrahman Mohamed.
 1. **[HunYuanDenseV1](https://huggingface.co/docs/transformers/model_doc/hunyuan_v1_dense)** (from Tencent) released with the paper [Hunyuan-Large: An Open-Source MoE Model with 52 Billion Activated Parameters by Tencent](https://huggingface.co/papers/2411.02265) by the Tencent team.
diff --git a/packages/transformers/src/configs.js b/packages/transformers/src/configs.js
@@ -223,6 +223,13 @@ function getNormalizedConfig(config) {
             mapping['dim_kv'] = 'head_dim';
             mapping['num_attention_heads'] = 'num_attention_heads';
             break;
+        case 'hrm_text':
+            mapping['num_heads'] = 'num_key_value_heads';
+            mapping['num_layers'] = 'num_hidden_layers';
+            mapping['hidden_size'] = 'hidden_size';
+            mapping['dim_kv'] = 'head_dim';
+            mapping['num_attention_heads'] = 'num_attention_heads';
+            break;
 
         // Encoder-decoder models
         case 't5':
diff --git a/packages/transformers/src/models/hrm_text/modeling_hrm_text.js b/packages/transformers/src/models/hrm_text/modeling_hrm_text.js
@@ -0,0 +1,18 @@
+import { PreTrainedModel, decoder_prepare_inputs_for_generation } from '../modeling_utils.js';
+import { ones_like, zeros_like } from '../../utils/tensor.js';
+
+export class HrmTextPreTrainedModel extends PreTrainedModel {}
+export class HrmTextModel extends HrmTextPreTrainedModel {}
+export class HrmTextForCausalLM extends HrmTextPreTrainedModel {
+    forward_params = ['input_ids', 'attention_mask', 'token_type_ids', 'past_key_values'];
+
+    prepare_inputs_for_generation(input_ids, model_inputs, generation_config) {
+        const prepared = decoder_prepare_inputs_for_generation(this, input_ids, model_inputs, generation_config);
+
+        // HRM-Text can be pretrained as a PrefixLM.
+        // token_type_ids=1 marks tokens inside the bidirectional prefix block,
+        // and 0 marks autoregressively generated tokens.
+        prepared.token_type_ids = (prepared.past_key_values ? zeros_like : ones_like)(prepared.input_ids);
+        return prepared;
+    }
+}
diff --git a/packages/transformers/src/models/models.js b/packages/transformers/src/models/models.js
@@ -73,6 +73,7 @@ export * from './grounding_dino/modeling_grounding_dino.js';
 export * from './groupvit/modeling_groupvit.js';
 export * from './helium/modeling_helium.js';
 export * from './hiera/modeling_hiera.js';
+export * from './hrm_text/modeling_hrm_text.js';
 export * from './hubert/modeling_hubert.js';
 export * from './hunyuan_v1_dense/modeling_hunyuan_v1_dense.js';
 export * from './idefics3/modeling_idefics3.js';
diff --git a/packages/transformers/src/models/registry.js b/packages/transformers/src/models/registry.js
@@ -182,6 +182,7 @@ const MODEL_MAPPING_NAMES_DECODER_ONLY = new Map([
     ['hunyuan_v1_dense', 'HunYuanDenseV1Model'],
     ['youtu', 'YoutuModel'],
     ['zaya', 'ZayaModel'],
+    ['hrm_text', 'HrmTextModel'],
 ]);
 
 export const MODEL_FOR_SPEECH_SEQ_2_SEQ_MAPPING_NAMES = new Map([
@@ -331,6 +332,7 @@ export const MODEL_FOR_CAUSAL_LM_MAPPING_NAMES = new Map([
     ['hunyuan_v1_dense', 'HunYuanDenseV1ForCausalLM'],
     ['youtu', 'YoutuForCausalLM'],
     ['zaya', 'ZayaForCausalLM'],
+    ['hrm_text', 'HrmTextForCausalLM'],
 
     // Also image-text-to-text
     ['phi3_v', 'Phi3VForCausalLM'],
