Fixing an out-of-bounds bug in BPE Tokenizer (#1058)

pavel-petrof · Pavel Petrov · sayanshaw24 · web-flow · commit a851c7c76b54 · 2026-05-07T15:06:17.000-07:00
Co-authored-by: Pavel Petrov &lt;pavelpetrov@microsoft.com&gt;
Co-authored-by: Sayan Shaw &lt;52221015+sayanshaw24@users.noreply.github.com&gt;
diff --git a/operators/tokenizer/bpe_tokenizer_model.hpp b/operators/tokenizer/bpe_tokenizer_model.hpp
@@ -184,7 +184,7 @@ class BpeModel {
       if (i > static_cast<uint32_t>((std::numeric_limits<int32_t>::max)())) {
         continue;  // safe purpose.
       }
-      if (i > id2token_map_.size()) {
+      if (i >= id2token_map_.size()) {
         id2token_map_.resize(static_cast<size_t>(i) + 1);
       }
       id2token_map_[i] = t;
@@ -260,7 +260,7 @@ class BpeModel {
       if (i > static_cast<uint32_t>((std::numeric_limits<int32_t>::max)())) {
         continue;  // safe purpose.
       }
-      if (i > id2token_map_.size()) {
+      if (i >= id2token_map_.size()) {
         id2token_map_.resize(static_cast<size_t>(i) + 1);
       }
       id2token_map_[i] = t;
@@ -298,7 +298,7 @@ class BpeModel {
       if (i > static_cast<uint32_t>((std::numeric_limits<int32_t>::max)())) {
         continue;  // safe purpose.
       }
-      if (i > id2token_map_.size()) {
+      if (i >= id2token_map_.size()) {
         id2token_map_.resize(static_cast<size_t>(i) + 1);
       }
       id2token_map_[i] = t;

Original file line number	Diff line number	Diff line change
`@@ -184,7 +184,7 @@ class BpeModel {`
`184`	`184`	`if (i > static_cast<uint32_t>((std::numeric_limits<int32_t>::max)())) {`
`185`	`185`	`continue; // safe purpose.`
`186`	`186`	`}`
`187`		`- if (i > id2token_map_.size()) {`
	`187`	`+ if (i >= id2token_map_.size()) {`
`188`	`188`	`id2token_map_.resize(static_cast<size_t>(i) + 1);`
`189`	`189`	`}`
`190`	`190`	`id2token_map_[i] = t;`
`@@ -260,7 +260,7 @@ class BpeModel {`
`260`	`260`	`if (i > static_cast<uint32_t>((std::numeric_limits<int32_t>::max)())) {`
`261`	`261`	`continue; // safe purpose.`
`262`	`262`	`}`
`263`		`- if (i > id2token_map_.size()) {`
	`263`	`+ if (i >= id2token_map_.size()) {`
`264`	`264`	`id2token_map_.resize(static_cast<size_t>(i) + 1);`
`265`	`265`	`}`
`266`	`266`	`id2token_map_[i] = t;`
`@@ -298,7 +298,7 @@ class BpeModel {`
`298`	`298`	`if (i > static_cast<uint32_t>((std::numeric_limits<int32_t>::max)())) {`
`299`	`299`	`continue; // safe purpose.`
`300`	`300`	`}`
`301`		`- if (i > id2token_map_.size()) {`
	`301`	`+ if (i >= id2token_map_.size()) {`
`302`	`302`	`id2token_map_.resize(static_cast<size_t>(i) + 1);`
`303`	`303`	`}`
`304`	`304`	`id2token_map_[i] = t;`