Skip to content

Commit 72a28e1

Browse files
- Updated shape inference functions (SetShapeFn) to explicitly validate array
lengths and return `absl::InvalidArgumentError` when input_splits size is 0. - Added `OP_REQUIRES` bounds checking ahead of size evaluations in C++ kernel `Compute` routines using modern `absl::InvalidArgumentError` status factories. - Added comprehensive unit test coverage verifying safe exception throwing across both Eager execution and Graph compilation paths. PiperOrigin-RevId: 921664166
1 parent 73a0f34 commit 72a28e1

7 files changed

Lines changed: 47 additions & 5 deletions

File tree

tensorflow_text/core/kernels/BUILD

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -832,6 +832,7 @@ tf_cc_library(
832832
# tf:lib tensorflow dep,
833833
],
834834
deps = [
835+
"@com_google_absl//absl/status",
835836
"@com_google_absl//absl/strings",
836837
],
837838
)

tensorflow_text/core/kernels/sentencepiece/sentencepiece_detokenizer_kernel.cc

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -33,6 +33,7 @@ limitations under the License.
3333
#include "tensorflow/core/framework/tensor.h"
3434
#include "tensorflow/core/platform/errors.h"
3535
#include "tensorflow/core/framework/op_requires.h"
36+
#include "absl/status/status.h"
3637
#include "tensorflow_text/core/kernels/sentencepiece/optimized_decoder.h"
3738
#include "tensorflow_text/core/kernels/sentencepiece/sentencepiece_detokenizer.h"
3839

@@ -51,6 +52,9 @@ class TFSentencepieceDetokenizerOp : public tensorflow::OpKernel {
5152
input_values_tensor.flat<tensorflow::int32>();
5253
const auto& input_splits_tensor = ctx->input(kInputSplits);
5354
const auto input_splits_flat = input_splits_tensor.flat<Tsplits>();
55+
OP_REQUIRES(ctx, input_splits_flat.size() > 0,
56+
absl::InvalidArgumentError(
57+
"input_splits must have at least 1 element."));
5458
const int num_of_sentences = input_splits_flat.size() - 1;
5559
Tensor* output_tensor = nullptr;
5660
OP_REQUIRES_OK(ctx,
@@ -65,8 +69,8 @@ class TFSentencepieceDetokenizerOp : public tensorflow::OpKernel {
6569
ctx,
6670
split_size >= 0 &&
6771
(input_offset + split_size) <= input_values_flat.size(),
68-
errors::InvalidArgument("input_splits must be monotonically "
69-
"non-decreasing and within bounds."));
72+
absl::InvalidArgumentError("input_splits must be monotonically "
73+
"non-decreasing and within bounds."));
7074
codes_for_split.clear();
7175
codes_for_split.reserve(split_size);
7276
for (int j = 0; j < split_size; ++j) {

tensorflow_text/core/kernels/sentencepiece_kernels.cc

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -576,6 +576,9 @@ class SentencepieceDetokenizeOp : public OpKernel {
576576
const auto input_values_flat = input_values_tensor.flat<T>();
577577
const Tensor& input_splits_tensor = ctx->input(2);
578578
const auto input_splits_flat = input_splits_tensor.flat<Tsplits>();
579+
OP_REQUIRES(ctx, input_splits_flat.size() > 0,
580+
absl::InvalidArgumentError(
581+
"input_splits must have at least 1 element."));
579582
const int64 num_of_sentences = input_splits_flat.size() - 1;
580583

581584
OP_REQUIRES_OK(ctx, HandleExtraOptions(ctx, sp));

tensorflow_text/core/ops/fast_sentencepiece_ops.cc

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,7 @@
1515
#include "tensorflow/core/framework/common_shape_fns.h"
1616
#include "tensorflow/core/framework/op.h"
1717
#include "tensorflow/core/framework/shape_inference.h"
18+
#include "absl/status/status.h"
1819
#include "tensorflow/core/lib/core/errors.h"
1920

2021
namespace tensorflow {
@@ -62,8 +63,14 @@ REGISTER_OP("TFText>FastSentencepieceDetokenize")
6263
TF_RETURN_IF_ERROR(c->WithRank(c->input(1), 1, &unused));
6364
TF_RETURN_IF_ERROR(c->WithRank(c->input(2), 1, &unused));
6465

66+
shape_inference::DimensionHandle num_splits = c->NumElements(c->input(2));
6567
shape_inference::DimensionHandle dim;
66-
TF_RETURN_IF_ERROR(c->Subtract(c->NumElements(c->input(2)), 1, &dim));
68+
if (c->ValueKnown(num_splits) && c->Value(num_splits) <= 0) {
69+
return absl::InvalidArgumentError(
70+
"input_splits must have at least 1 element.");
71+
} else {
72+
TF_RETURN_IF_ERROR(c->Subtract(num_splits, 1, &dim));
73+
}
6774
c->set_output(0, c->Vector(dim));
6875
return absl::OkStatus();
6976
});

tensorflow_text/core/ops/rouge_l_op.cc

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,7 @@
1212
// See the License for the specific language governing permissions and
1313
// limitations under the License.
1414

15+
#include "absl/status/status.h"
1516
#include "tensorflow/core/framework/op.h"
1617
#include "tensorflow/core/framework/shape_inference.h"
1718

@@ -88,8 +89,13 @@ absl::Status RougeLShapeFn(InferenceContext* c) {
8889
&output_nrows_plus_one));
8990

9091
// Output shape is a 1-D tensor with size equal to number of splits minus 1.
92+
DimensionHandle num_splits = c->Dim(output_nrows_plus_one, 0);
9193
DimensionHandle dim;
92-
TF_RETURN_IF_ERROR(c->Subtract(c->Dim(output_nrows_plus_one, 0), 1, &dim));
94+
if (c->ValueKnown(num_splits) && c->Value(num_splits) == 0) {
95+
return absl::InvalidArgumentError("splits must have at least 1 element.");
96+
} else {
97+
TF_RETURN_IF_ERROR(c->Subtract(num_splits, 1, &dim));
98+
}
9399

94100
// All outputs have the same shape.
95101
c->set_output(0, c->Vector(dim));

tensorflow_text/core/ops/sentencepiece_ops.cc

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,7 @@
1515
#include "tensorflow/core/framework/common_shape_fns.h"
1616
#include "tensorflow/core/framework/op.h"
1717
#include "tensorflow/core/framework/shape_inference.h"
18+
#include "absl/status/status.h"
1819
#include "tensorflow/core/lib/core/errors.h"
1920

2021
namespace tensorflow {
@@ -129,8 +130,14 @@ REGISTER_OP("SentencepieceDetokenizeOp")
129130
TF_RETURN_IF_ERROR(c->WithRank(c->input(4), 0, &unused));
130131
TF_RETURN_IF_ERROR(c->WithRank(c->input(5), 0, &unused));
131132

133+
shape_inference::DimensionHandle num_splits = c->NumElements(c->input(2));
132134
shape_inference::DimensionHandle dim;
133-
TF_RETURN_IF_ERROR(c->Subtract(c->NumElements(c->input(2)), 1, &dim));
135+
if (c->ValueKnown(num_splits) && c->Value(num_splits) <= 0) {
136+
return absl::InvalidArgumentError(
137+
"input_splits must have at least 1 element.");
138+
} else {
139+
TF_RETURN_IF_ERROR(c->Subtract(num_splits, 1, &dim));
140+
}
134141
c->set_output(0, c->Vector(dim));
135142
return absl::OkStatus();
136143
});

tensorflow_text/python/ops/sentencepiece_tokenizer_test.py

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -37,6 +37,9 @@
3737
from tensorflow.python.platform import test
3838
from tensorflow.python.saved_model import load
3939
from tensorflow.python.saved_model import save
40+
from tensorflow.python.framework import load_library
41+
from tensorflow.python.platform import resource_loader
42+
gen_sentencepiece_tokenizer = load_library.load_op_library(resource_loader.get_path_to_datafile('_sentencepiece_tokenizer.so'))
4043
from tensorflow_text.python.ops.sentencepiece_tokenizer import SentencepieceTokenizer
4144

4245

@@ -451,6 +454,17 @@ def testEmptyInputDetokenize(self):
451454
detokenized = sp.detokenize(constant_op.constant([], dtypes.int32))
452455
self.assertAllEqual('', detokenized)
453456

457+
def testEmptyInputSplitsDetokenizeOpZeroShape(self):
458+
sp = SentencepieceTokenizer(self.model)
459+
# Providing an empty tensor (length 0) as input_splits directly to the op
460+
# should safely raise InvalidArgumentError instead of crashing.
461+
with self.assertRaises((errors.InvalidArgumentError, ValueError)):
462+
_ = gen_sentencepiece_tokenizer.sentencepiece_detokenize_op(
463+
sp._model_resource.resource_handle,
464+
constant_op.constant([], dtypes.int32),
465+
constant_op.constant([], dtypes.int64),
466+
add_bos=False, add_eos=False, reverse=False)
467+
454468
def testReturnNbestAndDetokenize(self):
455469
sp = SentencepieceTokenizer(
456470
self.model, nbest_size=2, out_type=dtypes.int32, return_nbest=True)

0 commit comments

Comments
 (0)