Skip to content

Commit d92678d

Browse files
yinggehyinggeh
committed
Update related tests and model generations
1 parent 126fd7e commit d92678d

15 files changed

Lines changed: 78 additions & 141 deletions

File tree

‎docs/user_guide/model_configuration.md‎

Lines changed: 16 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -326,22 +326,22 @@ If a model framework does not have an entry for a given datatype, then Triton do
326326
The sixth column, labeled "API", shows the corresponding datatype for the TRITONSERVER C API, TRITONBACKEND C API, HTTP/REST protocol and GRPC protocol.
327327
The last column shows the corresponding datatype for the Python numpy library.
328328

329-
|Model Config |TensorRT |ONNX Runtime |PyTorch |API |NumPy |
330-
|--------------|--------------|--------------|---------|---------|--------------|
331-
|TYPE_BOOL | kBOOL |BOOL |kBool |BOOL |bool |
332-
|TYPE_UINT8 | kUINT8 |UINT8 |kByte |UINT8 |uint8 |
333-
|TYPE_UINT16 | |UINT16 | |UINT16 |uint16 |
334-
|TYPE_UINT32 | |UINT32 | |UINT32 |uint32 |
335-
|TYPE_UINT64 | |UINT64 | |UINT64 |uint64 |
336-
|TYPE_INT8 | kINT8 |INT8 |kChar |INT8 |int8 |
337-
|TYPE_INT16 | |INT16 |kShort |INT16 |int16 |
338-
|TYPE_INT32 | kINT32 |INT32 |kInt |INT32 |int32 |
339-
|TYPE_INT64 | kINT64 |INT64 |kLong |INT64 |int64 |
340-
|TYPE_FP16 | kHALF |FLOAT16 | |FP16 |float16 |
341-
|TYPE_FP32 | kFLOAT |FLOAT |kFloat |FP32 |float32 |
342-
|TYPE_FP64 | |DOUBLE |kDouble |FP64 |float64 |
343-
|TYPE_STRING | |STRING | |BYTES |dtype(object) |
344-
|TYPE_BF16 | kBF16 | | |BF16 | |
329+
|Model Config |TensorRT |ONNX Runtime |PyTorch |API |NumPy |
330+
|--------------|--------------|--------------|---------|---------|-------------------|
331+
|TYPE_BOOL | kBOOL |BOOL |kBool |BOOL |bool |
332+
|TYPE_UINT8 | kUINT8 |UINT8 |kByte |UINT8 |uint8 |
333+
|TYPE_UINT16 | |UINT16 | |UINT16 |uint16 |
334+
|TYPE_UINT32 | |UINT32 | |UINT32 |uint32 |
335+
|TYPE_UINT64 | |UINT64 | |UINT64 |uint64 |
336+
|TYPE_INT8 | kINT8 |INT8 |kChar |INT8 |int8 |
337+
|TYPE_INT16 | |INT16 |kShort |INT16 |int16 |
338+
|TYPE_INT32 | kINT32 |INT32 |kInt |INT32 |int32 |
339+
|TYPE_INT64 | kINT64 |INT64 |kLong |INT64 |int64 |
340+
|TYPE_FP16 | kHALF |FLOAT16 | |FP16 |float16 |
341+
|TYPE_FP32 | kFLOAT |FLOAT |kFloat |FP32 |float32 |
342+
|TYPE_FP64 | |DOUBLE |kDouble |FP64 |float64 |
343+
|TYPE_STRING | |STRING | |BYTES |dtype(object) |
344+
|TYPE_BF16 | kBF16 |BFLOAT16 | |BF16 |ml_dtypes.bfloat16 |
345345

346346
For TensorRT each value is in the nvinfer1::DataType namespace.
347347
For example, nvinfer1::DataType::kFLOAT is the 32-bit floating-point datatype.

‎qa/L0_backend_identity/identity_test.py‎

Lines changed: 5 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
#!/usr/bin/python
22

3-
# Copyright 2019-2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
3+
# Copyright 2019-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
44
#
55
# Redistribution and use in source and binary forms, with or without
66
# modification, are permitted provided that the following conditions
@@ -30,6 +30,7 @@
3030
import sys
3131
from builtins import range
3232

33+
import ml_dtypes
3334
import numpy as np
3435
import requests as httpreq
3536
import tritonclient.grpc as grpcclient
@@ -201,8 +202,8 @@
201202
("identity_nobatch_int8", np.int8, [0]),
202203
("identity_nobatch_int8", np.int8, [7]),
203204
("identity_bytes", object, [1, 1]),
204-
("identity_bf16", np.float32, [1, 0]),
205-
("identity_bf16", np.float32, [1, 5])
205+
("identity_bf16", ml_dtypes.bfloat16, [1, 0]),
206+
("identity_bf16", ml_dtypes.bfloat16, [1, 5])
206207
):
207208
# yapf: enable
208209
if np_dtype != object:
@@ -211,10 +212,7 @@
211212
in0 = 16384 * np.ones(shape, dtype="int")
212213
in0n = np.array([str(x) for x in in0.reshape(in0.size)], dtype=object)
213214
input_data = in0n.reshape(in0.shape)
214-
if model_name != "identity_bf16":
215-
triton_type = np_to_triton_dtype(input_data.dtype)
216-
else:
217-
triton_type = "BF16"
215+
triton_type = np_to_triton_dtype(input_data.dtype)
218216
inputs = [client_util.InferInput("INPUT0", input_data.shape, triton_type)]
219217
inputs[0].set_data_from_numpy(input_data)
220218

‎qa/L0_backend_onnxruntime/bfloat16_test.py‎

Lines changed: 17 additions & 53 deletions
Original file line numberDiff line numberDiff line change
@@ -31,6 +31,7 @@
3131

3232
import ml_dtypes
3333
import numpy as np
34+
import pytest
3435
import tritonclient.grpc as grpcclient
3536
import tritonclient.http as httpclient
3637

@@ -79,64 +80,27 @@ def _infer_bf16(self, input0_data, input1_data):
7980
results = self.client_.infer(self.model_name_, [input0, input1])
8081
return results.as_numpy("OUTPUT")
8182

82-
def test_bf16_add_variants(self):
83-
"""Run multiple BF16 add cases in one test: zeros, negatives, large, small, cancellation, identical."""
83+
@pytest.mark.parametrize(
84+
"input0_val,input1_val,expected_val",
85+
[
86+
(0.0, 0.0, 0.0), # zeros
87+
(-1.5, 3.5, 2.0), # negatives / mixed
88+
(100.0, 200.0, 300.0), # large
89+
(1e-2, 1e-2, 2e-2), # small (near underflow)
90+
(1.0, -1.0, 0.0), # cancellation
91+
(2.0, 2.0, 4.0), # identical inputs
92+
],
93+
)
94+
def test_bf16_add_variants(self, input0_val, input1_val, expected_val):
95+
"""Run BF16 add for one case: zeros, negatives, large, small, cancellation, or identical."""
8496
shape = (5, 5)
85-
86-
# Zeros: 0.0 + 0.0 = 0.0
87-
output = self._infer_bf16(
88-
np.zeros(shape, dtype=ml_dtypes.bfloat16),
89-
np.zeros(shape, dtype=ml_dtypes.bfloat16),
90-
)
91-
self.assertEqual(output.dtype, ml_dtypes.bfloat16)
92-
self._assert_allclose_bf16(output, np.zeros(shape, dtype=ml_dtypes.bfloat16))
93-
94-
# Negative and mixed: -1.5 + 3.5 = 2.0
95-
output = self._infer_bf16(
96-
np.full(shape, -1.5, dtype=ml_dtypes.bfloat16),
97-
np.full(shape, 3.5, dtype=ml_dtypes.bfloat16),
98-
)
99-
self.assertEqual(output.dtype, ml_dtypes.bfloat16)
100-
self._assert_allclose_bf16(
101-
output, np.full(shape, 2.0, dtype=ml_dtypes.bfloat16)
102-
)
103-
104-
# Large values within BF16 range: 100 + 200 = 300
105-
output = self._infer_bf16(
106-
np.full(shape, 100.0, dtype=ml_dtypes.bfloat16),
107-
np.full(shape, 200.0, dtype=ml_dtypes.bfloat16),
108-
)
109-
self.assertEqual(output.dtype, ml_dtypes.bfloat16)
110-
self._assert_allclose_bf16(
111-
output, np.full(shape, 300.0, dtype=ml_dtypes.bfloat16)
112-
)
113-
114-
# Small values (near underflow / precision limit): 0.01 + 0.01 = 0.02
115-
output = self._infer_bf16(
116-
np.full(shape, 1e-2, dtype=ml_dtypes.bfloat16),
117-
np.full(shape, 1e-2, dtype=ml_dtypes.bfloat16),
118-
)
119-
self.assertEqual(output.dtype, ml_dtypes.bfloat16)
120-
self._assert_allclose_bf16(
121-
output, np.full(shape, 2e-2, dtype=ml_dtypes.bfloat16)
122-
)
123-
124-
# Exact cancellation: 1.0 + (-1.0) = 0.0
125-
output = self._infer_bf16(
126-
np.full(shape, 1.0, dtype=ml_dtypes.bfloat16),
127-
np.full(shape, -1.0, dtype=ml_dtypes.bfloat16),
128-
)
129-
self.assertEqual(output.dtype, ml_dtypes.bfloat16)
130-
self._assert_allclose_bf16(output, np.zeros(shape, dtype=ml_dtypes.bfloat16))
131-
132-
# Identical inputs: 2.0 + 2.0 = 4.0
13397
output = self._infer_bf16(
134-
np.full(shape, 2.0, dtype=ml_dtypes.bfloat16),
135-
np.full(shape, 2.0, dtype=ml_dtypes.bfloat16),
98+
np.full(shape, input0_val, dtype=ml_dtypes.bfloat16),
99+
np.full(shape, input1_val, dtype=ml_dtypes.bfloat16),
136100
)
137101
self.assertEqual(output.dtype, ml_dtypes.bfloat16)
138102
self._assert_allclose_bf16(
139-
output, np.full(shape, 4.0, dtype=ml_dtypes.bfloat16)
103+
output, np.full(shape, expected_val, dtype=ml_dtypes.bfloat16)
140104
)
141105

142106

‎qa/L0_backend_python/python_test.py‎

Lines changed: 5 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
#!/usr/bin/python
22

3-
# Copyright 2019-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
3+
# Copyright 2019-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
44
#
55
# Redistribution and use in source and binary forms, with or without
66
# modification, are permitted provided that the following conditions
@@ -33,6 +33,7 @@
3333
import os
3434
import unittest
3535

36+
import ml_dtypes
3637
import numpy as np
3738
import requests as httpreq
3839
import shm_util
@@ -374,7 +375,7 @@ def test_bf16(self):
374375
) as client:
375376
# NOTE: Client will truncate FP32 to BF16 internally
376377
# since numpy has no built-in BF16 representation.
377-
np_input = np.ones(shape, dtype=np.float32)
378+
np_input = np.ones(shape, dtype=ml_dtypes.bfloat16)
378379
inputs = [
379380
httpclient.InferInput(
380381
"INPUT0", np_input.shape, "BF16"
@@ -391,8 +392,8 @@ def test_bf16(self):
391392
np_output = result.as_numpy("OUTPUT0")
392393
self.assertIsNotNone(np_output)
393394
# BF16 tensors are held in FP32 when converted to numpy due to
394-
# lack of native BF16 support in numpy, so verify that.
395-
self.assertEqual(np_output.dtype, np.float32)
395+
# lack of native support in numpy, so verify that.
396+
self.assertEqual(np_output.dtype, ml_dtypes.bfloat16)
396397
self.assertTrue(np.allclose(np_output, np_input))
397398

398399
def test_infer_pytorch(self):

‎qa/L0_infer/infer_test.py‎

Lines changed: 1 addition & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
#!/usr/bin/env python3
22

3-
# Copyright 2018-2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
3+
# Copyright 2018-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
44
#
55
# Redistribution and use in source and binary forms, with or without
66
# modification, are permitted provided that the following conditions
@@ -157,9 +157,6 @@ def _infer_exact_helper(
157157
input_dtype,
158158
output0_dtype,
159159
output1_dtype,
160-
(input_size,),
161-
(input_size,),
162-
(input_size,),
163160
):
164161
ensemble_prefix.append(prefix)
165162

‎qa/L0_infer_reshape/infer_reshape_test.py‎

Lines changed: 1 addition & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
#!/usr/bin/env python3
22

3-
# Copyright 2019-2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
3+
# Copyright 2019-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
44
#
55
# Redistribution and use in source and binary forms, with or without
66
# modification, are permitted provided that the following conditions
@@ -155,9 +155,6 @@ def _full_reshape(self, dtype, input_shapes, output_shapes=None, no_batch=True):
155155
dtype,
156156
dtype,
157157
dtype,
158-
input_shapes[0],
159-
input_shapes[0],
160-
input_shapes[0],
161158
):
162159
# model that supports batching
163160
for bs in (1, 8):

‎qa/L0_infer_variable/infer_variable_test.py‎

Lines changed: 1 addition & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
#!/usr/bin/env python3
22

3-
# Copyright 2019-2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
3+
# Copyright 2019-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
44
#
55
# Redistribution and use in source and binary forms, with or without
66
# modification, are permitted provided that the following conditions
@@ -140,9 +140,6 @@ def _infer_exact_helper(
140140
input_dtype,
141141
output0_dtype,
142142
output1_dtype,
143-
input_shape,
144-
input_shape,
145-
input_shape,
146143
):
147144
ensemble_prefix.append(prefix)
148145

‎qa/L0_infer_zero/infer_zero_test.py‎

Lines changed: 2 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
#!/usr/bin/env python3
22

3-
# Copyright 2019-2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
3+
# Copyright 2019-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
44
#
55
# Redistribution and use in source and binary forms, with or without
66
# modification, are permitted provided that the following conditions
@@ -93,9 +93,7 @@ def _full_zero(self, dtype, shapes):
9393
)
9494

9595
for name in ["simple_zero", "sequence_zero", "fan_zero"]:
96-
if tu.validate_for_ensemble_model(
97-
name, dtype, dtype, dtype, shapes[0], shapes[0], shapes[0]
98-
):
96+
if tu.validate_for_ensemble_model(name, dtype, dtype, dtype):
9997
# model that supports batching
10098
for bs in (1, 8):
10199
batch_shapes = [

‎qa/L0_trt_bf16_dtype/trt_bf16_dtype_test.py‎

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
#!/usr/bin/env python3
2-
# Copyright 2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
2+
# Copyright 2024-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
33
#
44
# Redistribution and use in source and binary forms, with or without
55
# modification, are permitted provided that the following conditions
@@ -32,6 +32,7 @@
3232

3333
import unittest
3434

35+
import ml_dtypes
3536
import numpy as np
3637
import test_util as tu
3738
import tritonclient.http as client
@@ -49,8 +50,8 @@ def _infer_helper(self, model_name, shape):
4950
inputs.append(client.InferInput("INPUT0", shape, "BF16"))
5051
inputs.append(client.InferInput("INPUT1", shape, "BF16"))
5152

52-
input0_data = np.ones(shape=shape).astype(np.float32)
53-
input1_data = np.ones(shape=shape).astype(np.float32)
53+
input0_data = np.ones(shape=shape).astype(ml_dtypes.bfloat16)
54+
input1_data = np.ones(shape=shape).astype(ml_dtypes.bfloat16)
5455

5556
inputs[0].set_data_from_numpy(input0_data, binary_data=True)
5657
inputs[1].set_data_from_numpy(input1_data, binary_data=True)

‎qa/common/gen_common.py‎

Lines changed: 6 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
# Copyright 2023-2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
1+
# Copyright 2023-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
22
#
33
# Redistribution and use in source and binary forms, with or without
44
# modification, are permitted provided that the following conditions
@@ -28,14 +28,11 @@
2828
from typing import List
2929

3030
# Common utilities for model generation scripts
31+
import ml_dtypes
3132
import numpy as np
3233

3334
np_dtype_string = np.dtype(object)
3435

35-
# Numpy does not support the BF16 datatype natively.
36-
# We use this dummy dtype as a representative for BF16.
37-
np_dtype_bfloat16 = np.dtype([("bf16", object)])
38-
3936

4037
def np_to_onnx_dtype(np_dtype):
4138
import onnx
@@ -62,6 +59,8 @@ def np_to_onnx_dtype(np_dtype):
6259
return onnx.TensorProto.DOUBLE
6360
elif np_dtype == np_dtype_string:
6461
return onnx.TensorProto.STRING
62+
elif np_dtype == ml_dtypes.bfloat16:
63+
return onnx.TensorProto.BFLOAT16
6564
return None
6665

6766

@@ -88,7 +87,7 @@ def np_to_model_dtype(np_dtype):
8887
return "TYPE_FP64"
8988
elif np_dtype == np_dtype_string:
9089
return "TYPE_STRING"
91-
elif np_dtype == np_dtype_bfloat16:
90+
elif np_dtype == ml_dtypes.bfloat16:
9291
return "TYPE_BF16"
9392
return None
9493

@@ -110,7 +109,7 @@ def np_to_trt_dtype(np_dtype):
110109
return trt.float16
111110
elif np_dtype == np.float32:
112111
return trt.float32
113-
elif np_dtype == np_dtype_bfloat16:
112+
elif np_dtype == ml_dtypes.bfloat16:
114113
return trt.bfloat16
115114
return None
116115

0 commit comments

Comments
 (0)