Skip to content

Commit 007570a

Browse files
NXP backend: Enable aten.upsample_bilinear2d with new Neutron flow. (pytorch#19793)
### Summary Enable `aten.upsample_bilinear2d` with new Neutron flow. ### Test plan Unit tests provided. cc @robert-kalmar @JakeStevens @digantdesai @rascani
1 parent 4de16d0 commit 007570a

2 files changed

Lines changed: 353 additions & 32 deletions

File tree

backends/nxp/backend/ir/converter/node_converters/ops_converters/upsample_bilinear2d_converter.py

Lines changed: 78 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -4,24 +4,49 @@
44
# LICENSE file in the root directory of this source tree.
55

66
import numpy as np
7+
import torch
78

89
from executorch.backends.nxp.backend.data_format import DataFormat, NXP_NODE_FORMAT
910
from executorch.backends.nxp.backend.edge_helper import node_has_well_defined_shape
1011
from executorch.backends.nxp.backend.ir.converter.node_converter import (
1112
CustomDelegationOptions,
13+
is_not_qdq_node,
1214
NodeConverter,
1315
)
1416
from executorch.backends.nxp.backend.ir.tflite_generator.builtin_options.resize_bilinear_options import (
1517
ResizeBilinear,
1618
)
1719
from executorch.backends.nxp.backend.neutron_target_spec import NeutronTargetSpec
1820
from torch.fx import Node
21+
from torch.fx.passes.infra.partitioner import Partition
1922
from torch.nn import Parameter
2023

2124

2225
# noinspection SpellCheckingInspection
2326
class UpsampleBilinear2DConverter(NodeConverter):
2427

28+
@classmethod
29+
def supports_partitioning_result(
30+
cls,
31+
node: Node,
32+
partition_list: list[Partition],
33+
custom_delegation_options: CustomDelegationOptions,
34+
neutron_target_spec: NeutronTargetSpec,
35+
parameters_mapping: dict[str, Parameter],
36+
) -> bool:
37+
input_shape = node.all_input_nodes[0].meta["val"].shape
38+
output_shape = node.meta["val"].shape
39+
is_alone_in_partition = cls.is_node_alone_in_partition(
40+
node, partition_list, filter_fn=is_not_qdq_node
41+
)
42+
43+
if is_alone_in_partition and input_shape == output_shape:
44+
# The operator is a no-op, so the Neutron Converter will skip it. If it's the only node in the
45+
# partition, the graph would end up empty.
46+
return False
47+
48+
return True
49+
2550
@staticmethod
2651
def _is_supported_in_IR(
2752
node: Node,
@@ -36,6 +61,14 @@ def _is_supported_in_IR(
3661
" format. Please report this."
3762
)
3863

64+
# The conversion requires the output shape to be known and static.
65+
if not node_has_well_defined_shape(node):
66+
return False
67+
68+
if len(node.meta["val"].shape) != 4:
69+
# Unexpected case. The input should always be 4D.
70+
return False
71+
3972
return True
4073

4174
@staticmethod
@@ -45,38 +78,58 @@ def _is_supported_on_target(
4578
parameters_mapping: dict[str, Parameter],
4679
custom_delegation_options: CustomDelegationOptions,
4780
) -> bool:
48-
# Neutron requires static shapes.
49-
# neutron-converter/src/OperatorC/UpsamplePlugin.cpp?at=NEUTRON_SOFTWARE_2.2.3#74
50-
if not node_has_well_defined_shape(node):
51-
return False
52-
53-
if len(node.meta["val"].shape) != 4:
54-
# Unexpected case. The input should always be 4D.
55-
return False
56-
57-
# The tensors here use the channels first format (NCHW).
81+
# The tensors are always 4D and use the channels first format (NCHW).
5882
_, in_c, in_h, in_w = node.all_input_nodes[0].meta["val"].shape
5983
_, _, out_h, out_w = node.meta["val"].shape
6084

61-
# Neutron supports only the doubling and quadrupleing of both height and width at the same time.
62-
# neutron-library/src/utils/NeutronLibraryInterrogation.cpp?at=refs%2Ftags%2FNEUTRON_SOFTWARE_2.2.3#778
63-
supported_scales = [2, 4]
64-
if not any(
65-
in_h * scale == out_h and in_w * scale == out_w
66-
for scale in supported_scales
67-
):
68-
return False
69-
70-
# Neutron requires the input channels to be a multiple of `num_macs`.
71-
# neutron-library/src/utils/NeutronLibraryInterrogation.cpp?at=refs%2Ftags%2FNEUTRON_SOFTWARE_2.2.3#777
72-
if in_c % neutron_target_spec.get_num_macs() != 0:
73-
return False
85+
if custom_delegation_options.use_new_flow_neutron_c:
86+
# Requirements specified by the new Neutron flow documentation.
87+
88+
if not NodeConverter.uses_quantization_type_for_io(
89+
node,
90+
supported_types=[torch.int8, torch.uint8],
91+
input_indices=[0],
92+
output_indices=[0],
93+
):
94+
return False
95+
96+
supported_scales = [1, 2, 4, 8]
97+
align_corners = node.args[2]
98+
if align_corners:
99+
if in_h == 1 or in_w == 1:
100+
return False # Avoid division by 0.
101+
h_scale = (out_h - 1) / (in_h - 1)
102+
w_scale = (out_w - 1) / (in_w - 1)
103+
else:
104+
h_scale = out_h / in_h
105+
w_scale = out_w / in_w
106+
107+
# The H and W scales don't need to be equal, but both must be supported.
108+
if (h_scale not in supported_scales) or (w_scale not in supported_scales):
109+
return False
110+
111+
else:
112+
# Requirements of the old Neutron flow.
113+
114+
# Neutron supports only the doubling and quadrupleing of both height and width at the same time.
115+
# neutron-library/src/utils/NeutronLibraryInterrogation.cpp?at=refs%2Ftags%2FNEUTRON_SOFTWARE_2.2.3#778
116+
supported_scales = [2, 4]
117+
if not any(
118+
in_h * scale == out_h and in_w * scale == out_w
119+
for scale in supported_scales
120+
):
121+
return False
122+
123+
# Neutron requires the input channels to be a multiple of `num_macs`.
124+
# neutron-library/src/utils/NeutronLibraryInterrogation.cpp?at=refs%2Ftags%2FNEUTRON_SOFTWARE_2.2.3#777
125+
if in_c % neutron_target_spec.get_num_macs() != 0:
126+
return False
74127

75128
return True
76129

77130
def convert(self, node: Node):
78131
"""Convert the `aten.upsample_bilinear2d.vec` operator to Neutron IR `ResizeBilinear`.
79-
The schema is:
132+
The ExecuTorch schema is:
80133
aten::upsample_bilinear2d.vec(
81134
Tensor input,
82135
SymInt[]? output_size,
@@ -109,6 +162,7 @@ def convert(self, node: Node):
109162
# and the second one is what NeutronIR uses when `align_corners == False and half_pixel_centers == True`.
110163
# https://github.com/tensorflow/tensorflow/blob/v2.20.0/tensorflow/lite/kernels/internal/reference/resize_bilinear.h#L82-L88
111164
# https://github.com/tensorflow/tensorflow/blob/v2.20.0/tensorflow/lite/kernels/internal/reference/resize_bilinear.h#L172-L180
165+
# Also, the new Neutron flow requires that `align_corners` and `half_pixel_centers` are not True simultainiously.
112166
align_corners = node.args[2]
113167
half_pixel_centers = not align_corners
114168
t_op.builtin_options = ResizeBilinear(align_corners, half_pixel_centers)

0 commit comments

Comments
 (0)