Skip to content

Commit 369f5ca

Browse files
DannyYuyang-quicDannyYuyang-quic
authored
Qualcomm AI Engine Direct - [Multimodal] granite-speech-3.3-2b (pytorch#18740)
1 parent 9ce1b55 commit 369f5ca

37 files changed

Lines changed: 1644 additions & 535 deletions

‎backends/qualcomm/_passes/canonicalize_conv.py‎

Lines changed: 87 additions & 75 deletions
Original file line numberDiff line numberDiff line change
@@ -116,86 +116,98 @@ def call(self, graph_module: torch.fx.GraphModule):
116116
)
117117

118118
with graph_module.graph.inserting_after(qdq_node_after_unsqueeze):
119-
filter_arg = node.args[1]
120-
filter_node = (
121-
filter_arg
122-
if filter_arg.op == "placeholder"
123-
else node.args[1].args[0]
124-
)
125-
filter_node.meta["val"] = (
126-
filter_node.meta["val"].unsqueeze(2).contiguous()
127-
)
128-
filter_tensor = get_parameter(
129-
filter_node, self.edge_program
130-
).unsqueeze(2)
131-
set_parameter(
132-
(
133-
torch.nn.Parameter(filter_tensor)
134-
if filter_tensor.dtype == torch.float
135-
else filter_tensor
136-
),
137-
filter_node,
138-
self.edge_program,
139-
)
140-
141-
num_args = len(node.args)
142-
143-
bias_node = node.args[2] if num_args > 2 else None
144-
stride = [1] + node.args[3] if num_args > 3 else [1, 1]
145-
padding = [0] + node.args[4] if num_args > 4 else [0, 0]
146-
if node.target == torch.ops.aten.conv1d.default:
147-
dilation = [1] + node.args[5] if num_args > 5 else [1, 1]
148-
groups = node.args[6] if num_args > 6 else 1
149-
conv_args = (
150-
qdq_node_after_unsqueeze,
151-
node.args[1],
152-
bias_node,
153-
stride,
154-
padding,
155-
dilation,
156-
groups,
119+
# conv2d must be inserted before conv1d in the graph to preserve correct
120+
# topological ordering. This is required due to conv-bn fusion: when conv1d
121+
# has no bias, the fused bias (from batchnorm) is introduced as a new node,
122+
# and its corresponding dq (dequantize) node must appear before conv2d in
123+
# the execution order.
124+
with graph_module.graph.inserting_before(node):
125+
filter_arg = node.args[1]
126+
filter_node = (
127+
filter_arg
128+
if filter_arg.op == "placeholder"
129+
else node.args[1].args[0]
157130
)
158-
else:
159-
output_padding = (
160-
[0] + node.args[5] if num_args > 5 else [0, 0]
131+
filter_node.meta["val"] = filter_node.meta["val"].unsqueeze(
132+
2
161133
)
162-
groups = node.args[6] if num_args > 6 else 1
163-
dilation = [1] + node.args[7] if num_args > 7 else [1, 1]
164-
conv_args = (
165-
qdq_node_after_unsqueeze,
166-
node.args[1],
167-
bias_node,
168-
stride,
169-
padding,
170-
output_padding,
171-
groups,
172-
dilation,
173-
)
174-
conv2d_node = graph.create_node(
175-
"call_function",
176-
self.conv1d_op_map[node.target],
177-
conv_args,
178-
)
179-
conv2d_node.meta = copy_meta(
180-
node.meta, lambda m: {**m, "val": m["val"].unsqueeze(2)}
181-
)
182-
qdq_node_after_conv2d = append_qdq(
183-
graph_module=graph_module,
184-
node=conv2d_node,
185-
qdq_node=list(node.users)[0],
186-
)
187-
188-
with graph_module.graph.inserting_after(qdq_node_after_conv2d):
189-
squeeze_op = torch.ops.aten.squeeze_copy.dims
190-
squeeze_node = graph.create_node(
191-
"call_function",
192-
squeeze_op,
134+
filter_tensor = get_parameter(
135+
filter_node, self.edge_program
136+
).unsqueeze(2)
137+
set_parameter(
193138
(
194-
qdq_node_after_conv2d,
195-
[2],
139+
torch.nn.Parameter(filter_tensor)
140+
if filter_tensor.dtype == torch.float
141+
else filter_tensor
196142
),
143+
filter_node,
144+
self.edge_program,
145+
)
146+
147+
num_args = len(node.args)
148+
149+
bias_node = node.args[2] if num_args > 2 else None
150+
stride = [1] + node.args[3] if num_args > 3 else [1, 1]
151+
padding = [0] + node.args[4] if num_args > 4 else [0, 0]
152+
if node.target == torch.ops.aten.conv1d.default:
153+
dilation = (
154+
[1] + node.args[5] if num_args > 5 else [1, 1]
155+
)
156+
groups = node.args[6] if num_args > 6 else 1
157+
conv_args = (
158+
qdq_node_after_unsqueeze,
159+
node.args[1],
160+
bias_node,
161+
stride,
162+
padding,
163+
dilation,
164+
groups,
165+
)
166+
else:
167+
output_padding = (
168+
[0] + node.args[5] if num_args > 5 else [0, 0]
169+
)
170+
groups = node.args[6] if num_args > 6 else 1
171+
dilation = (
172+
[1] + node.args[7] if num_args > 7 else [1, 1]
173+
)
174+
conv_args = (
175+
qdq_node_after_unsqueeze,
176+
node.args[1],
177+
bias_node,
178+
stride,
179+
padding,
180+
output_padding,
181+
groups,
182+
dilation,
183+
)
184+
conv2d_node = graph.create_node(
185+
"call_function",
186+
self.conv1d_op_map[node.target],
187+
conv_args,
188+
)
189+
conv2d_node.meta = copy_meta(
190+
node.meta, lambda m: {**m, "val": m["val"].unsqueeze(2)}
197191
)
198-
squeeze_node.meta = copy_meta(node.meta)
192+
qdq_node_after_conv2d = append_qdq(
193+
graph_module=graph_module,
194+
node=conv2d_node,
195+
qdq_node=list(node.users)[0],
196+
)
197+
198+
with graph_module.graph.inserting_after(
199+
qdq_node_after_conv2d
200+
):
201+
squeeze_op = torch.ops.aten.squeeze_copy.dims
202+
squeeze_node = graph.create_node(
203+
"call_function",
204+
squeeze_op,
205+
(
206+
qdq_node_after_conv2d,
207+
[2],
208+
),
209+
)
210+
squeeze_node.meta = copy_meta(node.meta)
199211

200212
for user in node.users.copy():
201213
user.replace_input_with(node, squeeze_node)

‎backends/qualcomm/tests/models.py‎

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -524,6 +524,25 @@ def example_inputs(self):
524524
}
525525

526526

527+
class Conv1dBn(torch.nn.Module):
528+
def __init__(self, bias=True):
529+
super().__init__()
530+
self.conv = torch.nn.Conv1d(
531+
in_channels=2048,
532+
out_channels=2048,
533+
kernel_size=15,
534+
groups=2048,
535+
bias=bias,
536+
)
537+
self.batch_norm = torch.nn.BatchNorm1d(2048)
538+
self.eval()
539+
540+
def forward(self, x):
541+
x = self.conv(x)
542+
x = self.batch_norm(x)
543+
return x
544+
545+
527546
class Conv1dSequential(torch.nn.Module):
528547
def __init__(self, bias=True):
529548
super().__init__()

‎backends/qualcomm/tests/test_qnn_delegate.py‎

Lines changed: 123 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -414,6 +414,13 @@ def test_qnn_backend_conv1d(self):
414414
with self.subTest(i=i):
415415
self.lower_module_and_test_output(module, sample_input)
416416

417+
def test_qnn_conv1d_batch_norm(self):
418+
modules = [Conv1dBn(), Conv1dBn(bias=False)] # noqa: F405
419+
sample_input = (torch.randn([1, 2048, 858]),)
420+
for i, module in enumerate(modules):
421+
with self.subTest(i=i):
422+
self.lower_module_and_test_output(module, sample_input)
423+
417424
def test_qnn_backend_conv2d(self):
418425
modules = [Conv2dSequential(), Conv2dSequential(bias=False)] # noqa: F405
419426
sample_input = (torch.randn([1, 1, 3, 3]),)
@@ -2829,6 +2836,14 @@ def test_qnn_backend_conv1d(self):
28292836
module = self.get_qdq_module(module, sample_input)
28302837
self.lower_module_and_test_output(module, sample_input)
28312838

2839+
def test_qnn_conv1d_batch_norm(self):
2840+
modules = [Conv1dBn(), Conv1dBn(bias=False)] # noqa: F405
2841+
sample_input = (torch.randn([1, 2048, 858]),)
2842+
for i, module in enumerate(modules):
2843+
with self.subTest(i=i):
2844+
module = self.get_qdq_module(module, sample_input)
2845+
self.lower_module_and_test_output(module, sample_input)
2846+
28322847
def test_qnn_backend_conv2d(self):
28332848
modules = [Conv2dSequential(), Conv2dSequential(bias=False)] # noqa: F405
28342849
sample_input = (torch.randn([1, 1, 3, 3]),)
@@ -7356,13 +7371,30 @@ class MLLMSpecs:
73567371
tok_embedding_pte_size: float
73577372
decoder_pte_size: float
73587373

7374+
@dataclass(frozen=True)
7375+
class ALMSpecs(MLLMSpecs):
7376+
audio_path: str
7377+
golden_audio_feature: str
7378+
73597379
@dataclass(frozen=True)
73607380
class VLMSpecs(MLLMSpecs):
73617381
image_path: str
73627382
golden_image_feature: str
73637383

73647384
# TODO: refactor to support different backends
73657385
def setUp(self):
7386+
self.alm_specs = {
7387+
"granite_speech_3_3-2b": TestExampleMultimodalityScript.ALMSpecs(
7388+
max_seq_len=512,
7389+
sm8650_token_rate=5,
7390+
sm8750_token_rate=8,
7391+
encoder_pte_size=900_000_000, # 900MB
7392+
tok_embedding_pte_size=240_000_000, # 240MB
7393+
decoder_pte_size=3_000_000_000, # 3GB
7394+
audio_path="https://huggingface.co/ibm-granite/granite-speech-3.3-2b/resolve/main/10226_10111_000000.wav?download=true", # Audio content: after his nap,...
7395+
golden_audio_feature="after his nap,",
7396+
),
7397+
}
73667398
self.vlm_specs = {
73677399
"smolvlm_500m_instruct": TestExampleMultimodalityScript.VLMSpecs(
73687400
max_seq_len=128,
@@ -7386,6 +7418,96 @@ def setUp(self):
73867418
),
73877419
}
73887420

7421+
def test_static_asr(self):
7422+
if not self.required_envs([self.model_name]):
7423+
self.skipTest("missing required envs")
7424+
7425+
if self.enable_x86_64:
7426+
# Running on host is extremely slow for large models, so we skip this check to avoid timeouts.
7427+
# Please verify the output on the actual device instead.
7428+
self.skipTest(
7429+
"Skipping the check for the static ASR model on x86 due to long execution time."
7430+
)
7431+
7432+
alm_specs: TestExampleMultimodalityScript.ALMSpecs = self.alm_specs[
7433+
self.model_name
7434+
]
7435+
prompt = "can you transcribe the speech into a written format?"
7436+
audio_path = alm_specs.audio_path
7437+
cmds = [
7438+
"python",
7439+
f"{self.executorch_root}/examples/qualcomm/oss_scripts/llama/llama.py",
7440+
"--artifact",
7441+
self.artifact_dir,
7442+
"--build_folder",
7443+
self.build_folder,
7444+
"--soc_model",
7445+
self.soc_model,
7446+
"--ip",
7447+
self.ip,
7448+
"--port",
7449+
str(self.port),
7450+
"--prompt",
7451+
prompt,
7452+
"--audio_path",
7453+
audio_path,
7454+
"--temperature",
7455+
"0",
7456+
"--decoder_model",
7457+
f"{self.model_name}",
7458+
"--model_mode",
7459+
"kv",
7460+
"--max_seq_len",
7461+
f"{alm_specs.max_seq_len}",
7462+
]
7463+
if self.compile_only:
7464+
cmds.extend(["--compile_only"])
7465+
elif self.device:
7466+
cmds.extend(["--device", self.device])
7467+
if self.host:
7468+
cmds.extend(["--host", self.host])
7469+
if self.pre_gen_pte:
7470+
cmds.extend(["--pre_gen_pte", self.pre_gen_pte])
7471+
7472+
p = subprocess.Popen(cmds, stdout=subprocess.DEVNULL)
7473+
with Listener((self.ip, self.port)) as listener:
7474+
conn = listener.accept()
7475+
p.communicate()
7476+
msg = json.loads(conn.recv())
7477+
if "Error" in msg:
7478+
self.fail(msg["Error"])
7479+
else:
7480+
if not self.compile_only:
7481+
model_out = msg["result"][0]
7482+
self.assertTrue(
7483+
alm_specs.golden_audio_feature in model_out.lower(),
7484+
f"Expected Output contains feature: '{alm_specs.golden_audio_feature}' Actual Output: '{model_out}'",
7485+
)
7486+
print(f"Audio Path: {audio_path}")
7487+
print(f"Query: {prompt}")
7488+
print(f"Answer: {model_out}")
7489+
7490+
encoder_pte_size = msg["audio_encoder_pte_size"]
7491+
tok_embedding_pte_size = msg["tok_embedding_pte_size"]
7492+
decoder_pte_size = msg["pte_size"]
7493+
self.assertLessEqual(encoder_pte_size, alm_specs.encoder_pte_size)
7494+
self.assertLessEqual(
7495+
tok_embedding_pte_size, alm_specs.tok_embedding_pte_size
7496+
)
7497+
self.assertLessEqual(decoder_pte_size, alm_specs.decoder_pte_size)
7498+
print(f"Encoder PTE Size: {encoder_pte_size} bytes")
7499+
print(f"Token Embedding PTE Size: {tok_embedding_pte_size} bytes")
7500+
print(f"Text Decoder PTE Size: {decoder_pte_size} bytes")
7501+
7502+
attr_name = f"{self.soc_model.lower()}_token_rate"
7503+
if not self.compile_only and hasattr(alm_specs, attr_name):
7504+
device_inference_speed = msg["inference_speed"]
7505+
expected_inference_speed = getattr(alm_specs, attr_name)
7506+
print(f"Prompt Evaluation: {device_inference_speed} tokens/second")
7507+
self.assertGreaterEqual(
7508+
device_inference_speed, expected_inference_speed
7509+
)
7510+
73897511
def test_static_vlm(self):
73907512
if not self.required_envs([self.model_name]):
73917513
self.skipTest("missing required envs")
@@ -7450,7 +7572,7 @@ def test_static_vlm(self):
74507572
print(f"Query: {prompt}")
74517573
print(f"Answer: {model_out}")
74527574
if not self.enable_x86_64:
7453-
encoder_pte_size = msg["encoder_pte_size"]
7575+
encoder_pte_size = msg["vision_encoder_pte_size"]
74547576
tok_embedding_pte_size = msg["tok_embedding_pte_size"]
74557577
decoder_pte_size = msg["pte_size"]
74567578
self.assertLessEqual(encoder_pte_size, vlm_specs.encoder_pte_size)

‎examples/models/granite_speech/BUCK‎

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,24 @@
1+
load("@fbcode_macros//build_defs:build_file_migration.bzl", "fbcode_target", "non_fbcode_target")
2+
load("@fbsource//xplat/executorch/build:runtime_wrapper.bzl", "runtime")
3+
4+
oncall("executorch")
5+
6+
fbcode_target(_kind = runtime.python_library,
7+
name = "granite_speech",
8+
srcs = [
9+
"__init__.py",
10+
"convert_weights.py",
11+
],
12+
_is_external_target = True,
13+
base_module = "executorch.examples.models.granite_speech",
14+
resources = {
15+
"config/2b_config.json": "config/2b_config.json",
16+
},
17+
deps = [
18+
"//caffe2:torch",
19+
"//executorch/examples/models/llama:llama2_model",
20+
"fbcode//pytorch/torchtune:lib",
21+
"fbsource//third-party/pypi/safetensors:safetensors",
22+
],
23+
visibility = ["PUBLIC"],
24+
)

0 commit comments

Comments
 (0)