forked from pytorch/torchtitan
-
Notifications
You must be signed in to change notification settings - Fork 0
Expand file tree
/
Copy pathmodels.py
More file actions
executable file
·178 lines (172 loc) · 6.82 KB
/
Copy pathmodels.py
File metadata and controls
executable file
·178 lines (172 loc) · 6.82 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
# Copyright (c) Meta Platforms, Inc. and affiliates.
# All rights reserved.
#
# This source code is licensed under the BSD-style license found in the
# LICENSE file in the root directory of this source tree.
from tests.integration_tests import OverrideDefinitions
def build_model_tests_list() -> list[OverrideDefinitions]:
"""
Build a dictionary of model parallelism test configurations.
This test suite is aimed at testing the model parallelism of torchtitan, and will broadly cover
all the supported model parallelism patterns on all the supported models.
Returns:
A dictionary where each key is a model name and value is a list of OverrideDefinitions
"""
model_tests = [
# Integration Test Cases for DeepSeek V3
OverrideDefinitions(
[
[
"--module deepseek_v3 --config deepseek_v3_debugmodel",
"--parallelism.data_parallel_shard_degree 4",
"--parallelism.expert_parallel_degree 2",
"--compile.enable",
],
],
"DeepSeek V3 FSDP+EP+compile",
"deepseek_v3_fsdp+ep+compile",
ngpu=4,
),
OverrideDefinitions(
[
[
"--module deepseek_v3 --config deepseek_v3_debugmodel",
"--parallelism.pipeline_parallel_degree 2",
"--parallelism.pipeline_parallel_schedule Interleaved1F1B",
"--parallelism.data_parallel_shard_degree 2",
"--parallelism.tensor_parallel_degree 2",
"--parallelism.expert_parallel_degree 4",
],
],
"DeepSeek V3 PP+FSDP+TP+EP",
"deepseek_v3_pp+fsdp+tp+ep",
ngpu=8,
),
OverrideDefinitions(
[
[
"--module deepseek_v3 --config deepseek_v3_debugmodel",
"--parallelism.data_parallel_replicate_degree 2",
"--parallelism.data_parallel_shard_degree 2",
"--parallelism.expert_parallel_degree 2",
],
],
"DeepSeek V3 HSDP+EP",
"deepseek_v3_hsdp+ep",
ngpu=4,
),
# Integration Test Cases for Qwen3 dense and MoE model
OverrideDefinitions(
[
[
"--module qwen3 --config qwen3_debugmodel_moe_param_groups",
"--parallelism.data_parallel_shard_degree 2",
"--parallelism.tensor_parallel_degree 2",
"--parallelism.expert_parallel_degree 4",
],
],
"Qwen3 MoE FSDP+TP+EP (param groups)",
"qwen3_moe_fsdp+tp+ep_param_groups",
ngpu=4,
),
OverrideDefinitions(
[
[
"--module qwen3 --config qwen3_debugmodel",
"--parallelism.data_parallel_shard_degree 2",
"--parallelism.tensor_parallel_degree 2",
"--parallelism.no-enable-sequence-parallel",
"--parallelism.context_parallel_degree 2",
],
[
"--module qwen3 --config qwen3_debugmodel",
"--parallelism.data_parallel_shard_degree 2",
"--parallelism.tensor_parallel_degree 2",
"--parallelism.context_parallel_degree 2",
],
],
"Qwen3 FSDP+TP+CP (SP disabled)",
"qwen3_fsdp+tp+cp_no_sp",
ngpu=8,
),
OverrideDefinitions(
[
[
"--module qwen3 --config qwen3_debugmodel",
"--parallelism.data_parallel_shard_degree 2",
"--parallelism.tensor_parallel_degree 2",
"--parallelism.context_parallel_degree 2",
"--compile.enable",
"--override.imports torchtitan.overrides.helion_rope",
],
],
"Qwen3 fused QKV FSDP+TP+CP + compile + Helion RoPE override",
"qwen3_fused_qkv_fsdp+tp+cp_compile_helion_rope",
ngpu=8,
# The Helion fused cos/sin RoPE kernel is CUDA-only and its autotuned
# configs are tuned for NVIDIA H100; skip on ROCm where it is
# unvalidated (see torchtitan/overrides/helion_rope.py).
skip_rocm_test=True,
),
OverrideDefinitions(
[
[
"--module qwen3 --config qwen3_debugmodel_non_fused_qkv",
"--parallelism.data_parallel_shard_degree 2",
"--parallelism.tensor_parallel_degree 2",
"--parallelism.context_parallel_degree 2",
],
],
# Reverse test: fused QKV is the debugmodel default, so exercise the
# separate wq/wk/wv projection path under FSDP+TP+CP.
"Qwen3 non-fused QKV FSDP+TP+CP",
"qwen3_non_fused_qkv_fsdp+tp+cp",
ngpu=8,
),
# Integration Test Cases for Qwen3.5
OverrideDefinitions(
[
[
"--module qwen3_5 --config qwen35_debugmodel_moe",
"--parallelism.data_parallel_shard_degree 2",
"--parallelism.pipeline_parallel_degree 2",
"--parallelism.tensor_parallel_degree 2",
"--parallelism.expert_parallel_degree 4",
],
],
"Qwen3.5 MoE FSDP+TP+EP+PP",
"qwen3_5_moe_fsdp+tp+ep+pp",
ngpu=8,
),
# Integration Test Cases for gpt-oss
OverrideDefinitions(
[
[
"--module gpt_oss --config gpt_oss_debugmodel",
"--parallelism.data_parallel_shard_degree 4",
"--parallelism.tensor_parallel_degree 2",
"--parallelism.expert_parallel_degree 4",
"--compile.enable",
],
],
"Gpt-oss FSDP+TP+EP+compile",
"gpt_oss_fsdp+tp+ep+compile",
ngpu=8,
),
OverrideDefinitions(
[
[
"--module gpt_oss --config gpt_oss_debugmodel_flex",
"--parallelism.data_parallel_shard_degree 4",
"--parallelism.pipeline_parallel_degree 2",
"--parallelism.pipeline_parallel_schedule Interleaved1F1B",
"--parallelism.expert_parallel_degree 4",
"activation-checkpoint:selective",
],
],
"Gpt-oss PP+FSDP+EP+SACOP",
"gpt_oss_pp+fsdp+ep+sacop",
ngpu=8,
),
]
return model_tests