Skip to content

Commit e43ab0b

Browse files
xuanyuanminzhengxuanyuanminzheng
authored
[Cherry Pick] [metax_gpu] use method for ernie in metax-gpu. #3576 (#3606)
1 parent d9e9351 commit e43ab0b

8 files changed

Lines changed: 290 additions & 0 deletions

File tree

‎examples/config/metax/ERNIE-4.5-0.3B/sft/lora.yaml‎

Lines changed: 58 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,58 @@
1+
### data
2+
train_dataset_type: erniekit
3+
eval_dataset_type: erniekit
4+
train_dataset_path: ./tests/fixtures/dummy/sft/train.jsonl
5+
train_dataset_prob: "1.0"
6+
eval_dataset_path: ./tests/fixtures/dummy/sft/eval.jsonl
7+
eval_dataset_prob: "1.0"
8+
max_seq_len: 8192
9+
packing: false
10+
mix_strategy: concat
11+
template_backend: custom
12+
template: ernie_nothink
13+
14+
### model
15+
model_name_or_path: baidu/ERNIE-4.5-0.3B-PT
16+
attn_impl: eager
17+
lora: true
18+
lora_rank: 8
19+
20+
### finetuning
21+
# base
22+
device: metax_gpu
23+
stage: SFT
24+
fine_tuning: lora
25+
seed: 23
26+
do_train: true
27+
do_eval: true
28+
per_device_eval_batch_size: 1
29+
per_device_train_batch_size: 1
30+
num_train_epochs: 1
31+
max_steps: -1
32+
eval_steps: 100
33+
evaluation_strategy: steps
34+
save_steps: 100
35+
save_strategy: steps
36+
logging_steps: 1
37+
gradient_accumulation_steps: 4
38+
logging_dir: ./vdl_log
39+
output_dir: ./checkpoints/ERNIE-4.5-0.3B-PT-sft-lora
40+
disable_tqdm: true
41+
eval_accumulation_steps: 16
42+
43+
# train
44+
warmup_steps: 20
45+
learning_rate: 1.0e-4
46+
47+
# performance
48+
tensor_model_parallel_size: 1
49+
pipeline_model_parallel_size: 1
50+
sharding: stage1
51+
recompute_granularity: full
52+
recompute_method: uniform
53+
recompute_num_layers: 1
54+
bf16: true
55+
fp16_opt_level: O2
56+
unified_checkpoint: false
57+
save_checkpoint_format: flex_checkpoint
58+
load_checkpoint_format: flex_checkpoint

‎examples/config/metax/ERNIE-4.5-0.3B/sft/run_lora.sh‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,15 @@
1+
# Copyright (c) 2026 PaddlePaddle Authors. All Rights Reserved.
2+
#
3+
# Licensed under the Apache License, Version 2.0 (the "License");
4+
# you may not use this file except in compliance with the License.
5+
# You may obtain a copy of the License at
6+
#
7+
# http://www.apache.org/licenses/LICENSE-2.0
8+
#
9+
# Unless required by applicable law or agreed to in writing, software
10+
# distributed under the License is distributed on an "AS IS" BASIS,
11+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
# See the License for the specific language governing permissions and
13+
# limitations under the License.
14+
15+
CUDA_VISIBLE_DEVICES=0 paddleformers-cli train examples/config/metax/ERNIE-4.5-0.3B/sft/lora.yaml

‎examples/config/metax/ERNIE-4.5-0.3B/sft/run_sft.sh‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,15 @@
1+
# Copyright (c) 2026 PaddlePaddle Authors. All Rights Reserved.
2+
#
3+
# Licensed under the Apache License, Version 2.0 (the "License");
4+
# you may not use this file except in compliance with the License.
5+
# You may obtain a copy of the License at
6+
#
7+
# http://www.apache.org/licenses/LICENSE-2.0
8+
#
9+
# Unless required by applicable law or agreed to in writing, software
10+
# distributed under the License is distributed on an "AS IS" BASIS,
11+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
# See the License for the specific language governing permissions and
13+
# limitations under the License.
14+
15+
CUDA_VISIBLE_DEVICES=0 paddleformers-cli train examples/config/metax/ERNIE-4.5-0.3B/sft/sft.yaml

‎examples/config/metax/ERNIE-4.5-0.3B/sft/sft.yaml‎

Lines changed: 56 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,56 @@
1+
### data
2+
train_dataset_type: erniekit
3+
eval_dataset_type: erniekit
4+
train_dataset_path: ./tests/fixtures/dummy/sft/train.jsonl
5+
train_dataset_prob: "1.0"
6+
eval_dataset_path: ./tests/fixtures/dummy/sft/eval.jsonl
7+
eval_dataset_prob: "1.0"
8+
max_seq_len: 8192
9+
packing: false
10+
mix_strategy: concat
11+
template_backend: custom
12+
template: ernie_nothink
13+
14+
### model
15+
model_name_or_path: baidu/ERNIE-4.5-0.3B-PT
16+
attn_impl: eager
17+
18+
### finetuning
19+
# base
20+
device: metax_gpu
21+
stage: SFT
22+
fine_tuning: full
23+
seed: 23
24+
do_train: true
25+
do_eval: true
26+
per_device_eval_batch_size: 1
27+
per_device_train_batch_size: 1
28+
num_train_epochs: 1
29+
max_steps: -1
30+
eval_steps: 100
31+
evaluation_strategy: steps
32+
save_steps: 100
33+
save_strategy: steps
34+
logging_steps: 1
35+
gradient_accumulation_steps: 4
36+
logging_dir: ./vdl_log
37+
output_dir: ./checkpoints/ERNIE-4.5-0.3B-PT-sft-full
38+
disable_tqdm: true
39+
eval_accumulation_steps: 16
40+
41+
# train
42+
warmup_steps: 20
43+
learning_rate: 1.0e-5
44+
45+
# performance
46+
tensor_model_parallel_size: 1
47+
pipeline_model_parallel_size: 1
48+
sharding: stage1
49+
recompute_granularity: full
50+
recompute_method: uniform
51+
recompute_num_layers: 1
52+
bf16: true
53+
fp16_opt_level: O2
54+
unified_checkpoint: false
55+
save_checkpoint_format: flex_checkpoint
56+
load_checkpoint_format: flex_checkpoint

‎examples/config/metax/ERNIE-4.5-21B-A3B/sft/lora.yaml‎

Lines changed: 59 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,59 @@
1+
### data
2+
train_dataset_type: erniekit
3+
eval_dataset_type: erniekit
4+
train_dataset_path: ./tests/fixtures/dummy/sft/train.jsonl
5+
train_dataset_prob: "1.0"
6+
eval_dataset_path: ./tests/fixtures/dummy/sft/eval.jsonl
7+
eval_dataset_prob: "1.0"
8+
max_seq_len: 8192
9+
packing: true
10+
mix_strategy: concat
11+
template_backend: custom
12+
template: ernie_nothink
13+
14+
### model
15+
model_name_or_path: baidu/ERNIE-4.5-21B-A3B-PT
16+
attn_impl: eager
17+
lora: true
18+
lora_rank: 8
19+
20+
### finetuning
21+
# base
22+
device: metax_gpu
23+
stage: SFT
24+
fine_tuning: lora
25+
seed: 23
26+
do_train: true
27+
do_eval: true
28+
per_device_eval_batch_size: 1
29+
per_device_train_batch_size: 1
30+
num_train_epochs: 1
31+
max_steps: -1
32+
eval_steps: 100
33+
evaluation_strategy: steps
34+
save_steps: 100
35+
save_strategy: steps
36+
logging_steps: 1
37+
gradient_accumulation_steps: 4
38+
logging_dir: ./vdl_log
39+
output_dir: ./checkpoints/ERNIE-4.5-21B-A3B-PT-sft-lora-tp-pp
40+
disable_tqdm: true
41+
eval_accumulation_steps: 16
42+
43+
# train
44+
warmup_steps: 20
45+
learning_rate: 1.0e-4
46+
47+
# performance
48+
tensor_model_parallel_size: 2
49+
pipeline_model_parallel_size: 2
50+
sequence_parallel: true
51+
sharding: stage1
52+
recompute_granularity: full
53+
recompute_method: uniform
54+
recompute_num_layers: 1
55+
bf16: true
56+
fp16_opt_level: O2
57+
unified_checkpoint: false
58+
save_checkpoint_format: flex_checkpoint
59+
load_checkpoint_format: flex_checkpoint

‎examples/config/metax/ERNIE-4.5-21B-A3B/sft/run_lora.sh‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,15 @@
1+
# Copyright (c) 2026 PaddlePaddle Authors. All Rights Reserved.
2+
#
3+
# Licensed under the Apache License, Version 2.0 (the "License");
4+
# you may not use this file except in compliance with the License.
5+
# You may obtain a copy of the License at
6+
#
7+
# http://www.apache.org/licenses/LICENSE-2.0
8+
#
9+
# Unless required by applicable law or agreed to in writing, software
10+
# distributed under the License is distributed on an "AS IS" BASIS,
11+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
# See the License for the specific language governing permissions and
13+
# limitations under the License.
14+
15+
CUDA_VISIBLE_DEVICES=0,1,2,3 paddleformers-cli train examples/config/metax/ERNIE-4.5-21B-A3B/sft/lora.yaml

‎examples/config/metax/ERNIE-4.5-21B-A3B/sft/run_sft.sh‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,15 @@
1+
# Copyright (c) 2026 PaddlePaddle Authors. All Rights Reserved.
2+
#
3+
# Licensed under the Apache License, Version 2.0 (the "License");
4+
# you may not use this file except in compliance with the License.
5+
# You may obtain a copy of the License at
6+
#
7+
# http://www.apache.org/licenses/LICENSE-2.0
8+
#
9+
# Unless required by applicable law or agreed to in writing, software
10+
# distributed under the License is distributed on an "AS IS" BASIS,
11+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
# See the License for the specific language governing permissions and
13+
# limitations under the License.
14+
15+
CUDA_VISIBLE_DEVICES=0,1,2,3,4,5,6,7 paddleformers-cli train examples/config/metax/ERNIE-4.5-21B-A3B/sft/sft.yaml

‎examples/config/metax/ERNIE-4.5-21B-A3B/sft/sft.yaml‎

Lines changed: 57 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,57 @@
1+
### data
2+
train_dataset_type: erniekit
3+
eval_dataset_type: erniekit
4+
train_dataset_path: ./tests/fixtures/dummy/sft/train.jsonl
5+
train_dataset_prob: "1.0"
6+
eval_dataset_path: ./tests/fixtures/dummy/sft/eval.jsonl
7+
eval_dataset_prob: "1.0"
8+
max_seq_len: 8192
9+
packing: true
10+
mix_strategy: concat
11+
template_backend: custom
12+
template: ernie_nothink
13+
14+
### model
15+
model_name_or_path: baidu/ERNIE-4.5-21B-A3B-PT
16+
attn_impl: eager
17+
18+
### finetuning
19+
# base
20+
device: metax_gpu
21+
stage: SFT
22+
fine_tuning: full
23+
seed: 23
24+
do_train: true
25+
do_eval: true
26+
per_device_eval_batch_size: 1
27+
per_device_train_batch_size: 1
28+
num_train_epochs: 1
29+
max_steps: -1
30+
eval_steps: 100
31+
evaluation_strategy: steps
32+
save_steps: 100
33+
save_strategy: steps
34+
logging_steps: 1
35+
gradient_accumulation_steps: 4
36+
logging_dir: ./vdl_log
37+
output_dir: ./checkpoints/ERNIE-4.5-21B-A3B-PT-sft-full-tp-pp
38+
disable_tqdm: true
39+
eval_accumulation_steps: 16
40+
41+
# train
42+
warmup_steps: 20
43+
learning_rate: 1.0e-5
44+
45+
# performance
46+
tensor_model_parallel_size: 2
47+
pipeline_model_parallel_size: 2
48+
sequence_parallel: true
49+
sharding: stage1
50+
recompute_granularity: full
51+
recompute_method: uniform
52+
recompute_num_layers: 1
53+
bf16: true
54+
fp16_opt_level: O2
55+
unified_checkpoint: false
56+
save_checkpoint_format: flex_checkpoint
57+
load_checkpoint_format: flex_checkpoint

0 commit comments

Comments
 (0)