We read every piece of feedback, and take your input very seriously.
To see all available qualifiers, see our documentation.
1 parent 3f0ae30 commit ec6f0d9Copy full SHA for ec6f0d9
1 file changed
scripts/build_eagle3_dataset.py
@@ -7,7 +7,6 @@
7
import os
8
from pathlib import Path
9
10
-import torch
11
from datasets import load_dataset
12
from transformers import AutoTokenizer
13
@@ -32,13 +31,12 @@ def main():
32
31
Separated script to build eagle3 dataset from the training.
33
34
Usage:
35
- python ./scripts/build_eagle3_dataset.py \
+ python ./scripts/build_eagle3_dataset.py \
36
--data-path "cache/dataset/sharegpt.jsonl" \
37
--model-path /shared/public/models/meta-llama/Meta-Llama-3.1-8B-Instruct \
38
--chat-template llama3
39
"""
40
args = parse_args()
41
- torch.distributed.init_process_group(backend="nccl")
42
assert os.path.exists(
43
args.data_path
44
), f"Dataset path {args.data_path} does not exist"
0 commit comments