Skip to content

Commit 68b2710

Browse files
Update ai-model/train.py
Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>
1 parent 40642c2 commit 68b2710

1 file changed

Lines changed: 17 additions & 3 deletions

File tree

ai-model/train.py

Lines changed: 17 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -77,11 +77,25 @@ def setup_model_and_tokenizer(config: dict):
7777

7878
def load_training_data(config: dict):
7979
"""Load training and validation datasets."""
80+
train_path = Path(config["dataset"]["train_file"])
81+
val_path = Path(config["dataset"]["val_file"])
82+
83+
# Ensure required dataset files exist before loading
84+
missing_files = [str(p) for p in (train_path, val_path) if not p.is_file()]
85+
if missing_files:
86+
missing_list = "\n - ".join(missing_files)
87+
message = (
88+
"Required dataset file(s) not found:\n"
89+
f" - {missing_list}\n\n"
90+
"Please generate the dataset before training "
91+
"(for example, by running the dataset generation script)."
92+
)
93+
raise FileNotFoundError(message)
94+
8095
data_files = {
81-
"train": config["dataset"]["train_file"],
82-
"validation": config["dataset"]["val_file"],
96+
"train": str(train_path),
97+
"validation": str(val_path),
8398
}
84-
8599
dataset = load_dataset("json", data_files=data_files)
86100
return dataset
87101

0 commit comments

Comments
 (0)