File tree Expand file tree Collapse file tree
Expand file tree Collapse file tree Original file line number Diff line number Diff line change @@ -77,11 +77,25 @@ def setup_model_and_tokenizer(config: dict):
7777
7878def load_training_data (config : dict ):
7979 """Load training and validation datasets."""
80+ train_path = Path (config ["dataset" ]["train_file" ])
81+ val_path = Path (config ["dataset" ]["val_file" ])
82+
83+ # Ensure required dataset files exist before loading
84+ missing_files = [str (p ) for p in (train_path , val_path ) if not p .is_file ()]
85+ if missing_files :
86+ missing_list = "\n - " .join (missing_files )
87+ message = (
88+ "Required dataset file(s) not found:\n "
89+ f" - { missing_list } \n \n "
90+ "Please generate the dataset before training "
91+ "(for example, by running the dataset generation script)."
92+ )
93+ raise FileNotFoundError (message )
94+
8095 data_files = {
81- "train" : config [ "dataset" ][ "train_file" ] ,
82- "validation" : config [ "dataset" ][ "val_file" ] ,
96+ "train" : str ( train_path ) ,
97+ "validation" : str ( val_path ) ,
8398 }
84-
8599 dataset = load_dataset ("json" , data_files = data_files )
86100 return dataset
87101
You can’t perform that action at this time.
0 commit comments