fix: handle None test_ds in skipped messages count (#232)

RuixiangMa · web-flow · commit 07157bd2957e · 2025-09-21T12:03:20.000+08:00
Co-authored-by: Lancer &lt;maruixiang6688@gmail.com&gt;
diff --git a/scripts/prepare_data.py b/scripts/prepare_data.py
@@ -191,8 +191,9 @@ def process_and_save_ds(train_ds, test_ds, output_path, proc_fn, dataset_name):
                 f.write(json.dumps(row) + "\n")
 
     if total_skipped_count > 0:
+        total_messages = len(train_ds) + (len(test_ds) if test_ds is not None else 0)
         print(
-            f"Skipped {total_skipped_count}/{len(train_ds)+len(test_ds)} messages for {dataset_name}"
+            f"Skipped {total_skipped_count}/{total_messages} messages for {dataset_name}"
         )
 
 

Original file line number	Diff line number	Diff line change
`@@ -191,8 +191,9 @@ def process_and_save_ds(train_ds, test_ds, output_path, proc_fn, dataset_name):`
`191`	`191`	`f.write(json.dumps(row) + "\n")`
`192`	`192`
`193`	`193`	`if total_skipped_count > 0:`
	`194`	`+ total_messages = len(train_ds) + (len(test_ds) if test_ds is not None else 0)`
`194`	`195`	`print(`
`195`		`- f"Skipped {total_skipped_count}/{len(train_ds)+len(test_ds)} messages for {dataset_name}"`
	`196`	`+ f"Skipped {total_skipped_count}/{total_messages} messages for {dataset_name}"`
`196`	`197`	`)`
`197`	`198`
`198`	`199`