Add shuffle=False option for time series data (closes #44)

Add shuffle parameter (default True) to ratio(), fixed(), and kfold().
When False, files are split in sorted order without randomization.
Add --no-shuffle CLI flag. Update README with documentation.

Author: jfilter

README.md

@@ -47,7 +47,7 @@ This should get you started to do some serious deep learning on your data. [Read
 
 -   Split files into a training set and a validation set (and optionally a test set).
 -   Works on any file types.
--   The files get shuffled.
+-   The files get shuffled (can be disabled for time series data).
 -   A [seed](https://docs.python.org/3/library/random.html#random.seed) makes splits reproducible.
 -   Allows randomized [oversampling](https://en.wikipedia.org/wiki/Oversampling_and_undersampling_in_data_analysis) for imbalanced datasets.
 -   Optionally group files by prefix or by stem.
@@ -87,14 +87,14 @@ import splitfolders
 # To only split into training and validation set, set a tuple to `ratio`, i.e, `(.8, .2)`.
 splitfolders.ratio("input_folder", output="output",
     seed=1337, ratio=(.8, .1, .1), group_prefix=None, group=None,
-    formats=None, move=False) # default values
+    formats=None, move=False, shuffle=True) # default values
 
 # Split val/test with a fixed number of items, e.g. `(100, 100)`, for each set.
 # To only split into training and validation set, use a single number to `fixed`, i.e., `10`.
 # Set 3 values, e.g. `(300, 100, 100)`, to limit the number of training values.
 splitfolders.fixed("input_folder", output="output",
     seed=1337, fixed=(100, 100), oversample=False, group_prefix=None, group=None,
-    formats=None, move=False) # default values
+    formats=None, move=False, shuffle=True) # default values
 
 # Use `fixed="auto"` with oversampling to auto-compute the val size from the smallest class.
 # Allocates ~20% of the smallest class to validation, rest to training.
@@ -106,7 +106,11 @@ splitfolders.fixed("input_folder", output="output",
 # Uses symlinks by default to avoid k× disk usage.
 splitfolders.kfold("input_folder", output="output",
     seed=1337, k=5, group_prefix=None, group=None,
-    formats=None, move="symlink") # default values
+    formats=None, move="symlink", shuffle=True) # default values
+
+# Split without shuffling (e.g. for time series data).
+splitfolders.ratio("input_folder", output="output",
+    ratio=(.8, .1, .1), shuffle=False)
 ```
 
 ### Grouping files
@@ -215,7 +219,7 @@ Set
 
 ```
 Usage:
-    splitfolders [--output] [--ratio] [--fixed] [--kfold] [--seed] [--oversample] [--group_prefix] [--group] [--formats] [--move] folder_with_images
+    splitfolders [--output] [--ratio] [--fixed] [--kfold] [--seed] [--oversample] [--group_prefix] [--group] [--formats] [--move] [--no-shuffle] folder_with_images
 Options:
     --output        path to the output folder. defaults to `output`. Get created if non-existent.
     --ratio         the ratio to split. e.g. for train/val/test `.8 .1 .1 --` or for train/val `.8 .2 --`.
@@ -231,6 +235,7 @@ Options:
     --formats       split the files based on specified extension(s)
     --move          move the files instead of copying
     --symlink       symlink(create shortcut) the files instead of copying
+    --no-shuffle    do not shuffle files before splitting (useful for time series data)
 Example:
     splitfolders --ratio .8 .1 .1 -- folder_with_images
     splitfolders --kfold 5 folder_with_images

splitfolders/cli.py

@@ -88,6 +88,12 @@ def run():
         default=None,
         help="specify the file format(s) which should be considered for spliting the data e.g. `.png .jpeg .jpg`",
     )
+    parser.add_argument(
+        "--no-shuffle",
+        action="store_true",
+        default=False,
+        help="do not shuffle files before splitting (useful for time series data)",
+    )
 
     args = parser.parse_args()
 
@@ -97,34 +103,29 @@ def run():
     if args.symlink:
         args.move = "symlink"
 
+    shuffle = not args.no_shuffle
+
     if args.ratio:
-        ratio(args.input, args.output, args.seed, args.ratio, args.group_prefix, args.group, args.move, args.formats)
+        ratio(
+            args.input, args.output, args.seed, args.ratio,
+            args.group_prefix, args.group, args.move, args.formats, shuffle,
+        )
     elif args.fixed:
         if args.fixed == ["auto"]:
             fixed_value = "auto"
         else:
             fixed_value = [int(x) for x in args.fixed]
         fixed(
-            args.input,
-            args.output,
-            args.seed,
-            fixed_value,
-            args.oversample,
-            args.group_prefix,
-            args.group,
-            args.move,
-            args.formats,
+            args.input, args.output, args.seed, fixed_value,
+            args.oversample, args.group_prefix, args.group, args.move,
+            args.formats, shuffle,
         )
     elif args.kfold:
         kfold(
-            args.input,
-            args.output,
-            args.seed,
-            args.kfold,
-            args.group_prefix,
-            args.group,
+            args.input, args.output, args.seed, args.kfold,
+            args.group_prefix, args.group,
             args.move if args.move else "symlink",
-            args.formats,
+            args.formats, shuffle,
         )
     else:
         print("Please specify either your `--ratio`, `--fixed`, or `--kfold` for the split. see -h for more help.")

splitfolders/grouping.py

@@ -52,7 +52,7 @@ def group_by_stem(files):
     return [tuple(sorted(g)) for g in sorted(stem_groups.values(), key=lambda g: g[0])]
 
 
-def setup_sibling_files(input_dir, seed, formats=None):
+def setup_sibling_files(input_dir, seed, formats=None, shuffle=True):
     """Lists type dirs, groups files by stem across all dirs.
     Validates every stem exists in every dir. Returns (type_dir_names, groups)."""
     from .utils import list_dirs, list_files
@@ -86,12 +86,13 @@ def setup_sibling_files(input_dir, seed, formats=None):
                     "All stems must exist in every subdirectory for group='sibling'."
                 )
 
-    # Build groups: each group is a dict mapping type_dir_name -> Path
+    # Build groups: each group is a tuple mapping type_dir_name -> Path
     import random
 
     random.seed(seed)
     sorted_stems = sorted(all_stems)
-    random.shuffle(sorted_stems)
+    if shuffle:
+        random.shuffle(sorted_stems)
 
     groups = []
     for stem in sorted_stems:

splitfolders/split.py

@@ -107,7 +107,7 @@ def valid_extensions(formats):
 
 def ratio(
     input, output="output", seed=1337, ratio=(0.8, 0.1, 0.1),
-    group_prefix=None, group=None, move=False, formats=None,
+    group_prefix=None, group=None, move=False, formats=None, shuffle=True,
 ):
     if not round(sum(ratio), 5) == 1:  # round for floating imprecision
         raise ValueError("The sums of `ratio` is over 1.")
@@ -121,11 +121,14 @@ def ratio(
         prog_bar = tqdm(desc="Copying files", unit=" files")
 
     if group == "sibling":
-        split_sibling_dirs_ratio(input, output, ratio, seed, prog_bar if use_tqdm else None, move, formats)
+        split_sibling_dirs_ratio(
+            input, output, ratio, seed, prog_bar if use_tqdm else None, move, formats, shuffle,
+        )
     else:
         for class_dir in list_dirs(input):
             split_class_dir_ratio(
-                class_dir, output, ratio, seed, prog_bar if use_tqdm else None, group_prefix, group, move, formats
+                class_dir, output, ratio, seed, prog_bar if use_tqdm else None,
+                group_prefix, group, move, formats, shuffle,
             )
 
     if use_tqdm:
@@ -134,7 +137,7 @@ def ratio(
 
 def fixed(
     input, output="output", seed=1337, fixed=(100, 100), oversample=False,
-    group_prefix=None, group=None, move=False, formats=None,
+    group_prefix=None, group=None, move=False, formats=None, shuffle=True,
 ):
     check_input_format(input)
     valid_extensions(formats)
@@ -170,7 +173,9 @@ def fixed(
         prog_bar = tqdm(desc="Copying files", unit=" files")
 
     if group == "sibling":
-        split_sibling_dirs_fixed(input, output, fixed, seed, prog_bar if use_tqdm else None, move, formats)
+        split_sibling_dirs_fixed(
+            input, output, fixed, seed, prog_bar if use_tqdm else None, move, formats, shuffle,
+        )
         if use_tqdm:
             prog_bar.close()
         return
@@ -180,7 +185,8 @@ def fixed(
     for class_dir in classes_dirs:
         num_items.append(
             split_class_dir_fixed(
-                class_dir, output, fixed, seed, prog_bar if use_tqdm else None, group_prefix, group, move, formats
+                class_dir, output, fixed, seed, prog_bar if use_tqdm else None,
+                group_prefix, group, move, formats, shuffle,
             )
         )
 
@@ -222,7 +228,10 @@ def fixed(
                     shutil.copy2(str(f_orig), str(f_dest))
 
 
-def kfold(input, output="output", seed=1337, k=5, group_prefix=None, group=None, move="symlink", formats=None):
+def kfold(
+    input, output="output", seed=1337, k=5, group_prefix=None, group=None,
+    move="symlink", formats=None, shuffle=True,
+):
     if k < 2:
         raise ValueError("`k` must be 2 or greater.")
 
@@ -233,24 +242,26 @@ def kfold(input, output="output", seed=1337, k=5, group_prefix=None, group=None,
         prog_bar = tqdm(desc="Copying files", unit=" files")
 
     if group == "sibling":
-        split_sibling_dirs_kfold(input, output, k, seed, prog_bar if use_tqdm else None, move, formats)
+        split_sibling_dirs_kfold(
+            input, output, k, seed, prog_bar if use_tqdm else None, move, formats, shuffle,
+        )
     else:
         for class_dir in list_dirs(input):
             split_class_dir_kfold(
                 class_dir, output, k, seed, prog_bar if use_tqdm else None,
-                group_prefix, group, move, formats,
+                group_prefix, group, move, formats, shuffle,
             )
 
     if use_tqdm:
         prog_bar.close()
 
 
-def split_class_dir_kfold(class_dir, output, k, seed, prog_bar, group_prefix, group, move, formats):
+def split_class_dir_kfold(class_dir, output, k, seed, prog_bar, group_prefix, group, move, formats, shuffle=True):
     """
     Splits a class folder into k folds for cross-validation.
     Each fold directory gets train/ and val/ subdirectories.
     """
-    files = setup_files(class_dir, seed, group_prefix, group, formats)
+    files = setup_files(class_dir, seed, group_prefix, group, formats, shuffle)
 
     # Partition files into k roughly equal chunks
     fold_size = len(files) // k
@@ -271,25 +282,26 @@ def split_class_dir_kfold(class_dir, output, k, seed, prog_bar, group_prefix, gr
         copy_files(li, class_dir, fold_output, prog_bar, move)
 
 
-def setup_files(class_dir, seed, group_prefix=None, group=None, formats=None):
+def setup_files(class_dir, seed, group_prefix=None, group=None, formats=None, shuffle=True):
     """
-    Returns shuffled list of filenames
+    Returns sorted (and optionally shuffled) list of filenames
     """
     random.seed(seed)  # make sure its reproducible
 
     files = list_files(class_dir, formats)
     files = resolve_grouping(files, group_prefix, group)
 
     files.sort()
-    random.shuffle(files)
+    if shuffle:
+        random.shuffle(files)
     return files
 
 
-def split_class_dir_ratio(class_dir, output, ratio, seed, prog_bar, group_prefix, group, move, formats):
+def split_class_dir_ratio(class_dir, output, ratio, seed, prog_bar, group_prefix, group, move, formats, shuffle=True):
     """
     Splits a class folder
     """
-    files = setup_files(class_dir, seed, group_prefix, group, formats)
+    files = setup_files(class_dir, seed, group_prefix, group, formats, shuffle)
 
     # the data was shuffled already
     split_train_idx = int(ratio[0] * len(files))
@@ -299,11 +311,11 @@ def split_class_dir_ratio(class_dir, output, ratio, seed, prog_bar, group_prefix
     copy_files(li, class_dir, output, prog_bar, move)
 
 
-def split_class_dir_fixed(class_dir, output, fixed, seed, prog_bar, group_prefix, group, move, formats):
+def split_class_dir_fixed(class_dir, output, fixed, seed, prog_bar, group_prefix, group, move, formats, shuffle=True):
     """
     Splits a class folder and returns the total number of files
     """
-    files = setup_files(class_dir, seed, group_prefix, group, formats)
+    files = setup_files(class_dir, seed, group_prefix, group, formats, shuffle)
 
     if not len(files) >= sum(fixed):
         raise ValueError(
@@ -389,8 +401,8 @@ def copy_sibling_files(files_type, type_dir_names, output, prog_bar, move):
                 copy_fn(f, full_path)
 
 
-def split_sibling_dirs_ratio(input_dir, output, ratio, seed, prog_bar, move, formats):
-    type_dir_names, groups = setup_sibling_files(input_dir, seed, formats)
+def split_sibling_dirs_ratio(input_dir, output, ratio, seed, prog_bar, move, formats, shuffle=True):
+    type_dir_names, groups = setup_sibling_files(input_dir, seed, formats, shuffle)
 
     split_train_idx = int(ratio[0] * len(groups))
     split_val_idx = split_train_idx + int(ratio[1] * len(groups))
@@ -399,8 +411,8 @@ def split_sibling_dirs_ratio(input_dir, output, ratio, seed, prog_bar, move, for
     copy_sibling_files(li, type_dir_names, output, prog_bar, move)
 
 
-def split_sibling_dirs_fixed(input_dir, output, fixed, seed, prog_bar, move, formats):
-    type_dir_names, groups = setup_sibling_files(input_dir, seed, formats)
+def split_sibling_dirs_fixed(input_dir, output, fixed, seed, prog_bar, move, formats, shuffle=True):
+    type_dir_names, groups = setup_sibling_files(input_dir, seed, formats, shuffle)
 
     if not len(groups) >= sum(fixed):
         raise ValueError(
@@ -425,8 +437,8 @@ def split_sibling_dirs_fixed(input_dir, output, fixed, seed, prog_bar, move, for
     copy_sibling_files(li, type_dir_names, output, prog_bar, move)
 
 
-def split_sibling_dirs_kfold(input_dir, output, k, seed, prog_bar, move, formats):
-    type_dir_names, groups = setup_sibling_files(input_dir, seed, formats)
+def split_sibling_dirs_kfold(input_dir, output, k, seed, prog_bar, move, formats, shuffle=True):
+    type_dir_names, groups = setup_sibling_files(input_dir, seed, formats, shuffle)
 
     fold_size = len(groups) // k
     remainder = len(groups) % k