Add shuffle=False for time series and flat directory support (closes #44, #47)

Add shuffle parameter (default True) to ratio(), fixed(), kfold().
When False, files are split in sorted order without randomization.
Add --no-shuffle CLI flag.

Auto-detect flat input directories (no class subdirectories) and split
files directly into train/val/test without class nesting. Oversampling
is not supported with flat directories.

Author: jfilter

README.md

@@ -2,7 +2,7 @@
 
 Split folders with files (e.g. images) into **train**, **validation** and **test** (dataset) folders.
 
-The input folder should have the following format:
+The input folder should have the following format (with class subdirectories):
 
 ```
 input/
@@ -16,6 +16,15 @@ input/
     ...
 ```
 
+Or a **flat directory** without class subdirectories:
+
+```
+input/
+    file1.jpg
+    file2.jpg
+    ...
+```
+
 In order to give you this:
 
 ```
@@ -47,6 +56,7 @@ This should get you started to do some serious deep learning on your data. [Read
 
 -   Split files into a training set and a validation set (and optionally a test set).
 -   Works on any file types.
+-   Supports both class-based directory structures and flat directories.
 -   The files get shuffled (can be disabled for time series data).
 -   A [seed](https://docs.python.org/3/library/random.html#random.seed) makes splits reproducible.
 -   Allows randomized [oversampling](https://en.wikipedia.org/wiki/Oversampling_and_undersampling_in_data_analysis) for imbalanced datasets.
@@ -113,6 +123,28 @@ splitfolders.ratio("input_folder", output="output",
     ratio=(.8, .1, .1), shuffle=False)
 ```
 
+### Flat directories
+
+If your input folder contains files directly (no class subdirectories), `splitfolders` auto-detects this and splits files into `train/`, `val/`, `test/` without creating class subfolders:
+
+```python
+# input_folder/ contains file1.jpg, file2.jpg, ... (no subdirs)
+splitfolders.ratio("input_folder", output="output", ratio=(.8, .1, .1))
+```
+
+Output:
+```
+output/
+    train/
+        file1.jpg
+        ...
+    val/
+        file5.jpg
+        ...
+```
+
+> **Note:** Oversampling is not available with flat directories (there are no classes to balance).
+
 ### Grouping files
 
 When your data has multiple files per sample (e.g. an image and its annotation), you need to keep them together during the split. There are several ways to do this.

splitfolders/split.py

@@ -48,7 +48,7 @@
     use_tqdm = False
 
 
-def check_input_format(input):
+def check_input_format(input, allow_flat=False):
     p_input = Path(input)
     if not p_input.exists():
         err_msg = f'The provided input folder "{input}" does not exists.'
@@ -60,14 +60,19 @@ def check_input_format(input):
         raise ValueError(f'The provided input folder "{input}" is not a directory')
 
     dirs = list_dirs(input)
-    if len(dirs) == 0:
+    if len(dirs) == 0 and not allow_flat:
         raise ValueError(
             f'The input data is not in a right format. Within your folder "{input}"'
             " there are no directories. Consult the documentation how to the folder"
             " structure should look like."
         )
 
 
+def _is_flat(input):
+    """Returns True if the input directory has no subdirectories (flat file layout)."""
+    return len(list_dirs(input)) == 0
+
+
 def _get_copy_fn(move):
     """Return a function(src, dst_dir) that copies/moves/symlinks a file into dst_dir."""
     if move is True or move == "move":
@@ -114,7 +119,7 @@ def ratio(
     if len(ratio) not in (2, 3):
         raise ValueError("`ratio` should")
 
-    check_input_format(input)
+    check_input_format(input, allow_flat=(group != "sibling"))
     valid_extensions(formats)
 
     if use_tqdm:
@@ -124,6 +129,11 @@ def ratio(
         split_sibling_dirs_ratio(
             input, output, ratio, seed, prog_bar if use_tqdm else None, move, formats, shuffle,
         )
+    elif _is_flat(input):
+        split_flat_dir_ratio(
+            input, output, ratio, seed, prog_bar if use_tqdm else None,
+            group_prefix, group, move, formats, shuffle,
+        )
     else:
         for class_dir in list_dirs(input):
             split_class_dir_ratio(
@@ -139,12 +149,17 @@ def fixed(
     input, output="output", seed=1337, fixed=(100, 100), oversample=False,
     group_prefix=None, group=None, move=False, formats=None, shuffle=True,
 ):
-    check_input_format(input)
+    check_input_format(input, allow_flat=(group != "sibling"))
     valid_extensions(formats)
 
     if group == "sibling" and oversample:
         raise ValueError("Cannot use `oversample=True` with `group='sibling'` (no classes to balance).")
 
+    flat = _is_flat(input) and group != "sibling"
+
+    if flat and oversample:
+        raise ValueError("Cannot use `oversample=True` with a flat input directory (no classes to balance).")
+
     if fixed == "auto":
         if not oversample:
             raise ValueError(
@@ -180,6 +195,15 @@ def fixed(
             prog_bar.close()
         return
 
+    if flat:
+        split_flat_dir_fixed(
+            input, output, fixed, seed, prog_bar if use_tqdm else None,
+            group_prefix, group, move, formats, shuffle,
+        )
+        if use_tqdm:
+            prog_bar.close()
+        return
+
     classes_dirs = list_dirs(input)
     num_items = []
     for class_dir in classes_dirs:
@@ -235,7 +259,7 @@ def kfold(
     if k < 2:
         raise ValueError("`k` must be 2 or greater.")
 
-    check_input_format(input)
+    check_input_format(input, allow_flat=(group != "sibling"))
     valid_extensions(formats)
 
     if use_tqdm:
@@ -245,6 +269,11 @@ def kfold(
         split_sibling_dirs_kfold(
             input, output, k, seed, prog_bar if use_tqdm else None, move, formats, shuffle,
         )
+    elif _is_flat(input):
+        split_flat_dir_kfold(
+            input, output, k, seed, prog_bar if use_tqdm else None,
+            group_prefix, group, move, formats, shuffle,
+        )
     else:
         for class_dir in list_dirs(input):
             split_class_dir_kfold(
@@ -384,6 +413,85 @@ def copy_files(files_type, class_dir, output, prog_bar, move):
                 copy_fn(f, full_path)
 
 
+def copy_files_flat(files_type, output, prog_bar, move):
+    """
+    Copies files into output/split_name/ without class subdirectories.
+    """
+    copy_fn = _get_copy_fn(move)
+
+    for files, folder_type in files_type:
+        full_path = Path(output) / folder_type
+
+        full_path.mkdir(parents=True, exist_ok=True)
+        for f in files:
+            if prog_bar is not None:
+                prog_bar.update()
+            if isinstance(f, tuple):
+                for x in f:
+                    copy_fn(x, full_path)
+            else:
+                copy_fn(f, full_path)
+
+
+def split_flat_dir_ratio(input_dir, output, ratio, seed, prog_bar, group_prefix, group, move, formats, shuffle=True):
+    """Splits a flat directory (no class subdirs)."""
+    files = setup_files(input_dir, seed, group_prefix, group, formats, shuffle)
+
+    split_train_idx = int(ratio[0] * len(files))
+    split_val_idx = split_train_idx + int(ratio[1] * len(files))
+
+    li = split_files(files, split_train_idx, split_val_idx, len(ratio) == 3)
+    copy_files_flat(li, output, prog_bar, move)
+
+
+def split_flat_dir_fixed(input_dir, output, fixed, seed, prog_bar, group_prefix, group, move, formats, shuffle=True):
+    """Splits a flat directory with fixed counts."""
+    files = setup_files(input_dir, seed, group_prefix, group, formats, shuffle)
+
+    if not len(files) >= sum(fixed):
+        raise ValueError(
+            f"Not enough files for the requested fixed split."
+            f" There are only {len(files)} files but fixed={fixed} requires at least {sum(fixed)}."
+        )
+
+    if len(fixed) <= 2:
+        split_train_idx = len(files) - sum(fixed)
+        split_val_idx = split_train_idx + fixed[0]
+    else:
+        split_train_idx = fixed[0]
+        split_val_idx = fixed[0] + fixed[1]
+
+    li = split_files(
+        files,
+        split_train_idx,
+        split_val_idx,
+        len(fixed) >= 2,
+        None if len(fixed) != 3 else fixed[2],
+    )
+    copy_files_flat(li, output, prog_bar, move)
+
+
+def split_flat_dir_kfold(input_dir, output, k, seed, prog_bar, group_prefix, group, move, formats, shuffle=True):
+    """Splits a flat directory into k folds."""
+    files = setup_files(input_dir, seed, group_prefix, group, formats, shuffle)
+
+    fold_size = len(files) // k
+    remainder = len(files) % k
+    partitions = []
+    idx = 0
+    for i in range(k):
+        size = fold_size + (1 if i < remainder else 0)
+        partitions.append(files[idx : idx + size])
+        idx += size
+
+    for i in range(k):
+        val_files = partitions[i]
+        train_files = [f for j, part in enumerate(partitions) if j != i for f in part]
+        fold_output = str(Path(output) / f"fold_{i + 1}")
+        li = [(train_files, "train"), (val_files, "val")]
+        copy_files_flat(li, fold_output, prog_bar, move)
+
+
 def copy_sibling_files(files_type, type_dir_names, output, prog_bar, move):
     """
     Copies files in sibling mode: each group is a tuple of files across type dirs.