shuffle cli command

telmomenezes · telmomenezes · commit ece0141f7113 · 2026-06-09T10:36:53.000+02:00
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -15,7 +15,7 @@
 - built-in REPL setting `check_badness` to render a badness panel after each parse, available regardless of the active parser plugin.
 - parser plugin settings have an optional flag to indicate that no parser reload is nodes, REPL adjusted to accommodate this mechanism.
 - functional pattern `deep`.
-- CLI `stats` and `dedup` commands.
+- CLI `stats`, `dedup`, and `shuffle` commands.
 
 ### Changed
 
diff --git a/src/hyperbase/cli/__init__.py b/src/hyperbase/cli/__init__.py
@@ -211,6 +211,35 @@ def main() -> None:
         help="Overwrite the input file in place",
     )
 
+    # --- shuffle subcommand ------------------------------------------------
+    shuffle_parser = subparsers.add_parser(
+        "shuffle",
+        help="Randomly shuffle the lines of a JSONL parse-results file",
+    )
+    shuffle_parser.add_argument(
+        "file",
+        type=str,
+        help="Path to a .jsonl parse-results file",
+    )
+    shuffle_parser.add_argument(
+        "-o",
+        "--output",
+        type=str,
+        default=None,
+        help="Output .jsonl path (required unless --in-place)",
+    )
+    shuffle_parser.add_argument(
+        "--in-place",
+        action="store_true",
+        help="Overwrite the input file in place",
+    )
+    shuffle_parser.add_argument(
+        "--seed",
+        type=int,
+        default=None,
+        help="Random seed for a reproducible shuffle (default: random)",
+    )
+
     # Dynamically inject parser-specific args, derived from the active
     # parser's ``accepted_params()``. We do this in two passes so that
     # plugin packages stay the source of truth for their CLI surface.
@@ -254,6 +283,12 @@ def main() -> None:
         run_dedup(args)
         sys.exit(0)
 
+    if args.command == "shuffle":
+        from hyperbase.cli.shuffle import run_shuffle
+
+        run_shuffle(args)
+        sys.exit(0)
+
     if args.command == "repl":
         from hyperbase.cli.repl import run_repl
 
diff --git a/src/hyperbase/cli/_io.py b/src/hyperbase/cli/_io.py
@@ -0,0 +1,54 @@
+import argparse
+import contextlib
+import os
+import sys
+import tempfile
+from collections.abc import Iterator
+from contextlib import contextmanager
+from pathlib import Path
+from typing import IO
+
+
+def resolve_io(args: argparse.Namespace) -> tuple[Path, Path]:
+    """Validate a transform command's args and return ``(src, dest)``.
+
+    Shared by parses-file -> parses-file commands (``dedup``, ``shuffle``): the
+    input ``file`` must exist, and exactly one of ``-o/--output`` or
+    ``--in-place`` must be given. Exits with a stderr message on any violation.
+    """
+    src = Path(args.file).expanduser()
+    if not src.is_file():
+        print(f"Error: file not found: {src}", file=sys.stderr)
+        sys.exit(1)
+
+    in_place = bool(getattr(args, "in_place", False))
+    output = getattr(args, "output", None)
+    if in_place and output:
+        print("Error: use either -o/--output or --in-place, not both", file=sys.stderr)
+        sys.exit(1)
+    if not in_place and not output:
+        print("Error: specify -o/--output <path> or --in-place", file=sys.stderr)
+        sys.exit(1)
+
+    dest = src if in_place else Path(output).expanduser()
+    dest.parent.mkdir(parents=True, exist_ok=True)
+    return src, dest
+
+
+@contextmanager
+def atomic_write(dest: Path) -> Iterator[IO[str]]:
+    """Yield a text handle that is atomically moved onto *dest* on success.
+
+    Writes go to a temp file in *dest*'s directory; on clean exit it is
+    ``os.replace``-d onto *dest* -- atomic, and safe even when *dest* is also the
+    input being read (``--in-place``). On any exception the temp file is removed.
+    """
+    fd, tmp_name = tempfile.mkstemp(dir=dest.parent, suffix=".tmp")
+    try:
+        with os.fdopen(fd, "w", encoding="utf-8") as fout:
+            yield fout
+        os.replace(tmp_name, dest)
+    except BaseException:
+        with contextlib.suppress(OSError):
+            os.unlink(tmp_name)
+        raise
diff --git a/src/hyperbase/cli/dedup.py b/src/hyperbase/cli/dedup.py
@@ -1,10 +1,8 @@
 import argparse
-import contextlib
 import json
-import os
 import sys
-import tempfile
-from pathlib import Path
+
+from hyperbase.cli._io import atomic_write, resolve_io
 
 
 def run_dedup(args: argparse.Namespace) -> None:
@@ -14,60 +12,34 @@ def run_dedup(args: argparse.Namespace) -> None:
     key used by ``stats`` duplicate detection) and drops the rest. Writes either
     to ``-o/--output`` or back over the input with ``--in-place``.
     """
-    path = Path(args.file).expanduser()
-    if not path.is_file():
-        print(f"Error: file not found: {path}", file=sys.stderr)
-        sys.exit(1)
-
-    in_place = bool(getattr(args, "in_place", False))
-    output = getattr(args, "output", None)
-    if in_place and output:
-        print("Error: use either -o/--output or --in-place, not both", file=sys.stderr)
-        sys.exit(1)
-    if not in_place and not output:
-        print("Error: specify -o/--output <path> or --in-place", file=sys.stderr)
-        sys.exit(1)
-
-    dest = path if in_place else Path(output).expanduser()
-    dest.parent.mkdir(parents=True, exist_ok=True)
+    src, dest = resolve_io(args)
 
     total = kept = duplicates = malformed = 0
     seen: set[str] = set()
 
-    # Stream to a temp file in the destination directory, then atomically
-    # replace — so --in-place never reads and overwrites the same file at once,
-    # and an interrupted run never leaves a half-written output.
-    fd, tmp_name = tempfile.mkstemp(dir=dest.parent, suffix=".tmp")
-    try:
-        with (
-            open(path, encoding="utf-8") as fin,
-            os.fdopen(fd, "w", encoding="utf-8") as fout,
-        ):
-            for line in fin:
-                stripped = line.strip()
-                if not stripped:
-                    continue
-                total += 1
-                # Key on the raw ``text`` field only. We deliberately do NOT
-                # reconstruct a ParseResult: that would run hedge() on the edge
-                # string and drop an otherwise-valid line whose edge fails to
-                # parse. Kept lines are written verbatim (byte-lossless).
-                try:
-                    text = json.loads(stripped)["text"]
-                except Exception:
-                    malformed += 1
-                    continue
-                if text in seen:
-                    duplicates += 1
-                    continue
-                seen.add(text)
-                kept += 1
-                fout.write(stripped + "\n")
-        os.replace(tmp_name, dest)
-    except BaseException:
-        with contextlib.suppress(OSError):
-            os.unlink(tmp_name)
-        raise
+    # ``atomic_write`` is the outer context so the input is closed before the
+    # final os.replace (matters for --in-place, where dest is the input).
+    with atomic_write(dest) as fout, open(src, encoding="utf-8") as fin:
+        for line in fin:
+            stripped = line.strip()
+            if not stripped:
+                continue
+            total += 1
+            # Key on the raw ``text`` field only. We deliberately do NOT
+            # reconstruct a ParseResult: that would run hedge() on the edge
+            # string and drop an otherwise-valid line whose edge fails to
+            # parse. Kept lines are written verbatim (byte-lossless).
+            try:
+                text = json.loads(stripped)["text"]
+            except Exception:
+                malformed += 1
+                continue
+            if text in seen:
+                duplicates += 1
+                continue
+            seen.add(text)
+            kept += 1
+            fout.write(stripped + "\n")
 
     print(f"Total lines:   {total}", file=sys.stderr)
     print(f"Kept (unique): {kept}", file=sys.stderr)
diff --git a/src/hyperbase/cli/shuffle.py b/src/hyperbase/cli/shuffle.py
@@ -0,0 +1,36 @@
+import argparse
+import random
+import sys
+
+from hyperbase.cli._io import atomic_write, resolve_io
+
+
+def run_shuffle(args: argparse.Namespace) -> None:
+    """Randomly shuffle the lines of a JSONL parse-results file.
+
+    Lines are treated as opaque text -- nothing is parsed or dropped except
+    blank lines -- so the original ordering is the only thing changed. Writes
+    either to ``-o/--output`` or back over the input with ``--in-place``. Pass
+    ``--seed`` for a reproducible ordering.
+    """
+    src, dest = resolve_io(args)
+
+    lines: list[str] = []
+    with open(src, encoding="utf-8") as fin:
+        for line in fin:
+            stripped = line.strip()
+            if stripped:
+                lines.append(stripped)
+
+    # Random(None) seeds from OS entropy (random each run); Random(int) is
+    # reproducible -- so no conditional is needed here.
+    random.Random(getattr(args, "seed", None)).shuffle(lines)
+
+    with atomic_write(dest) as fout:
+        for line in lines:
+            fout.write(line + "\n")
+
+    print(f"Lines shuffled: {len(lines)}", file=sys.stderr)
+    if getattr(args, "seed", None) is not None:
+        print(f"Seed:           {args.seed}", file=sys.stderr)
+    print(f"Output:         {dest}", file=sys.stderr)
diff --git a/tests/test_cli_shuffle.py b/tests/test_cli_shuffle.py
@@ -0,0 +1,105 @@
+import argparse
+
+import pytest
+
+from hyperbase.builders import hedge
+from hyperbase.cli.shuffle import run_shuffle
+from hyperbase.parsers.result import ParseResult
+
+
+def _line(text: str) -> str:
+    tokens = text.split()
+    pr = ParseResult(
+        edge=hedge("(is/P a/C b/C)"),
+        text=text,
+        tokens=tokens,
+        tok_pos=hedge("(is/P a/C b/C)"),
+    )
+    return pr.to_json()
+
+
+def _write(path, lines: list[str]) -> str:
+    path.write_text("\n".join(lines) + "\n", encoding="utf-8")
+    return str(path)
+
+
+def _ns(file, output=None, in_place=False, seed=None) -> argparse.Namespace:
+    return argparse.Namespace(file=file, output=output, in_place=in_place, seed=seed)
+
+
+def _read(path) -> list[str]:
+    return path.read_text(encoding="utf-8").splitlines()
+
+
+def test_shuffle_is_a_permutation(tmp_path, capsys):
+    lines = [_line(f"sentence number {i}") for i in range(20)]
+    src = _write(tmp_path / "in.jsonl", lines)
+    out = tmp_path / "out.jsonl"
+    run_shuffle(_ns(src, output=str(out), seed=42))
+
+    result = _read(out)
+    # Same lines, same count — only the order changed.
+    assert sorted(result) == sorted(lines)
+    assert len(result) == len(lines)
+    # A fixed seed over 20 distinct lines reorders them.
+    assert result != lines
+    assert "Lines shuffled: 20" in capsys.readouterr().err
+
+
+def test_seed_is_reproducible(tmp_path):
+    lines = [_line(f"s {i}") for i in range(15)]
+    src = _write(tmp_path / "in.jsonl", lines)
+    out_a = tmp_path / "a.jsonl"
+    out_b = tmp_path / "b.jsonl"
+    run_shuffle(_ns(src, output=str(out_a), seed=7))
+    run_shuffle(_ns(src, output=str(out_b), seed=7))
+    assert _read(out_a) == _read(out_b)
+
+
+def test_in_place_shuffles_input(tmp_path, capsys):
+    lines = [_line(f"x {i}") for i in range(12)]
+    src_path = tmp_path / "data.jsonl"
+    _write(src_path, lines)
+    run_shuffle(_ns(str(src_path), in_place=True, seed=3))
+
+    result = _read(src_path)
+    assert sorted(result) == sorted(lines)
+    err = capsys.readouterr().err
+    assert "Lines shuffled: 12" in err
+    assert "Seed:           3" in err
+
+
+def test_blank_lines_dropped(tmp_path, capsys):
+    src_path = tmp_path / "in.jsonl"
+    src_path.write_text(
+        _line("one") + "\n\n" + _line("two") + "\n   \n" + _line("three") + "\n",
+        encoding="utf-8",
+    )
+    out = tmp_path / "out.jsonl"
+    run_shuffle(_ns(str(src_path), output=str(out), seed=1))
+
+    result = _read(out)
+    assert len(result) == 3
+    assert "Lines shuffled: 3" in capsys.readouterr().err
+
+
+def test_no_seed_omits_seed_line(tmp_path, capsys):
+    src = _write(tmp_path / "in.jsonl", [_line("a"), _line("b")])
+    run_shuffle(_ns(src, output=str(tmp_path / "out.jsonl")))
+    assert "Seed:" not in capsys.readouterr().err
+
+
+def test_neither_output_nor_in_place_exits(tmp_path, capsys):
+    src = _write(tmp_path / "in.jsonl", [_line("x")])
+    with pytest.raises(SystemExit) as exc:
+        run_shuffle(_ns(src))
+    assert exc.value.code == 1
+    assert "specify -o/--output" in capsys.readouterr().err
+
+
+def test_missing_file_exits(tmp_path, capsys):
+    missing = tmp_path / "nope.jsonl"
+    with pytest.raises(SystemExit) as exc:
+        run_shuffle(_ns(str(missing), output=str(tmp_path / "out.jsonl")))
+    assert exc.value.code == 1
+    assert "file not found" in capsys.readouterr().err
