shuf: feature: Add --random-seed option

This adds a new option to get reproducible output from a seed. This
was already possible with --random-source, but doing that properly was
tricky and had poor performance.

Adding this option implies a commitment to keep using the exact same
algorithms in the future. For that reason we only use third-party
libraries for well-known algorithms and implement our own
distributions on top of that.

-----

As a teenager on King's Day I once used `shuf` for divination. People
paid €0.50 to enter a cramped tent and sat down next to me behind an
old netbook. I would ask their name and their sun sign and pipe this
information into `shuf --random-source=/dev/stdin`, which selected
pseudo-random dictionary words and `tee`d them into `espeak`.

If someone's name was too short `shuf` crashed with an end of file
error. --random-seed would have worked better.

Author: blyxxyz

.vscode/cspell.dictionaries/jargon.wordlist.txt

@@ -55,6 +55,7 @@ fileio
 filesystem
 filesystems
 flamegraph
+footgun
 freeram
 fsxattr
 fullblock

.vscode/cspell.dictionaries/people.wordlist.txt

@@ -37,6 +37,9 @@ Boden Garman
 Chirag B Jadwani
     Chirag
     Jadwani
+Daniel Lemire
+    Daniel
+    Lemire
 Derek Chiang
     Derek
     Chiang

Cargo.lock

@@ -356,6 +356,7 @@ phf_codegen = "0.13.1"
 platform-info = "2.0.3"
 procfs = "0.18"
 rand = { version = "0.9.0", features = ["small_rng"] }
+rand_chacha = { version = "0.9.0" }
 rand_core = "0.9.0"
 rayon = "1.10"
 regex = "1.10.4"

Cargo.toml

@@ -21,7 +21,9 @@ path = "src/shuf.rs"
 clap = { workspace = true }
 itoa = { workspace = true }
 rand = { workspace = true }
+rand_chacha = { workspace = true }
 rand_core = { workspace = true }
+sha3 = { workspace = true }
 uucore = { workspace = true }
 fluent = { workspace = true }
 

src/uu/shuf/Cargo.toml

@@ -10,6 +10,7 @@ shuf-help-echo = treat each ARG as an input line
 shuf-help-input-range = treat each number LO through HI as an input line
 shuf-help-head-count = output at most COUNT lines
 shuf-help-output = write result to FILE instead of standard output
+shuf-help-random-seed = seed with STRING for reproducible output
 shuf-help-random-source = get random bytes from FILE
 shuf-help-repeat = output lines can be repeated
 shuf-help-zero-terminated = line delimiter is NUL, not newline

src/uu/shuf/locales/en-US.ftl

@@ -1,4 +1,9 @@
-use std::io::BufRead;
+// This file is part of the uutils coreutils package.
+//
+// For the full copyright and license information, please view the LICENSE
+// file that was distributed with this source code.
+
+use std::{io::BufRead, ops::RangeInclusive};
 
 use uucore::error::{FromIo, UResult, USimpleError};
 use uucore::translate;
@@ -42,7 +47,7 @@ impl<R> RandomSourceAdapter<R> {
 }
 
 impl<R: BufRead> RandomSourceAdapter<R> {
-    pub fn get_value(&mut self, at_most: u64) -> UResult<u64> {
+    fn generate_at_most(&mut self, at_most: u64) -> UResult<u64> {
         while self.entropy < at_most {
             let buf = self
                 .reader
@@ -88,18 +93,29 @@ impl<R: BufRead> RandomSourceAdapter<R> {
             self.state %= num_possibilities;
             self.entropy %= num_possibilities;
             // I sure hope the compiler optimizes this tail call.
-            self.get_value(at_most)
+            self.generate_at_most(at_most)
         }
     }
 
+    pub fn choose_from_range(&mut self, range: RangeInclusive<u64>) -> UResult<u64> {
+        let offset = self.generate_at_most(*range.end() - *range.start())?;
+        Ok(*range.start() + offset)
+    }
+
+    pub fn choose_from_slice<T: Copy>(&mut self, vals: &[T]) -> UResult<T> {
+        assert!(!vals.is_empty());
+        let idx = self.generate_at_most(vals.len() as u64 - 1)? as usize;
+        Ok(vals[idx])
+    }
+
     pub fn shuffle<'a, T>(&mut self, vals: &'a mut [T], amount: usize) -> UResult<&'a mut [T]> {
         // Fisher-Yates shuffle.
         // TODO: GNU does something different if amount <= vals.len() and the input is stdin.
         // The order changes completely and depends on --head-count.
         // No clue what they might do differently and why.
         let amount = amount.min(vals.len());
         for idx in 0..amount {
-            let other_idx = self.get_value((vals.len() - idx - 1) as u64)? as usize + idx;
+            let other_idx = self.generate_at_most((vals.len() - idx - 1) as u64)? as usize + idx;
             vals.swap(idx, other_idx);
         }
         Ok(&mut vals[..amount])

src/uu/shuf/src/compat_random_source.rs

@@ -0,0 +1,118 @@
+// This file is part of the uutils coreutils package.
+//
+// For the full copyright and license information, please view the LICENSE
+// file that was distributed with this source code.
+
+use std::ops::RangeInclusive;
+
+use rand::{RngCore as _, SeedableRng as _};
+use rand_chacha::ChaCha12Rng;
+use sha3::{Digest as _, Sha3_256};
+
+/// Reproducible seeded random number generation.
+///
+/// The behavior should stay the same between releases, so don't change it without
+/// a very good reason.
+///
+/// # How it works
+///
+/// - Take a Unicode string as the seed.
+///
+/// - Encode this seed as UTF-8.
+///
+/// - Take the SHA3-256 hash of the encoded seed.
+///
+/// - Use that hash as the input for a [`rand_chacha`] ChaCha12 RNG.
+///   (We don't touch the nonce, so that's probably zero.)
+///
+/// - Take 64-bit samples from the RNG.
+///
+/// - Use Lemire's method to generate uniformly distributed integers and:
+///
+///   - With --repeat, use these to pick elements from ranges.
+///
+///   - Without --repeat, use these to do left-to-right modern Fisher-Yates.
+///
+///   - Or for --input-range without --repeat, do whatever NonrepeatingIterator does.
+///     (We may want to change that. Watch this space.)
+///
+/// # Why it works like this
+///
+/// - Unicode string: Greatest common denominator between platforms. Windows doesn't
+///   let you pass raw bytes as a CLI argument and that would be bad practice anyway.
+///   A decimal or hex number would work but this is much more flexible without being
+///   unmanageable.
+///
+///   (Footgun: if the user passes a filename we won't read from the file but the
+///   command will run anyway.)
+///
+/// - UTF-8: That's what Rust likes and it's the least unreasonable Unicode encoding.
+///
+/// - SHA3-256: We want to make good use of the entire user input and SHA-3 is
+///   state of the art. ChaCha12 takes a 256-bit seed.
+///
+/// - ChaCha12: [`rand`]'s default rng as of writing. Seems state of the art.
+///
+/// - 64-bit samples: We could often get away with 32-bit samples but let's keep things
+///   simple and only use one width. (There doesn't seem to be much of a performance hit.)
+///
+/// - Lemire, Fisher-Yates: These are very easy to implement and maintain ourselves.
+///   `rand` provides fancier implementations but only promises reproducibility within
+///   patch releases: <https://rust-random.github.io/book/crate-reprod.html>
+///
+///   Strictly speaking even `ChaCha12` is subject to breakage. But since it's a very
+///   specific algorithm I assume it's safe in practice.
+pub struct SeededRng(Box<ChaCha12Rng>);
+
+impl SeededRng {
+    pub fn new(seed: &str) -> Self {
+        let mut hasher = Sha3_256::new();
+        hasher.update(seed.as_bytes());
+        let seed = hasher.finalize();
+        let seed = seed.as_slice().try_into().unwrap();
+        Self(Box::new(rand_chacha::ChaCha12Rng::from_seed(seed)))
+    }
+
+    #[allow(clippy::many_single_char_names)] // use original lemire names for easy comparison
+    fn generate_at_most(&mut self, at_most: u64) -> u64 {
+        if at_most == u64::MAX {
+            return self.0.next_u64();
+        }
+
+        // https://lemire.me/blog/2019/06/06/nearly-divisionless-random-integer-generation-on-various-systems/
+        let s: u64 = at_most + 1;
+        let mut x: u64 = self.0.next_u64();
+        let mut m: u128 = u128::from(x) * u128::from(s);
+        let mut l: u64 = m as u64;
+        if l < s {
+            let t: u64 = s.wrapping_neg() % s;
+            while l < t {
+                x = self.0.next_u64();
+                m = u128::from(x) * u128::from(s);
+                l = m as u64;
+            }
+        }
+        (m >> 64) as u64
+    }
+
+    pub fn choose_from_range(&mut self, range: RangeInclusive<u64>) -> u64 {
+        let offset = self.generate_at_most(*range.end() - *range.start());
+        *range.start() + offset
+    }
+
+    pub fn choose_from_slice<T: Copy>(&mut self, vals: &[T]) -> T {
+        assert!(!vals.is_empty());
+        let idx = self.generate_at_most(vals.len() as u64 - 1) as usize;
+        vals[idx]
+    }
+
+    pub fn shuffle<'a, T>(&mut self, vals: &'a mut [T], amount: usize) -> &'a mut [T] {
+        // Fisher-Yates shuffle.
+        let amount = amount.min(vals.len());
+        for idx in 0..amount {
+            let other_idx = self.generate_at_most((vals.len() - idx - 1) as u64) as usize + idx;
+            vals.swap(idx, other_idx);
+        }
+        &mut vals[..amount]
+    }
+}

src/uu/shuf/src/random_seed.rs

@@ -5,25 +5,32 @@
 
 // spell-checker:ignore (ToDO) cmdline evec nonrepeating seps shufable rvec fdata
 
-use clap::builder::ValueParser;
-use clap::{Arg, ArgAction, Command};
-use rand::Rng;
-use rand::seq::{IndexedRandom, SliceRandom};
 use std::ffi::{OsStr, OsString};
 use std::fs::File;
 use std::io::{BufReader, BufWriter, Error, Read, Write, stdin, stdout};
 use std::ops::RangeInclusive;
 use std::path::{Path, PathBuf};
 use std::str::FromStr;
+
+use clap::{Arg, ArgAction, Command, builder::ValueParser};
+use rand::rngs::ThreadRng;
+use rand::{
+    Rng,
+    seq::{IndexedRandom, SliceRandom},
+};
+
 use uucore::display::{OsWrite, Quotable};
 use uucore::error::{FromIo, UResult, USimpleError, UUsageError};
 use uucore::format_usage;
 use uucore::translate;
 
 mod compat_random_source;
 mod nonrepeating_iterator;
+mod random_seed;
 
+use compat_random_source::RandomSourceAdapter;
 use nonrepeating_iterator::NonrepeatingIterator;
+use random_seed::SeededRng;
 
 enum Mode {
     Default(PathBuf),
@@ -36,17 +43,24 @@ const BUF_SIZE: usize = 64 * 1024;
 struct Options {
     head_count: u64,
     output: Option<PathBuf>,
-    random_source: Option<PathBuf>,
+    random_source: RandomSource,
     repeat: bool,
     sep: u8,
 }
 
+enum RandomSource {
+    None,
+    Seed(String),
+    File(PathBuf),
+}
+
 mod options {
     pub static ECHO: &str = "echo";
     pub static INPUT_RANGE: &str = "input-range";
     pub static HEAD_COUNT: &str = "head-count";
     pub static OUTPUT: &str = "output";
     pub static RANDOM_SOURCE: &str = "random-source";
+    pub static RANDOM_SEED: &str = "random-seed";
     pub static REPEAT: &str = "repeat";
     pub static ZERO_TERMINATED: &str = "zero-terminated";
     pub static FILE_OR_ARGS: &str = "file-or-args";
@@ -80,6 +94,14 @@ pub fn uumain(args: impl uucore::Args) -> UResult<()> {
         Mode::Default(file.into())
     };
 
+    let random_source = if let Some(filename) = matches.get_one(options::RANDOM_SOURCE).cloned() {
+        RandomSource::File(filename)
+    } else if let Some(seed) = matches.get_one(options::RANDOM_SEED).cloned() {
+        RandomSource::Seed(seed)
+    } else {
+        RandomSource::None
+    };
+
     let options = Options {
         // GNU shuf takes the lowest value passed, so we imitate that.
         // It's probably a bug or an implementation artifact though.
@@ -92,7 +114,7 @@ pub fn uumain(args: impl uucore::Args) -> UResult<()> {
             .min()
             .unwrap_or(u64::MAX),
         output: matches.get_one(options::OUTPUT).cloned(),
-        random_source: matches.get_one(options::RANDOM_SOURCE).cloned(),
+        random_source,
         repeat: matches.get_flag(options::REPEAT),
         sep: if matches.get_flag(options::ZERO_TERMINATED) {
             b'\0'
@@ -120,14 +142,15 @@ pub fn uumain(args: impl uucore::Args) -> UResult<()> {
     }
 
     let mut rng = match options.random_source {
-        Some(ref r) => {
+        RandomSource::None => WrappedRng::Default(rand::rng()),
+        RandomSource::Seed(ref seed) => WrappedRng::Seed(SeededRng::new(seed)),
+        RandomSource::File(ref r) => {
             let file = File::open(r).map_err_context(
                 || translate!("shuf-error-failed-to-open-random-source", "file" => r.quote()),
             )?;
             let file = BufReader::new(file);
-            WrappedRng::RngFile(compat_random_source::RandomSourceAdapter::new(file))
+            WrappedRng::File(compat_random_source::RandomSourceAdapter::new(file))
         }
-        None => WrappedRng::RngDefault(rand::rng()),
     };
 
     match mode {
@@ -191,6 +214,15 @@ pub fn uu_app() -> Command {
                 .value_parser(ValueParser::path_buf())
                 .value_hint(clap::ValueHint::FilePath),
         )
+        .arg(
+            Arg::new(options::RANDOM_SEED)
+                .long(options::RANDOM_SEED)
+                .value_name("STRING")
+                .help(translate!("shuf-help-random-seed"))
+                .value_parser(ValueParser::string())
+                .value_hint(clap::ValueHint::Other)
+                .conflicts_with(options::RANDOM_SOURCE),
+        )
         .arg(
             Arg::new(options::RANDOM_SOURCE)
                 .long(options::RANDOM_SOURCE)
@@ -402,36 +434,33 @@ fn parse_range(input_range: &str) -> Result<RangeInclusive<u64>, String> {
 }
 
 enum WrappedRng {
-    RngDefault(rand::rngs::ThreadRng),
-    RngFile(compat_random_source::RandomSourceAdapter<BufReader<File>>),
+    Default(ThreadRng),
+    Seed(SeededRng),
+    File(RandomSourceAdapter<BufReader<File>>),
 }
 
 impl WrappedRng {
     fn choose<T: Copy>(&mut self, vals: &[T]) -> UResult<T> {
         match self {
-            Self::RngDefault(rng) => Ok(*vals.choose(rng).unwrap()),
-            Self::RngFile(adapter) => {
-                assert!(!vals.is_empty());
-                let idx = adapter.get_value(vals.len() as u64 - 1)? as usize;
-                Ok(vals[idx])
-            }
+            Self::Default(rng) => Ok(*vals.choose(rng).unwrap()),
+            Self::Seed(rng) => Ok(rng.choose_from_slice(vals)),
+            Self::File(rng) => rng.choose_from_slice(vals),
         }
     }
 
     fn shuffle<'a, T>(&mut self, vals: &'a mut [T], amount: usize) -> UResult<&'a mut [T]> {
         match self {
-            Self::RngDefault(rng) => Ok(vals.partial_shuffle(rng, amount).0),
-            Self::RngFile(adapter) => adapter.shuffle(vals, amount),
+            Self::Default(rng) => Ok(vals.partial_shuffle(rng, amount).0),
+            Self::Seed(rng) => Ok(rng.shuffle(vals, amount)),
+            Self::File(rng) => rng.shuffle(vals, amount),
         }
     }
 
     fn choose_from_range(&mut self, range: RangeInclusive<u64>) -> UResult<u64> {
         match self {
-            Self::RngDefault(rng) => Ok(rng.random_range(range)),
-            Self::RngFile(adapter) => {
-                let offset = adapter.get_value(*range.end() - *range.start())?;
-                Ok(*range.start() + offset)
-            }
+            Self::Default(rng) => Ok(rng.random_range(range)),
+            Self::Seed(rng) => Ok(rng.choose_from_range(range)),
+            Self::File(rng) => rng.choose_from_range(range),
         }
     }
 }