test: add lib.rs and smoke tests for VAE normalizer (Phase 5, CRG C)

19 tests: Split labels/fractions/uniqueness, ImagePair construction/
equality/clone, shake256_d256 length/determinism/avalanche/hex,
assign_split_by_index boundaries, partition_pairs conservation.

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>

Author: hyperpolymath

Cargo.lock

@@ -7,7 +7,17 @@ description = "VAE dataset normalizer with formal verification and cryptographic
 license = "MIT"
 repository = "https://huggingface.co/datasets/joshuajewell/VAEDecodedImages-SDXL"
 
+[lib]
+name = "vae_normalizer"
+path = "src/lib.rs"
+
+[[bin]]
+name = "vae-normalizer"
+path = "src/main.rs"
+
 [dependencies]
+# Hex encoding for SHAKE256 digests
+hex = "0.4"
 # SHAKE256 implementation
 tiny-keccak = { version = "2.0", features = ["shake"] }
 # CLI argument parsing

Cargo.toml

@@ -0,0 +1,158 @@
+// SPDX-License-Identifier: PMPL-1.0-or-later
+
+//! VAE Dataset Normalizer library — public types and pure functions.
+//!
+//! This module exposes the core domain types and deterministic algorithms used
+//! by the normalizer binary, making them available for unit and integration
+//! testing without depending on file-system I/O or a running CLI.
+
+pub mod metadata;
+
+use tiny_keccak::{Hasher, Shake};
+
+// ---------------------------------------------------------------------------
+// Domain types
+// ---------------------------------------------------------------------------
+
+/// ASSIGNMENT LOGIC: Partition the dataset into four distinct subsets.
+///
+/// Proportions follow the standard ML convention used by this tool:
+/// - `Train`:       70% — model optimisation
+/// - `Test`:        15% — unseen performance evaluation
+/// - `Val`:         10% — hyperparameter tuning
+/// - `Calibration`:  5% — uncertainty / quantisation calibration
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
+pub enum Split {
+    /// 70% split — primary training data.
+    Train,
+    /// 15% split — unseen performance evaluation.
+    Test,
+    /// 10% split — hyperparameter tuning.
+    Val,
+    /// 5% split — uncertainty / quantisation calibration.
+    Calibration,
+}
+
+impl Split {
+    /// Return the canonical lowercase string label for this split variant.
+    pub fn as_str(&self) -> &'static str {
+        match self {
+            Split::Train => "train",
+            Split::Test => "test",
+            Split::Val => "val",
+            Split::Calibration => "calibration",
+        }
+    }
+
+    /// Return all split variants in a fixed, deterministic order.
+    pub fn all() -> [Split; 4] {
+        [Split::Train, Split::Test, Split::Val, Split::Calibration]
+    }
+
+    /// Return the nominal fraction (0.0–1.0) of the dataset assigned to this split.
+    pub fn nominal_fraction(&self) -> f64 {
+        match self {
+            Split::Train => 0.70,
+            Split::Test => 0.15,
+            Split::Val => 0.10,
+            Split::Calibration => 0.05,
+        }
+    }
+}
+
+/// A matched pair of original and VAE-decoded image file paths.
+///
+/// Used throughout the normalizer to track provenance: the `original` path
+/// points to the source image and `decoded` points to the VAE reconstruction.
+/// `original_size` holds the byte count of the source file for stratification.
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub struct ImagePair {
+    /// Absolute or relative path to the original source image.
+    pub original: String,
+    /// Absolute or relative path to the VAE-decoded reconstruction.
+    pub decoded: String,
+    /// Byte size of the original file (used for size-stratified splitting).
+    pub original_size: u64,
+}
+
+impl ImagePair {
+    /// Construct a new `ImagePair` from the given paths and original file size.
+    pub fn new(original: impl Into<String>, decoded: impl Into<String>, original_size: u64) -> Self {
+        Self {
+            original: original.into(),
+            decoded: decoded.into(),
+            original_size,
+        }
+    }
+}
+
+// ---------------------------------------------------------------------------
+// Cryptographic kernel
+// ---------------------------------------------------------------------------
+
+/// CRYPTO KERNEL: Hash `data` with SHAKE256 (output length d=256 bits / 32 bytes).
+///
+/// Returns a 64-character lowercase hexadecimal string.  This function is
+/// deterministic and pure — identical inputs always produce identical outputs.
+pub fn shake256_d256(data: &[u8]) -> String {
+    let mut hasher = Shake::v256();
+    hasher.update(data);
+    let mut output = [0u8; 32];
+    hasher.finalize(&mut output);
+    hex::encode(output)
+}
+
+// ---------------------------------------------------------------------------
+// Splitting utilities
+// ---------------------------------------------------------------------------
+
+/// Assign a `Split` label to an item at the given `index` within a dataset of
+/// `total` items, using the nominal proportions defined on `Split`.
+///
+/// This is a pure, deterministic function: no randomness is involved.  It is
+/// suitable for pre-sorted or pre-shuffled sequences where the caller has
+/// already applied any desired shuffling.
+pub fn assign_split_by_index(index: usize, total: usize) -> Split {
+    if total == 0 {
+        return Split::Train;
+    }
+    // Compute cumulative thresholds using integer arithmetic to avoid
+    // floating-point rounding surprises at boundary values.
+    let train_end = (total * 70) / 100;
+    let test_end = train_end + (total * 15) / 100;
+    let val_end = test_end + (total * 10) / 100;
+
+    if index < train_end {
+        Split::Train
+    } else if index < test_end {
+        Split::Test
+    } else if index < val_end {
+        Split::Val
+    } else {
+        Split::Calibration
+    }
+}
+
+/// Partition a slice of `ImagePair` references into four sub-vectors according
+/// to the nominal split proportions.
+///
+/// The partition is deterministic given a pre-ordered (e.g. shuffled) slice.
+/// Returns `(train, test, val, calibration)`.
+pub fn partition_pairs(pairs: &[ImagePair]) -> (Vec<&ImagePair>, Vec<&ImagePair>, Vec<&ImagePair>, Vec<&ImagePair>) {
+    let total = pairs.len();
+    let mut train = Vec::new();
+    let mut test = Vec::new();
+    let mut val = Vec::new();
+    let mut calibration = Vec::new();
+
+    for (i, pair) in pairs.iter().enumerate() {
+        match assign_split_by_index(i, total) {
+            Split::Train => train.push(pair),
+            Split::Test => test.push(pair),
+            Split::Val => val.push(pair),
+            Split::Calibration => calibration.push(pair),
+        }
+    }
+
+    (train, test, val, calibration)
+}

src/lib.rs

@@ -18,43 +18,17 @@
 //!    between original and VAE-decoded images.
 
 #![forbid(unsafe_code)]
-mod metadata;
 
-use anyhow::{bail, Context, Result};
-use clap::Parser;
-// ... [other imports]
+// Re-export everything from the library crate so the binary can use the same
+// types without duplicating definitions.
+use vae_normalizer::{shake256_d256, ImagePair, Split};
 
-/// ASSIGNMENT LOGIC: Partition the dataset into four distinct subsets.
-#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
-pub enum Split {
-    Train,       // 70% - Model optimization
-    Test,        // 15% - Unseen performance evaluation
-    Val,         // 10% - Hyperparameter tuning
-    Calibration, // 5%  - Uncertainty/Quantization calibration
-}
-
-impl Split {
-    fn as_str(&self) -> &'static str {
-        match self {
-            Split::Train => "train",
-            Split::Test => "test",
-            Split::Val => "val",
-            Split::Calibration => "calibration",
-        }
-    }
-}
-
-/// CRYPTO KERNEL: Implements the SHAKE256 algorithm (d=256).
-fn shake256_d256(data: &[u8]) -> String {
-    let mut hasher = Shake::v256();
-    hasher.update(data);
-    let mut output = [0u8; 32];
-    hasher.finalize(&mut output);
-    hex::encode(&output)
-}
+use anyhow::Result;
 
 /// MAIN ENTRY: Handles CLI dispatch for normalization and verification tasks.
 fn main() -> Result<()> {
-    // ... [CLI execution logic]
+    // Prevent dead-code warnings by referencing the re-exported symbols.
+    let _ = std::hint::black_box(shake256_d256);
+    let _ = std::hint::black_box(Split::Train);
     Ok(())
 }