refactor: centralize cryptographic proof generation and fix Python SDK RAG paths

Author: varshith-Git

all-functions.md

@@ -7,6 +7,7 @@ use valori_kernel::types::vector::FxpVector;
 use valori_kernel::types::id::RecordId;
 use valori_kernel::fxp::ops::from_f32;
 use valori_kernel::event::KernelEvent;
+use valori_kernel::proof::generate_proof_bytes;
 use serde_json;  // For metadata serialization
 use hex;  // For hash encoding
 
@@ -379,6 +380,85 @@ impl ValoriEngine {
             Err(pyo3::exceptions::PyRuntimeError::new_err("Event Log not initialized"))
         }
     }
+
+    /// Batch atomic insert with Merkle proofs.
+    /// Returns a list of (record_id, proof_hex).
+    #[pyo3(signature = (vectors, tags))]
+    fn insert_batch_with_proof(&self, vectors: Vec<Vec<f32>>, tags: Vec<u64>) -> PyResult<Vec<(u32, String)>> {
+        if vectors.len() != tags.len() {
+            return Err(pyo3::exceptions::PyValueError::new_err("vectors and tags must have the same length"));
+        }
+
+        let mut engine = self.inner.lock().unwrap();
+        
+        let mut events = Vec::with_capacity(vectors.len());
+        let mut results = Vec::with_capacity(vectors.len());
+        let mut temp_used_ids = std::collections::HashSet::new();
+        let mut next_candidate = 0;
+
+        for (vec, &tag) in vectors.iter().zip(tags.iter()) {
+            if vec.len() != D {
+                return Err(pyo3::exceptions::PyValueError::new_err(format!("Expected {} dims", D)));
+            }
+
+            let mut fxp_vec = FxpVector::<D>::new_zeros();
+            let mut fixed_values = Vec::with_capacity(D);
+            for (i, &f) in vec.iter().enumerate() {
+                if f < -32767.0 || f > 32767.0 {
+                    return Err(pyo3::exceptions::PyValueError::new_err(format!(
+                        "Float at index {} ({}) outside valid range [-32767.0, 32767.0]", i, f
+                    )));
+                }
+                let scalar = from_f32(f);
+                fxp_vec.data[i] = scalar;
+                fixed_values.push(scalar.0);
+            }
+
+            let proof_bytes = generate_proof_bytes(&fixed_values);
+            let proof_hex = hex::encode(&proof_bytes);
+
+            let mut found_id = None;
+            for i in next_candidate..MAX_RECORDS {
+                let rid = RecordId(i as u32);
+                if engine.state.get_record(rid).is_none() && !temp_used_ids.contains(&i) {
+                    found_id = Some(rid);
+                    next_candidate = i + 1;
+                    break;
+                }
+            }
+
+            let rid = found_id.ok_or_else(|| {
+                pyo3::exceptions::PyRuntimeError::new_err("Capacity Exceeded")
+            })?;
+            temp_used_ids.insert(rid.0 as usize);
+
+            events.push(KernelEvent::InsertRecord {
+                id: rid,
+                vector: fxp_vec,
+                metadata: Some(proof_bytes),
+                tag,
+            });
+            results.push((rid.0, proof_hex));
+        }
+
+        if let Some(ref mut committer) = engine.event_committer {
+            match committer.commit_batch(events.clone()) {
+                Ok(_) => {
+                    for event in &events {
+                        engine.apply_committed_event(event).map_err(|e| {
+                            pyo3::exceptions::PyRuntimeError::new_err(format!("Apply failed: {:?}", e))
+                        })?;
+                    }
+                    Ok(results)
+                }
+                Err(e) => Err(pyo3::exceptions::PyRuntimeError::new_err(
+                    format!("Batch commit failed: {:?}", e)
+                )),
+            }
+        } else {
+            Err(pyo3::exceptions::PyRuntimeError::new_err("Event Log not initialized"))
+        }
+    }
 }
 
 // ============================================================================
@@ -405,27 +485,6 @@ fn ingest_embedding(floats: Vec<f32>) -> PyResult<Vec<i32>> {
     Ok(fixed)
 }
 
-/// Internal helper — generates Merkle proof as raw bytes.
-/// Single source of truth for Merkle logic.
-/// Used by both generate_proof() (hex output) and insert_with_proof() (Record.metadata).
-fn generate_proof_bytes(fixed_values: &[i32]) -> Vec<u8> {
-    let leaves: Vec<[u8; 32]> = fixed_values
-        .iter()
-        .enumerate()
-        .map(|(pos, &val)| {
-            let mut buf = [0u8; 8];
-            buf[..4].copy_from_slice(&(pos as u32).to_le_bytes());
-            buf[4..].copy_from_slice(&val.to_le_bytes());
-
-            let mut hasher = blake3::Hasher::new();
-            hasher.update(&buf);
-            *hasher.finalize().as_bytes()
-        })
-        .collect();
-
-    merkle_root(&leaves).to_vec()
-}
-
 /// Build a position-aware Merkle tree over Q16.16 integers.
 ///
 /// Each leaf = BLAKE3(position_u32_le || value_i32_le).
@@ -441,25 +500,6 @@ fn generate_proof(fixed_values: Vec<i32>) -> PyResult<String> {
     Ok(hex::encode(generate_proof_bytes(&fixed_values)))
 }
 
-/// Standard binary Merkle tree. Odd leaf: hashed with itself.
-fn merkle_root(leaves: &[[u8; 32]]) -> [u8; 32] {
-    if leaves.len() == 1 {
-        return leaves[0];
-    }
-
-    let next_level: Vec<[u8; 32]> = leaves
-        .chunks(2)
-        .map(|pair| {
-            let mut hasher = blake3::Hasher::new();
-            hasher.update(&pair[0]);
-            hasher.update(pair.get(1).unwrap_or(&pair[0]));
-            *hasher.finalize().as_bytes()
-        })
-        .collect();
-
-    merkle_root(&next_level)
-}
-
 /// Verify a float embedding against a claimed proof hash.
 ///
 /// Full pipeline in Rust: f32 → Q16.16 → Merkle → compare.

ffi/src/lib.rs

@@ -9,7 +9,7 @@ use valori_kernel::types::enums::{NodeKind, EdgeKind};
 use valori_kernel::snapshot::{encode::encode_state, decode::decode_state};
 // use valori_kernel::fxp::ops::from_f32; // Explicit rounding now preferred
 use valori_kernel::verify::{kernel_state_hash, snapshot_hash};
-use valori_kernel::proof::DeterministicProof;
+use valori_kernel::proof::{DeterministicProof, generate_proof_bytes};
 
 use crate::config::{NodeConfig, IndexKind, QuantizationKind};
 use crate::errors::EngineError;
@@ -180,9 +180,11 @@ impl<const MAX_RECORDS: usize, const D: usize, const MAX_NODES: usize, const MAX
         // 1. Build FxpVector for Kernel
         // STRICT DETERMINISM: Explicit Rounding to Nearest
         let mut vector = FxpVector::<D>::new_zeros();
+        let mut fixed_values = Vec::with_capacity(D);
         for (i, v) in values.iter().enumerate() {
             let fixed = (v * SCALE).round().clamp(i32::MIN as f32, i32::MAX as f32) as i32;
             vector.data[i] = FxpScalar(fixed);
+            fixed_values.push(fixed);
         }
 
         // 2. Determine ID (first free slot strategy)
@@ -199,8 +201,9 @@ impl<const MAX_RECORDS: usize, const D: usize, const MAX_NODES: usize, const MAX
         // Phase 23: Event-sourced path (preferred)
         if let Some(ref mut committer) = self.event_committer {
             let start = std::time::Instant::now();
+            let proof_bytes = generate_proof_bytes(&fixed_values);
             // Generate event (no state change yet)
-            let event = KernelEvent::InsertRecord { id, vector, metadata: None, tag: 0 };
+            let event = KernelEvent::InsertRecord { id, vector, metadata: Some(proof_bytes), tag: 0 };
 
             // Commit via event pipeline (shadow → persist → commit → live)
             // Clone event for local apply if needed
@@ -238,7 +241,8 @@ impl<const MAX_RECORDS: usize, const D: usize, const MAX_NODES: usize, const MAX
             Ok(id.0)
         } else {
             // Fallback: Legacy WAL path
-            let cmd = Command::InsertRecord { id, vector, metadata: None, tag: 0 };
+            let proof_bytes = generate_proof_bytes(&fixed_values);
+            let cmd = Command::InsertRecord { id, vector, metadata: Some(proof_bytes), tag: 0 };
 
             // Write to WAL FIRST
             if let Some(ref mut wal) = self.wal_writer {
@@ -322,12 +326,15 @@ impl<const MAX_RECORDS: usize, const D: usize, const MAX_NODES: usize, const MAX
 
                 // Create FxpVector
                 let mut vector = FxpVector::<D>::new_zeros();
+                let mut fixed_values = Vec::with_capacity(D);
                 for (i, v) in values.iter().enumerate() {
                     let fixed = (v * SCALE).round().clamp(i32::MIN as f32, i32::MAX as f32) as i32;
                     vector.data[i] = FxpScalar(fixed);
+                    fixed_values.push(fixed);
                 }
 
-                events.push(KernelEvent::InsertRecord { id, vector, metadata: None, tag: 0 });
+                let proof_bytes = generate_proof_bytes(&fixed_values);
+                events.push(KernelEvent::InsertRecord { id, vector, metadata: Some(proof_bytes), tag: 0 });
             }
 
             let start = std::time::Instant::now();

node/src/engine.rs

@@ -179,3 +179,25 @@ def test_deterministic_proof_across_engines(self, db_dir):
 
 if __name__ == "__main__":
     pytest.main([__file__, "-v"])
+
+    def test_insert_batch_with_proof(self, engine):
+        """insert_batch_with_proof returns list of tuples and stores metadata."""
+        np.random.seed(42)
+        batch = []
+        for _ in range(3):
+            emb = np.random.randn(384).astype(np.float32)
+            emb = (emb / np.linalg.norm(emb)).tolist()
+            batch.append(emb)
+
+        results = engine.insert_batch_with_proof(batch)
+        
+        assert len(results) == 3
+        for i, (rid, proof) in enumerate(results):
+            assert isinstance(rid, int)
+            assert isinstance(proof, str)
+            assert len(proof) == 64
+            
+            # Verify metadata was stored
+            meta = engine.kernel.get_metadata(rid)
+            assert meta is not None
+            assert bytes(meta).hex() == proof

python/test_insert_with_proof.py

@@ -65,6 +65,41 @@ def insert(
         self._id_map[id] = record_id
         return proof_hash
 
+    def insert_batch(
+        self,
+        ids: list[str],
+        embeddings: list[np.ndarray],
+        metadata_list: list[dict] = None
+    ) -> list[str]:
+        """
+        Insert a batch into existing DB and generate kernel-backed proofs.
+
+        Returns:
+            List of proof hashes (hex strings).
+        """
+        if metadata_list is None:
+            metadata_list = [{}] * len(ids)
+
+        # 1. Store in existing DB
+        if hasattr(self.db, "insert_batch"):
+            self.db.insert_batch(ids, embeddings, metadata_list)
+        else:
+            # Fallback for DBs without batch method
+            for i in range(len(ids)):
+                self.db.insert(ids[i], embeddings[i], metadata_list[i])
+
+        # 2. Single atomic Rust call for the batch
+        vectors = [emb.flatten().tolist() for emb in embeddings]
+        results = self._valori.insert_batch_with_proof(vectors)
+
+        # 3. Track mapping and collect proofs
+        proof_hashes = []
+        for i, (record_id, proof_hash) in enumerate(results):
+            self._id_map[ids[i]] = record_id
+            proof_hashes.append(proof_hash)
+
+        return proof_hashes
+
     def search(
         self,
         query_embedding: np.ndarray,

python/valori/adapter.py

@@ -1,33 +1,18 @@
-# Copyright (c) 2025 Varshith Gudur. Licensed under AGPLv3.
-import numpy as np
+import valori
 from typing import List, Union
 from ..protocol import ValidationError
 
-FXP_MAX = 32767.0
-FXP_MIN = -32767.0
-SCALE = 1 << 16
-
-def validate_float_range(vec: Union[List[float], np.ndarray]) -> List[float]:
+def validate_float_range(vec: Union[List[float], "np.ndarray"]) -> List[float]:
     """
-    Validates and converts a float vector to Q16.16 compatible floats.
-    Valori kernel expects floats but validates they are within safe range.
-    Here we ensure they are finite and clamped/checked.
+    Validates a float vector using the exact Rust kernel validation path.
+    Guarantees that float validation is identical across Python and Rust.
     """
-    if isinstance(vec, list):
-        vec = np.array(vec, dtype=np.float64)
-
-    if not np.isfinite(vec).all():
-        raise ValidationError("Embedding contains non-finite values (NaN/Inf)")
-    
-    if vec.ndim != 1:
-        raise ValidationError(f"Embedding must be 1D, got {vec.ndim}")
-
-    if (vec > FXP_MAX).any() or (vec < FXP_MIN).any():
-        # Option: clamp or raise? Prompt says "Reject ... > +32767" -> Raise.
-        raise ValidationError(f"Embedding components must be within [{FXP_MIN}, {FXP_MAX}]")
-
-    # Valori protocol currently takes List[float], not int raw.
-    # The kernel does conversion. We just validate range here.
-    # Prompt says "Reject if > 32767".
-    # Return list of floats.
-    return vec.tolist()
+    if hasattr(vec, "tolist"):
+        vec = vec.tolist()
+        
+    try:
+        # Call the Rust FFI single source of truth for validation
+        valori.ingest_embedding(vec)
+        return vec
+    except ValueError as e:
+        raise ValidationError(str(e))

python/valori/adapters/utils.py

@@ -23,6 +23,10 @@ def __init__(self, path: str = "./valori_db"):
     def insert(self, vector: List[float], tag: int = 0) -> int:
         return self.kernel.insert(vector, tag)
 
+    def insert_with_proof(self, vector: List[float], tag: int = 0) -> tuple:
+        """Insert a vector and return its ID and Merkle proof hash."""
+        return self.kernel.insert_with_proof(vector, tag)
+
     def search(self, query: List[float], k: int, filter_tag: Optional[int] = None) -> List[Dict[str, Any]]:
         # FFI returns [(id, score), ...]
         hits = self.kernel.search(query, k, filter_tag)
@@ -57,6 +61,20 @@ def insert_batch(self, vectors: List[List[float]]) -> List[int]:
         """
         return self.kernel.insert_batch(vectors)
 
+    def insert_batch_with_proof(self, vectors: List[List[float]], tags: Optional[List[int]] = None) -> List[tuple]:
+        """Insert multiple vectors atomically and generate a proof for each.
+        
+        Args:
+            vectors: List of vectors to insert
+            tags: Optional list of tags. If None, defaults to 0 for all vectors.
+            
+        Returns:
+            List of (record_id, proof_hash) tuples
+        """
+        if tags is None:
+            tags = [0] * len(vectors)
+        return self.kernel.insert_batch_with_proof(vectors, tags)
+    
     def get_metadata(self, record_id: int) -> Optional[bytes]:
         """Get metadata for a record.