feat: implement deterministic metadata support

- Added optional  field to
- Updated kernel state, events, and commands to handle metadata
- Implemented deterministic hashing and snapshot encoding/decoding for metadata
- Bumped snapshot schema version to 2
- Added comprehensive tests for metadata persistence, sizing, and backward compatibility
- Updated existing tests to accommodate API changes

Author: varshith-Git

src/config.rs

@@ -6,3 +6,6 @@ pub const FRAC_BITS: u32 = 16;
 
 /// Scaling factor for Fixed-Point representation (1 << FRAC_BITS).
 pub const SCALE: i32 = 1 << FRAC_BITS;
+
+/// Maximum size in bytes for record metadata.
+pub const MAX_METADATA_SIZE: usize = 65536;

src/error.rs

@@ -13,6 +13,8 @@ pub enum KernelError {
     InvalidOperation,
     /// Invalid input.
     InvalidInput,
+    /// Metadata too large.
+    MetadataTooLarge,
 }
 
 pub type KernelResult<T> = core::result::Result<T, KernelError>;

src/event.rs

@@ -19,18 +19,22 @@
 use crate::types::id::{RecordId, NodeId, EdgeId};
 use crate::types::vector::FxpVector;
 use crate::types::enums::{NodeKind, EdgeKind};
-use serde::{Serialize, Deserialize};
+use serde::{Serialize, Deserialize, Serializer, Deserializer};
+use serde::ser::{SerializeStruct, SerializeStructVariant};
+use serde::de::{self, Visitor, MapAccess, SeqAccess, EnumAccess, VariantAccess};
+use core::fmt;
 
 /// KernelEvent represents the canonical event language for Valori.
 /// This is the ONLY way to express state transitions.
 ///
 /// Each variant represents an atomic, deterministic operation on the kernel state.
-#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)]
+#[derive(Clone, Debug, PartialEq, Eq)]
 pub enum KernelEvent<const D: usize> {
     /// Insert a new vector record into the kernel
     InsertRecord {
         id: RecordId,
         vector: FxpVector<D>,
+        metadata: Option<alloc::vec::Vec<u8>>,
     },
 
     /// Delete an existing vector record from the kernel
@@ -72,6 +76,172 @@ impl<const D: usize> KernelEvent<D> {
     }
 }
 
+// Custom Serialization to support strict V2 Metadata format
+impl<const D: usize> Serialize for KernelEvent<D> {
+    fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
+    where
+        S: Serializer,
+    {
+        match self {
+            KernelEvent::InsertRecord { id, vector, metadata } => {
+                // We serialize as a struct variant with 3 fields for Serialize
+                // But specifically for metadata, we manually encode the length + bytes
+                // To achieve "No version flag", we just write the fields.
+                // Bincode enum serialization: [VariantIdx][Field1][Field2][...]
+                let mut state = serializer.serialize_struct_variant("KernelEvent", 0, "InsertRecord", 3)?;
+                state.serialize_field("id", id)?;
+                state.serialize_field("vector", vector)?;
+                
+                // Custom Metadata Serialization: u32 Len + Bytes
+                // We wrap this in a helper or just serialize a "RawMetadata" struct
+                let meta_wrapper = RawMetadata(metadata.as_ref());
+                state.serialize_field("metadata", &meta_wrapper)?;
+                
+                state.end()
+            }
+            KernelEvent::DeleteRecord { id } => {
+                let mut state = serializer.serialize_struct_variant("KernelEvent", 1, "DeleteRecord", 1)?;
+                state.serialize_field("id", id)?;
+                state.end()
+            }
+            KernelEvent::CreateNode { id, kind, record } => {
+                let mut state = serializer.serialize_struct_variant("KernelEvent", 2, "CreateNode", 3)?;
+                state.serialize_field("id", id)?;
+                state.serialize_field("kind", kind)?;
+                state.serialize_field("record", record)?;
+                state.end()
+            }
+            KernelEvent::CreateEdge { id, from, to, kind } => {
+                let mut state = serializer.serialize_struct_variant("KernelEvent", 3, "CreateEdge", 4)?;
+                state.serialize_field("id", id)?;
+                state.serialize_field("from", from)?;
+                state.serialize_field("to", to)?;
+                state.serialize_field("kind", kind)?;
+                state.end()
+            }
+            KernelEvent::DeleteEdge { id } => {
+                let mut state = serializer.serialize_struct_variant("KernelEvent", 4, "DeleteEdge", 1)?;
+                state.serialize_field("id", id)?;
+                state.end()
+            }
+        }
+    }
+}
+
+struct RawMetadata<'a>(Option<&'a alloc::vec::Vec<u8>>);
+
+impl<'a> Serialize for RawMetadata<'a> {
+    fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
+    where S: Serializer {
+        match self.0 {
+            Some(bytes) => {
+                let len = bytes.len() as u32;
+                // We can't write two fields here easily if we are just one field in the parent struct?
+                // Actually Bincode flattens structs.
+                // So if we serialize as a tuple `(len, bytes)`, it writes [len][bytes].
+                (len, bytes).serialize(serializer)
+            }
+            None => {
+                let len: u32 = 0;
+                // Write 0 length, no bytes.
+                len.serialize(serializer)
+            }
+        }
+    }
+}
+
+// Custom Deserialization
+impl<'de, const D: usize> Deserialize<'de> for KernelEvent<D> {
+    fn deserialize<Deser>(deserializer: Deser) -> Result<Self, Deser::Error>
+    where
+        Deser: Deserializer<'de>,
+    {
+        // Use a Shadow Enum that matches the structure but uses a custom type for Metadata
+        // This allows us to intercept the metadata deserialization for backward compatibility logic
+        #[derive(Serialize, Deserialize)]
+        enum KernelEventHelper<const D: usize> {
+             InsertRecord {
+                 id: RecordId,
+                 vector: FxpVector<D>,
+                 #[serde(with = "raw_metadata_serde")]
+                 metadata: Option<alloc::vec::Vec<u8>>,
+             },
+             DeleteRecord {
+                 id: RecordId,
+             },
+             CreateNode {
+                 id: NodeId,
+                 kind: NodeKind,
+                 record: Option<RecordId>,
+             },
+             CreateEdge {
+                 id: EdgeId,
+                 from: NodeId,
+                 to: NodeId,
+                 kind: EdgeKind,
+             },
+             DeleteEdge {
+                 id: EdgeId,
+             },
+        }
+        
+        // Delegate to the Helper
+        let helper = KernelEventHelper::<D>::deserialize(deserializer)?;
+        
+        Ok(match helper {
+            KernelEventHelper::InsertRecord { id, vector, metadata } => KernelEvent::InsertRecord { id, vector, metadata },
+            KernelEventHelper::DeleteRecord { id } => KernelEvent::DeleteRecord { id },
+            KernelEventHelper::CreateNode { id, kind, record } => KernelEvent::CreateNode { id, kind, record },
+            KernelEventHelper::CreateEdge { id, from, to, kind } => KernelEvent::CreateEdge { id, from, to, kind },
+            KernelEventHelper::DeleteEdge { id } => KernelEvent::DeleteEdge { id },
+        })
+    }
+}
+
+mod raw_metadata_serde {
+    use super::*;
+    use serde::{Serializer, Deserializer};
+    
+    pub fn serialize<S>(metadata: &Option<alloc::vec::Vec<u8>>, serializer: S) -> Result<S::Ok, S::Error>
+    where S: Serializer {
+        match metadata {
+            Some(bytes) => {
+                let len = bytes.len() as u32;
+                (len, bytes).serialize(serializer)
+            }
+            None => {
+                let len: u32 = 0;
+                len.serialize(serializer)
+            }
+        }
+    }
+    
+    pub fn deserialize<'de, D>(deserializer: D) -> Result<Option<alloc::vec::Vec<u8>>, D::Error>
+    where D: Deserializer<'de> {
+         struct MetadataVisitor;
+         impl<'de> Visitor<'de> for MetadataVisitor {
+             type Value = Option<alloc::vec::Vec<u8>>;
+             fn expecting(&self, formatter: &mut fmt::Formatter) -> fmt::Result {
+                 formatter.write_str("metadata length and bytes")
+             }
+             
+             fn visit_seq<A>(self, mut seq: A) -> Result<Self::Value, A::Error>
+             where A: SeqAccess<'de> {
+                 let len: u32 = seq.next_element()?.ok_or_else(|| de::Error::invalid_length(0, &self))?;
+                 
+                 if len == 0 {
+                     return Ok(None);
+                 }
+                 
+                 let bytes: alloc::vec::Vec<u8> = seq.next_element()?.ok_or_else(|| de::Error::invalid_length(1, &self))?;
+                 Ok(Some(bytes))
+             }
+         }
+         
+         deserializer.deserialize_tuple(2, MetadataVisitor)
+    }
+}
+
 #[cfg(test)]
 mod tests {
     use super::*;
@@ -82,6 +252,7 @@ mod tests {
         let event = KernelEvent::<16>::InsertRecord {
             id: RecordId(42),
             vector: FxpVector::new_zeros(),
+            metadata: Some(alloc::vec![0xAA, 0xBB]),
         };
 
         let bytes1 = bincode::serde::encode_to_vec(&event, bincode::config::standard()).unwrap();

src/replay_events.rs

@@ -152,6 +152,7 @@ mod tests {
         journal.append(KernelEvent::InsertRecord {
             id: RecordId(1),
             vector: FxpVector::new_zeros(),
+            metadata: None,
         });
 
         assert_eq!(journal.buffer_len(), 1);
@@ -171,6 +172,7 @@ mod tests {
         journal.append(KernelEvent::InsertRecord {
             id: RecordId(1),
             vector: FxpVector::new_zeros(),
+            metadata: None,
         });
 
         journal.discard_buffer();

src/snapshot/decode.rs

@@ -52,7 +52,8 @@ pub fn decode_state<const MAX_RECORDS: usize, const D: usize, const MAX_NODES: u
     offset += 4;
 
     let schema_ver = read_u32(buf, &mut offset)?;
-    if schema_ver != crate::snapshot::encode::SCHEMA_VERSION {
+    // We support V1 and V2
+    if schema_ver != 1 && schema_ver != 2 {
         return Err(KernelError::InvalidOperation); // Version mismatch
     }
 
@@ -81,16 +82,34 @@ pub fn decode_state<const MAX_RECORDS: usize, const D: usize, const MAX_NODES: u
         for i in 0..D {
             vector.data[i] = FxpScalar(read_i32(buf, &mut offset)?);
         }
+
+        // Metadata V2 logic
+        let metadata = if schema_ver >= 2 {
+            let meta_len = read_u32(buf, &mut offset)?;
+            if meta_len > 0 {
+                let len = meta_len as usize;
+                if offset + len > buf.len() {
+                    return Err(KernelError::InvalidOperation); // Truncated
+                }
+                let mut bytes = alloc::vec![0u8; len];
+                bytes.copy_from_slice(&buf[offset..offset+len]);
+                offset += len;
+                Some(bytes)
+            } else {
+                None
+            }
+        } else {
+            None
+        };
 
-        // Validation of ID bounds is implicit via array access check or capacity check above?
-        // But id_val could be > MAX_RECORDS inside the byte stream even if cap matches.
         let idx = id_val as usize;
         if idx >= MAX_RECORDS {
             return Err(KernelError::CapacityExceeded);
         }
         state.records.records[idx] = Some(Record {
             id: RecordId(id_val),
             vector,
+            metadata,
             flags,
         });
     }

src/snapshot/encode.rs

@@ -5,7 +5,7 @@ use crate::state::kernel::KernelState;
 use crate::error::{Result, KernelError};
 
 pub const MAGIC: &[u8; 4] = b"VALK";
-pub const SCHEMA_VERSION: u32 = 1;
+pub const SCHEMA_VERSION: u32 = 2; // Bumped for Metadata support
 
 /// writes a u32 to the buffer at offset
 fn write_u32(buf: &mut [u8], offset: &mut usize, val: u32) -> Result<()> {
@@ -18,14 +18,8 @@ fn write_u32(buf: &mut [u8], offset: &mut usize, val: u32) -> Result<()> {
     Ok(())
 }
 
-fn write_u64(buf: &mut [u8], offset: &mut usize, val: u64) -> Result<()> {
-    if *offset + 8 > buf.len() {
-        return Err(KernelError::CapacityExceeded);
-    }
-    let bytes = val.to_le_bytes();
-    buf[*offset..*offset + 8].copy_from_slice(&bytes);
-    *offset += 8;
-    Ok(())
+fn write_i32(buf: &mut [u8], offset: &mut usize, val: i32) -> Result<()> {
+    write_u32(buf, offset, val as u32)
 }
 
 fn write_u8(buf: &mut [u8], offset: &mut usize, val: u8) -> Result<()> {
@@ -37,13 +31,22 @@ fn write_u8(buf: &mut [u8], offset: &mut usize, val: u8) -> Result<()> {
     Ok(())
 }
 
-fn write_i32(buf: &mut [u8], offset: &mut usize, val: i32) -> Result<()> {
-    if *offset + 4 > buf.len() {
+fn write_u64(buf: &mut [u8], offset: &mut usize, val: u64) -> Result<()> {
+    if *offset + 8 > buf.len() {
         return Err(KernelError::CapacityExceeded);
     }
     let bytes = val.to_le_bytes();
-    buf[*offset..*offset + 4].copy_from_slice(&bytes);
-    *offset += 4;
+    buf[*offset..*offset + 8].copy_from_slice(&bytes);
+    *offset += 8;
+    Ok(())
+}
+
+fn write_bytes(buf: &mut [u8], offset: &mut usize, data: &[u8]) -> Result<()> {
+    if *offset + data.len() > buf.len() {
+        return Err(KernelError::CapacityExceeded);
+    }
+    buf[*offset..*offset + data.len()].copy_from_slice(data);
+    *offset += data.len();
     Ok(())
 }
 
@@ -52,21 +55,17 @@ pub fn encode_state<const MAX_RECORDS: usize, const D: usize, const MAX_NODES: u
     buf: &mut [u8],
 ) -> Result<usize> {
     let mut offset = 0;
-
+    
     // Header
-    if offset + 4 > buf.len() { return Err(KernelError::CapacityExceeded); }
-    buf[offset..offset+4].copy_from_slice(MAGIC);
-    offset += 4;
+    write_bytes(buf, &mut offset, MAGIC)?;
+    write_u32(buf, &mut offset, SCHEMA_VERSION)?; // Version
+    write_u64(buf, &mut offset, state.version.0)?; // State Version
 
-    write_u32(buf, &mut offset, SCHEMA_VERSION)?;
-    write_u64(buf, &mut offset, state.version.0)?;
-    
-    // Capacities (to check compatibility on restore)
+    // Capacities
     write_u32(buf, &mut offset, MAX_RECORDS as u32)?;
     write_u32(buf, &mut offset, D as u32)?;
     write_u32(buf, &mut offset, MAX_NODES as u32)?;
     write_u32(buf, &mut offset, MAX_EDGES as u32)?;
-
     // Records
     let record_count = state.records.len() as u32;
     write_u32(buf, &mut offset, record_count)?;
@@ -77,7 +76,18 @@ pub fn encode_state<const MAX_RECORDS: usize, const D: usize, const MAX_NODES: u
         for scalar in record.vector.data.iter() {
             write_i32(buf, &mut offset, scalar.0)?;
         }
+        // V2 Metadata
+        match &record.metadata {
+            Some(m) => {
+                write_u32(buf, &mut offset, m.len() as u32)?;
+                write_bytes(buf, &mut offset, m)?;
+            }
+            None => {
+                write_u32(buf, &mut offset, 0)?;
+            }
+        }
     }
+// ...
 
     // Nodes
     let mut node_count = 0;