feat: add HNSW and quantization kernels

Author: TACITVS

include/fp_blake3.h

@@ -0,0 +1,257 @@
+#pragma once
+
+/**
+ * FP_BLAKE3 - BLAKE3 Cryptographic Hash using FP_ASM_LIB primitives
+ *
+ * BLAKE3 is a cryptographic hash function that is:
+ * - Fast: ~3x faster than SHA-256 on modern CPUs
+ * - Secure: Based on ChaCha permutation, resistant to length extension
+ * - Parallelizable: Tree structure enables multi-threaded hashing
+ * - Versatile: Supports hashing, keyed hashing, and key derivation
+ *
+ * This implementation uses FP_ASM_LIB's AVX2-optimized u32 primitives
+ * for the core mixing operations.
+ *
+ * Output: 256-bit (32-byte) hash
+ * Block size: 64 bytes
+ */
+
+#include <stdint.h>
+#include <stddef.h>
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+/* ============================================================================
+ * Constants
+ * ============================================================================ */
+
+#define FP_BLAKE3_OUT_LEN       32    /* Default output length (256 bits) */
+#define FP_BLAKE3_KEY_LEN       32    /* Key length for keyed hashing */
+#define FP_BLAKE3_BLOCK_LEN     64    /* Block size in bytes */
+#define FP_BLAKE3_CHUNK_LEN     1024  /* Chunk size for tree hashing */
+
+/* ============================================================================
+ * Types
+ * ============================================================================ */
+
+/**
+ * BLAKE3 hasher state
+ *
+ * Maintains incremental hashing state for streaming input.
+ */
+typedef struct {
+    uint32_t cv[8];           /* Chaining value (current state) */
+    uint64_t chunk_counter;   /* Number of chunks processed */
+    uint8_t  buf[FP_BLAKE3_BLOCK_LEN];  /* Partial block buffer */
+    uint8_t  buf_len;         /* Bytes in buffer */
+    uint8_t  blocks_compressed; /* Blocks compressed in current chunk */
+    uint8_t  flags;           /* Domain separation flags */
+    uint8_t  key[FP_BLAKE3_KEY_LEN];  /* Key (for keyed mode) */
+} FpBlake3Hasher;
+
+/**
+ * BLAKE3 output structure
+ */
+typedef struct {
+    uint8_t hash[FP_BLAKE3_OUT_LEN];
+} FpBlake3Hash;
+
+/* ============================================================================
+ * Simple API (One-shot hashing)
+ * ============================================================================ */
+
+/**
+ * Hash data in one call
+ *
+ * @param input  Input data to hash
+ * @param len    Length of input in bytes
+ * @param output Output buffer (32 bytes)
+ */
+void fp_blake3_hash(const uint8_t* input, size_t len, uint8_t* output);
+
+/**
+ * Hash data with key (MAC)
+ *
+ * @param key    32-byte key
+ * @param input  Input data
+ * @param len    Length of input
+ * @param output Output buffer (32 bytes)
+ */
+void fp_blake3_hash_keyed(const uint8_t* key, const uint8_t* input,
+                          size_t len, uint8_t* output);
+
+/**
+ * Derive a key from context and key material
+ *
+ * @param context      Context string (domain separator)
+ * @param context_len  Length of context
+ * @param key_material Input key material
+ * @param km_len       Length of key material
+ * @param output       Output buffer (32 bytes)
+ */
+void fp_blake3_derive_key(const char* context, size_t context_len,
+                          const uint8_t* key_material, size_t km_len,
+                          uint8_t* output);
+
+/* ============================================================================
+ * Incremental API (Streaming hashing)
+ * ============================================================================ */
+
+/**
+ * Initialize hasher for incremental hashing
+ *
+ * @param hasher  Hasher state to initialize
+ */
+void fp_blake3_hasher_init(FpBlake3Hasher* hasher);
+
+/**
+ * Initialize hasher for keyed hashing
+ *
+ * @param hasher  Hasher state to initialize
+ * @param key     32-byte key
+ */
+void fp_blake3_hasher_init_keyed(FpBlake3Hasher* hasher, const uint8_t* key);
+
+/**
+ * Add input data to hasher
+ *
+ * @param hasher  Hasher state
+ * @param input   Input data
+ * @param len     Length of input
+ */
+void fp_blake3_hasher_update(FpBlake3Hasher* hasher, const uint8_t* input, size_t len);
+
+/**
+ * Finalize hash and produce output
+ *
+ * @param hasher  Hasher state
+ * @param output  Output buffer (32 bytes)
+ */
+void fp_blake3_hasher_finalize(const FpBlake3Hasher* hasher, uint8_t* output);
+
+/**
+ * Finalize with extended output (XOF mode)
+ *
+ * @param hasher      Hasher state
+ * @param output      Output buffer
+ * @param output_len  Desired output length (can be > 32 bytes)
+ */
+void fp_blake3_hasher_finalize_xof(const FpBlake3Hasher* hasher,
+                                    uint8_t* output, size_t output_len);
+
+/* ============================================================================
+ * Low-level API (For advanced use)
+ * ============================================================================ */
+
+/**
+ * BLAKE3 compression function (single block)
+ *
+ * This is the core primitive that transforms the state.
+ * Uses AVX2-optimized operations from FP_ASM_LIB.
+ *
+ * @param state         8 x u32 state (modified in-place)
+ * @param block         64-byte input block
+ * @param block_len     Actual length (for padding)
+ * @param counter       Block counter
+ * @param flags         Domain flags
+ */
+void fp_blake3_compress(uint32_t state[8], const uint8_t block[64],
+                        uint8_t block_len, uint64_t counter, uint8_t flags);
+
+/**
+ * BLAKE3 G mixing function (quarter-round)
+ *
+ * Core mixing operation, optimized with AVX2.
+ * Operates on 4 words at positions a, b, c, d with messages mx, my.
+ *
+ * @param state  16 x u32 working state
+ * @param a,b,c,d  Indices into state
+ * @param mx, my   Message words
+ */
+void fp_blake3_g(uint32_t* state, int a, int b, int c, int d,
+                 uint32_t mx, uint32_t my);
+
+/**
+ * BLAKE3 round function
+ *
+ * Applies 8 G functions in specific pattern.
+ *
+ * @param state  16 x u32 working state
+ * @param msg    Message schedule (16 x u32)
+ */
+void fp_blake3_round(uint32_t* state, const uint32_t* msg);
+
+/* ============================================================================
+ * SIMD-Optimized Batch Operations
+ * ============================================================================ */
+
+/**
+ * Hash multiple inputs in parallel (AVX2 optimized)
+ *
+ * Hashes up to 8 independent inputs simultaneously using SIMD.
+ * Significant speedup for batch processing.
+ *
+ * @param inputs      Array of input pointers
+ * @param lengths     Array of input lengths
+ * @param outputs     Array of output buffers (32 bytes each)
+ * @param count       Number of inputs (1-8)
+ */
+void fp_blake3_hash_batch(const uint8_t** inputs, const size_t* lengths,
+                          uint8_t** outputs, size_t count);
+
+/* ============================================================================
+ * AVX2-Accelerated Functions
+ * ============================================================================ */
+
+/**
+ * AVX2-accelerated hash (3-5x faster than scalar)
+ */
+void fp_blake3_hash_avx2(const uint8_t* input, size_t len, uint8_t* output);
+
+/**
+ * Maximum performance hash - all operations in registers
+ */
+void fp_blake3_hash_fast(const uint8_t* input, size_t len, uint8_t* output);
+
+/**
+ * Official BLAKE3 implementation (3-5x faster than SHA-256)
+ */
+void fp_blake3_hash_official(const uint8_t* input, size_t len, uint8_t* output);
+void fp_blake3_hash_keyed_official(const uint8_t* key, const uint8_t* input,
+                                    size_t len, uint8_t* output);
+void fp_blake3_derive_key_official(const char* context, size_t context_len,
+                                    const uint8_t* key_material, size_t km_len,
+                                    uint8_t* output);
+
+/**
+ * AVX2-accelerated keyed hash
+ */
+void fp_blake3_hash_keyed_avx2(const uint8_t* key, const uint8_t* input,
+                               size_t len, uint8_t* output);
+
+/* ============================================================================
+ * Utility Functions
+ * ============================================================================ */
+
+/**
+ * Convert hash to hexadecimal string
+ *
+ * @param hash   32-byte hash
+ * @param hex    Output buffer (65 bytes: 64 hex chars + null)
+ */
+void fp_blake3_to_hex(const uint8_t* hash, char* hex);
+
+/**
+ * Compare two hashes in constant time
+ *
+ * @param a  First hash (32 bytes)
+ * @param b  Second hash (32 bytes)
+ * @return   0 if equal, non-zero otherwise
+ */
+int fp_blake3_compare(const uint8_t* a, const uint8_t* b);
+
+#ifdef __cplusplus
+}
+#endif

include/fp_hnsw.h

@@ -0,0 +1,64 @@
+#ifndef FP_HNSW_H
+#define FP_HNSW_H
+
+#include <stdint.h>
+#include <stddef.h>
+
+/**
+ * HNSW Node structure.
+ * Each node represents a vector in the database.
+ */
+typedef struct {
+    int32_t vector_idx;     // Index into the database's columnar storage
+    int32_t num_layers;     // Number of layers this node belongs to
+    int32_t** neighbors;    // Array of neighbor indices per layer
+    int32_t* neighbor_counts; // Number of neighbors per layer
+} fp_hnsw_node;
+
+/**
+ * HNSW Index structure.
+ */
+typedef struct {
+    int32_t dim;            // Vector dimension
+    int32_t M;              // Max number of neighbors per node
+    int32_t ef_construction; // Search depth during construction
+    int32_t max_layers;     // Max layers allowed
+    int32_t entry_point;    // Node index of the entry point
+    int32_t current_max_layer;
+    
+    fp_hnsw_node* nodes;    // Array of nodes
+    size_t count;           // Number of nodes currently in index
+    size_t capacity;        // Max nodes
+} fp_hnsw_index;
+
+/**
+ * Initialize a new HNSW index.
+ */
+fp_hnsw_index* fp_hnsw_create(int32_t dim, int32_t M, int32_t ef_construction, size_t capacity);
+
+/**
+ * Free HNSW index.
+ */
+void fp_hnsw_free(fp_hnsw_index* index);
+
+/**
+ * Insert a vector into the HNSW index.
+ * Note: Caller must provide the vector data since HNSW only stores indices.
+ */
+void fp_hnsw_insert(fp_hnsw_index* index, int32_t vector_idx, const double* vector, const double** db_columns, size_t db_count);
+
+/**
+ * Search the HNSW index.
+ */
+size_t fp_hnsw_search(
+    const fp_hnsw_index* index,
+    const double* query_vec,
+    int32_t k,
+    int32_t ef_search,
+    const double** db_columns,
+    size_t db_count,
+    int32_t* results_indices,
+    double* results_scores
+);
+
+#endif /* FP_HNSW_H */

include/fp_kmeans.h

@@ -29,6 +29,7 @@
 
 #include <stdint.h>
 #include "fp_monads.h"
+#include "fp_progress.h"
 
 /* ============================================================================
  * K-MEANS RESULT STRUCTURE
@@ -72,6 +73,25 @@ KMeansResult fp_kmeans_f64(
     uint64_t seed
 );
 
+/**
+ * fp_kmeans_f64_progress - Run K-Means with progress callbacks per iteration
+ *
+ * @param progress  Progress callback + user pointer (can be {NULL, NULL})
+ *
+ * The callback receives (current_iter, max_iter, phase) and should return:
+ *   1 to continue, 0 to stop early (cancellation).
+ */
+KMeansResult fp_kmeans_f64_progress(
+    const double* data,
+    int n,
+    int d,
+    int k,
+    int max_iter,
+    double tol,
+    uint64_t seed,
+    fp_progress_t progress
+);
+
 /**
  * fp_kmeans_free - Free internal arrays of KMeansResult
  *

include/fp_math.h

@@ -4,16 +4,17 @@
 #include "fp_graphics_engine.h"
 #include <math.h>
 
-static inline Vec3f vec3f_add(Vec3f a, Vec3f b) { return (Vec3f){a.x + b.x, a.y + b.y, a.z + b.z}; }
-static inline Vec3f vec3f_sub(Vec3f a, Vec3f b) { return (Vec3f){a.x - b.x, a.y - b.y, a.z - b.z}; }
-static inline Vec3f vec3f_scale(Vec3f v, float s) { return (Vec3f){v.x * s, v.y * s, v.z * s}; }
+static inline Vec3f vec3f_add(Vec3f a, Vec3f b) { return (Vec3f){a.x + b.x, a.y + b.y, a.z + b.z, 0.0f}; }
+static inline Vec3f vec3f_sub(Vec3f a, Vec3f b) { return (Vec3f){a.x - b.x, a.y - b.y, a.z - b.z, 0.0f}; }
+static inline Vec3f vec3f_scale(Vec3f v, float s) { return (Vec3f){v.x * s, v.y * s, v.z * s, 0.0f}; }
 static inline float vec3f_length(Vec3f v) { return sqrtf(v.x * v.x + v.y * v.y + v.z * v.z); }
 static inline Vec3f vec3f_normalize(Vec3f v) { float l = vec3f_length(v); return l > 0 ? vec3f_scale(v, 1.0f / l) : v; }
 static inline Vec3f vec3f_cross(Vec3f a, Vec3f b) {
     return (Vec3f){
         a.y * b.z - a.z * b.y,
         a.z * b.x - a.x * b.z,
-        a.x * b.y - a.y * b.x
+        a.x * b.y - a.y * b.x,
+        0.0f
     };
 }
 

include/fp_progress.h

@@ -0,0 +1,20 @@
+#ifndef FP_PROGRESS_H
+#define FP_PROGRESS_H
+
+#include <stdint.h>
+
+typedef int (*fp_progress_callback)(void* user, uint64_t current, uint64_t total, const char* phase);
+
+typedef struct {
+    fp_progress_callback callback;
+    void* user;
+} fp_progress_t;
+
+static inline int fp_progress_emit(fp_progress_t progress, uint64_t current, uint64_t total, const char* phase) {
+    if (progress.callback) {
+        return progress.callback(progress.user, current, total, phase);
+    }
+    return 1;
+}
+
+#endif /* FP_PROGRESS_H */