Primary key is now TEXT

Author: marcobambini

README.md

@@ -36,7 +36,7 @@ sqlite-memory bridges these concepts, allowing any SQLite-powered application to
 - **Hybrid Search**: Combines vector similarity (cosine distance) with FTS5 full-text search for superior retrieval
 - **Smart Chunking**: Markdown-aware parsing preserves semantic boundaries
 - **Intelligent Sync**: Content-hash change detection skips unchanged files, atomically replaces modified ones, and cleans up deleted ones
-- **Transactional Safety**: Every sync operation runs inside a SAVEPOINT transaction - either fully succeeds or fully rolls back, no partially-indexed content
+- **Transactional Safety**: Text/file ingests run inside SAVEPOINT transactions, and directory sync uses transactional cleanup plus per-file transactional updates so failed files do not leave partial rows behind
 - **Efficient Storage**: Binary embeddings with configurable dimensions
 - **Embedding Cache**: Automatically caches computed embeddings, so re-indexing the same text skips redundant API calls and computation
 - **Flexible Embedding**: Use local models (llama.cpp) or [vectors.space](https://vectors.space) remote API
@@ -74,7 +74,7 @@ sqlite-memory bridges these concepts, allowing any SQLite-powered application to
 ```sql
 -- Load extensions (sync is optional)
 .load ./vector
-.load ./sync
+.load ./cloudsync
 .load ./memory
 
 -- Configure embedding model (choose one):
@@ -84,8 +84,8 @@ SELECT memory_set_model('local', '/path/to/nomic-embed-text-v1.5.Q8_0.gguf');
 
 -- Option 2: Remote embedding via vectors.space (requires free API key from https://vectors.space)
 -- The provider name 'openai' selects the vectors.space OpenAI-compatible endpoint.
--- SELECT memory_set_model('openai', 'text-embedding-3-small');
 -- SELECT memory_set_apikey('your-vectorspace-api-key');
+-- SELECT memory_set_model('openai', 'text-embedding-3-small');
 
 -- Add some knowledge
 SELECT memory_add_text('SQLite is a C-language library that implements a small, fast,
@@ -160,7 +160,7 @@ All `memory_add_*` functions use content-hash change detection to avoid redundan
   1. **Cleanup**: Removes database entries for files that no longer exist on disk
   2. **Scan**: Recursively processes all matching files - adding new ones, replacing modified ones, and skipping unchanged ones
 
-Every sync operation is wrapped in a SQLite SAVEPOINT transaction. If anything fails mid-sync (embedding error, disk issue, etc.), the entire operation rolls back cleanly. There is no risk of partially-indexed files or orphaned entries.
+`memory_add_text()` and `memory_add_file()` each run inside a SQLite SAVEPOINT transaction. `memory_add_directory()` performs its cleanup pass transactionally and then processes each file in its own transaction. If one file fails, that file rolls back cleanly and previously-committed files remain valid; there are no partially-indexed rows or orphaned chunk/FTS entries for the failed file.
 
 This makes all sync functions safe to call repeatedly - for example, on a cron schedule or at agent startup - with minimal overhead.
 
@@ -258,8 +258,8 @@ FROM dbmem_content;
 -- Delete by context
 SELECT memory_delete_context('old-project');
 
--- Delete specific memory
-SELECT memory_delete(1234567890);
+-- Delete specific memory by hash
+SELECT memory_delete('9e3779b97f4a7c15');
 
 -- Clear all memories
 SELECT memory_clear();
@@ -279,8 +279,11 @@ cd sqlite-memory
 # Build (full build with local + remote engines)
 make
 
-# Run tests
+# Run parser/core unit tests + extension loading smoke test
 make test
+
+# Run the full SQL extension unit suite
+make test DEFINES="-DTEST_SQLITE_EXTENSION"
 ```
 
 ### Build Configurations

src/dbmem-lembed.c

@@ -100,9 +100,15 @@ void dbmem_logger (enum ggml_log_level level, const char *text, void *user_data)
 
 // MARK: -
 
+static void dbmem_local_set_error(dbmem_local_engine_t *engine, const char *message) {
+    if (!engine || !engine->context) return;
+    dbmem_context_set_error(engine->context, message);
+}
+
 dbmem_local_engine_t *dbmem_local_engine_init (void *ctx, const char *model_path, char err_msg[DBMEM_ERRBUF_SIZE]) {
     dbmem_local_engine_t *engine = (dbmem_local_engine_t *)dbmemory_zeroalloc(sizeof(dbmem_local_engine_t));
     if (!engine) return NULL;
+    engine->context = (dbmem_context *)ctx;
 
     // set logger
     llama_log_set(dbmem_logger, engine);
@@ -212,7 +218,7 @@ int dbmem_local_compute_embedding (dbmem_local_engine_t *engine, const char *tex
     // Tokenize
     int n_tokens = llama_tokenize(engine->vocab, text, text_len, engine->tokens, engine->tokens_capacity, true, true);
     if (n_tokens < 0) {
-        dbmem_context_set_error(engine->context, "Tokenization failed (text too long?)");
+        dbmem_local_set_error(engine, "Tokenization failed (text too long?)");
         return -1;
     }
 
@@ -242,7 +248,7 @@ int dbmem_local_compute_embedding (dbmem_local_engine_t *engine, const char *tex
     // Encode
     int ret = llama_encode(engine->ctx, batch);
     if (ret != 0) {
-        dbmem_context_set_error(engine->context, "Llama_encode failed");
+        dbmem_local_set_error(engine, "Llama_encode failed");
         return -1;
     }
 
@@ -255,7 +261,7 @@ int dbmem_local_compute_embedding (dbmem_local_engine_t *engine, const char *tex
     }
 
     if (!emb_ptr) {
-        dbmem_context_set_error(engine->context, "Failed to get embeddings");
+        dbmem_local_set_error(engine, "Failed to get embeddings");
         return -1;
     }
 
@@ -301,5 +307,5 @@ void dbmem_local_engine_free (dbmem_local_engine_t *engine) {
     }
 
     llama_backend_free();
+    dbmemory_free(engine);
 }
-

src/dbmem-parser.c

@@ -28,6 +28,7 @@
 typedef struct {
     size_t  start;              // Byte offset in source buffer
     size_t  end;                // Byte end in source buffer
+    int     is_heading;         // True if this section starts with a heading block
     char   *text;               // Stripped plain text (allocated)
     size_t  text_len;           // Length of stripped text
 } section_t;
@@ -113,8 +114,6 @@ static size_t find_split (const char *text, size_t len, size_t max_chars) {
 
 // Push a section to dynamic array
 static int section_push (parse_ctx_t *ctx, size_t start, size_t end, int is_heading) {
-    UNUSED_PARAM(is_heading);
-    
     if (ctx->sec_count >= ctx->sec_cap) {
         size_t new_cap = ctx->sec_cap ? ctx->sec_cap * 2 : 16;
         section_t *tmp = (section_t *)dbmemory_realloc(ctx->sections, new_cap * sizeof(section_t));
@@ -126,6 +125,7 @@ static int section_push (parse_ctx_t *ctx, size_t start, size_t end, int is_head
     section_t *s = &ctx->sections[ctx->sec_count++];
     s->start = start;
     s->end = end;
+    s->is_heading = is_heading;
     s->text = NULL;
     s->text_len = 0;
 
@@ -607,7 +607,7 @@ static int parse_sections (const char *buffer, size_t buffer_size, bool skip_sem
     for (size_t i = 0; i < ctx->sec_count; i++) {
         section_t *s = &ctx->sections[i];
         // First section or heading starts new section
-        if (write_idx == 0) {
+        if (write_idx == 0 || s->is_heading) {
             ctx->sections[write_idx++] = *s;
         } else {
             // Extend previous section to include this one

src/dbmem-search.c

@@ -52,36 +52,54 @@ typedef struct {
     struct {
         int             count;
         double          *rank;
-        sqlite3_int64   *hash;
+        uint64_t        *hash;
         sqlite3_int64   *seq;
     } fts;
 
     struct {
         int             count;
         double          *rank;
-        sqlite3_int64   *hash;
+        uint64_t        *hash;
         sqlite3_int64   *seq;
     } semantic;
 
     struct {
         int             count;
         double          *vectorScore;
         double          *textScore;
-        sqlite3_int64   *hash;
+        uint64_t        *hash;
         sqlite3_int64   *seq;
         int             *hasVector;
         int             *hasFts;
     } merge;
 
     double              *rank;
-    sqlite3_int64       *hash;
+    uint64_t            *hash;
     sqlite3_int64       *seq;
 
 } vMemorySearchCursor;
 
+static int dbmem_search_bind_hash (sqlite3_stmt *vm, int index, uint64_t hash) {
+    char hash_text[DBMEM_HASH_STR_MAXLEN];
+    dbmem_hash_to_hex(hash, hash_text);
+    return sqlite3_bind_text(vm, index, hash_text, -1, SQLITE_TRANSIENT);
+}
+
+static bool dbmem_search_column_hash (sqlite3_stmt *vm, int column, uint64_t *hash) {
+    const char *hash_text = (const char *)sqlite3_column_text(vm, column);
+    return dbmem_hash_from_hex(hash_text, hash);
+}
+
 // MARK: - UTILS -
 
 int vMemorySearchCursorAllocate (vMemorySearchCursor *c, int entries, bool perform_fts) {
+    if (entries <= 0) {
+        memset(c, 0, sizeof(*c));
+        c->max_results = entries;
+        c->perform_fts = perform_fts;
+        return SQLITE_OK;
+    }
+
     // one buffer to rule them all
     // fts (if enabled): rank, hash, seq = 3 arrays * entries
     // semantic: rank, hash, seq = 3 arrays * entries
@@ -94,26 +112,26 @@ int vMemorySearchCursorAllocate (vMemorySearchCursor *c, int entries, bool perfo
     // fts arrays
     if (perform_fts) {
         size += sizeof(double) * entries;           // fts.rank
-        size += sizeof(sqlite3_int64) * entries;    // fts.hash
+        size += sizeof(uint64_t) * entries;         // fts.hash
         size += sizeof(sqlite3_int64) * entries;    // fts.seq
     }
 
     // semantic arrays
     size += sizeof(double) * entries;               // semantic.rank
-    size += sizeof(sqlite3_int64) * entries;        // semantic.hash
+    size += sizeof(uint64_t) * entries;             // semantic.hash
     size += sizeof(sqlite3_int64) * entries;        // semantic.seq
 
     // merge arrays (2x entries for union of both sources)
     size += sizeof(double) * merge_entries;         // merge.vectorScore
     size += sizeof(double) * merge_entries;         // merge.textScore
-    size += sizeof(sqlite3_int64) * merge_entries;  // merge.hash
+    size += sizeof(uint64_t) * merge_entries;       // merge.hash
     size += sizeof(sqlite3_int64) * merge_entries;  // merge.seq
     size += sizeof(int) * merge_entries;            // merge.hasVector
     size += sizeof(int) * merge_entries;            // merge.hasFts
 
     // final arrays
     size += sizeof(double) * entries;               // rank
-    size += sizeof(sqlite3_int64) * entries;        // hash
+    size += sizeof(uint64_t) * entries;             // hash
     size += sizeof(sqlite3_int64) * entries;        // seq
 
     char *buffer = (char *)dbmemory_zeroalloc(size);
@@ -127,17 +145,17 @@ int vMemorySearchCursorAllocate (vMemorySearchCursor *c, int entries, bool perfo
     if (perform_fts) {
         c->fts.rank = (double *)buffer;
         buffer += sizeof(double) * entries;
-        c->fts.hash = (sqlite3_int64 *)buffer;
-        buffer += sizeof(sqlite3_int64) * entries;
+        c->fts.hash = (uint64_t *)buffer;
+        buffer += sizeof(uint64_t) * entries;
         c->fts.seq = (sqlite3_int64 *)buffer;
         buffer += sizeof(sqlite3_int64) * entries;
     }
 
     // semantic
     c->semantic.rank = (double *)buffer;
     buffer += sizeof(double) * entries;
-    c->semantic.hash = (sqlite3_int64 *)buffer;
-    buffer += sizeof(sqlite3_int64) * entries;
+    c->semantic.hash = (uint64_t *)buffer;
+    buffer += sizeof(uint64_t) * entries;
     c->semantic.seq = (sqlite3_int64 *)buffer;
     buffer += sizeof(sqlite3_int64) * entries;
 
@@ -146,8 +164,8 @@ int vMemorySearchCursorAllocate (vMemorySearchCursor *c, int entries, bool perfo
     buffer += sizeof(double) * merge_entries;
     c->merge.textScore = (double *)buffer;
     buffer += sizeof(double) * merge_entries;
-    c->merge.hash = (sqlite3_int64 *)buffer;
-    buffer += sizeof(sqlite3_int64) * merge_entries;
+    c->merge.hash = (uint64_t *)buffer;
+    buffer += sizeof(uint64_t) * merge_entries;
     c->merge.seq = (sqlite3_int64 *)buffer;
     buffer += sizeof(sqlite3_int64) * merge_entries;
     c->merge.hasVector = (int *)buffer;
@@ -158,8 +176,8 @@ int vMemorySearchCursorAllocate (vMemorySearchCursor *c, int entries, bool perfo
     // final rowset
     c->rank = (double *)buffer;
     buffer += sizeof(double) * entries;
-    c->hash = (sqlite3_int64 *)buffer;
-    buffer += sizeof(sqlite3_int64) * entries;
+    c->hash = (uint64_t *)buffer;
+    buffer += sizeof(uint64_t) * entries;
     c->seq = (sqlite3_int64 *)buffer;
 
     return SQLITE_OK;
@@ -183,7 +201,7 @@ int vMemorySearchCursorMerge(vMemorySearchCursor *c, double vectorWeight, double
 
     // add/merge FTS results (already normalized to 0..1)
     for (int i = 0; i < c->fts.count; i++) {
-        sqlite3_int64 hash = c->fts.hash[i];
+        uint64_t hash = c->fts.hash[i];
         sqlite3_int64 seq = c->fts.seq[i];
 
         // check if already in merge list
@@ -230,7 +248,7 @@ int vMemorySearchCursorMerge(vMemorySearchCursor *c, double vectorWeight, double
     // swap all parallel arrays together
     for (int i = 1; i < c->merge.count; i++) {
         double tempScore = c->merge.textScore[i];
-        sqlite3_int64 tempHash = c->merge.hash[i];
+        uint64_t tempHash = c->merge.hash[i];
         sqlite3_int64 tempSeq = c->merge.seq[i];
 
         int j = i - 1;
@@ -269,7 +287,7 @@ static void vMemorySearchUpdateAccess(sqlite3 *db, vMemorySearchCursor *c) {
 
     for (int i = 0; i < c->count; i++) {
         sqlite3_bind_int64(vm, 1, now);
-        sqlite3_bind_int64(vm, 2, c->hash[i]);
+        dbmem_search_bind_hash(vm, 2, c->hash[i]);
         sqlite3_step(vm);
         sqlite3_reset(vm);
     }
@@ -308,8 +326,7 @@ static int dbmem_fts_search (sqlite3 *db, vMemorySearchCursor *c, const char *in
         "SELECT rank, hash, seq FROM dbmem_vault_fts WHERE content MATCH ?1 ORDER BY rank LIMIT ?2";
     static const char *sql_with_context =
         "SELECT fts.rank, fts.hash, fts.seq FROM dbmem_vault_fts AS fts "
-        "JOIN dbmem_vault AS v ON fts.hash = v.hash AND fts.seq = v.seq "
-        "WHERE fts.content MATCH ?1 AND INSTR(',' || ?3 || ',', ',' || v.context || ',') > 0 "
+        "WHERE fts.content MATCH ?1 AND INSTR(',' || ?3 || ',', ',' || fts.context || ',') > 0 "
         "ORDER BY fts.rank LIMIT ?2";
     const char *sql = (context) ? sql_with_context : sql_no_context;
 
@@ -347,7 +364,10 @@ static int dbmem_fts_search (sqlite3 *db, vMemorySearchCursor *c, const char *in
         if (rank > rank_max) rank_max = rank;
 
         c->fts.rank[count] = rank;
-        c->fts.hash[count] = sqlite3_column_int64(vm, 1);
+        if (!dbmem_search_column_hash(vm, 1, &c->fts.hash[count])) {
+            rc = SQLITE_MISMATCH;
+            break;
+        }
         c->fts.seq[count] = sqlite3_column_int64(vm, 2);
         c->fts.count++;
 
@@ -409,7 +429,10 @@ static int dbmem_semantic_search (sqlite3 *db, vMemorySearchCursor *c, float *em
 
         // SQLITE_ROW
         c->semantic.rank[count] = sqlite3_column_double(vm, 0);
-        c->semantic.hash[count] = sqlite3_column_int64(vm, 1);
+        if (!dbmem_search_column_hash(vm, 1, &c->semantic.hash[count])) {
+            rc = SQLITE_MISMATCH;
+            break;
+        }
         c->semantic.seq[count] = sqlite3_column_int64(vm, 2);
         c->semantic.count++;
 
@@ -530,7 +553,10 @@ static int vMemorySearchCursorColumn (sqlite3_vtab_cursor *cur, sqlite3_context
 
     switch (iCol) {
         case SEARCH_COLUMN_HASH:
-            sqlite3_result_int64(context, c->hash[c->index]);
+        {
+            char hash_text[DBMEM_HASH_STR_MAXLEN];
+            sqlite3_result_text(context, dbmem_hash_to_hex(c->hash[c->index], hash_text), -1, SQLITE_TRANSIENT);
+        }
             break;
 
         case SEARCH_COLUMN_SEQ:
@@ -546,7 +572,7 @@ static int vMemorySearchCursorColumn (sqlite3_vtab_cursor *cur, sqlite3_context
             const char *sql = (iCol == SEARCH_COLUMN_PATH) ? path_sql : snippet_sql;
             sqlite3_stmt *vm = NULL;
             if (sqlite3_prepare_v2(db, sql, -1, &vm, NULL) == SQLITE_OK) {
-                sqlite3_bind_int64(vm, 1, c->hash[c->index]);
+                dbmem_search_bind_hash(vm, 1, c->hash[c->index]);
                 if (iCol == SEARCH_COLUMN_SNIPPET) sqlite3_bind_int64(vm, 2, c->seq[c->index]);
                 if (sqlite3_step(vm) == SQLITE_ROW) sqlite3_result_value(context, sqlite3_column_value(vm, 0));
             }
@@ -612,6 +638,11 @@ static int vMemorySearchCursorFilter (sqlite3_vtab_cursor *cur, int idxNum, cons
         fetch_count = max_results;
     }
 
+    if (fetch_count <= 0) {
+        c->count = 0;
+        return SQLITE_OK;
+    }
+
     // allocate internal cursor buffer
     int rc = vMemorySearchCursorAllocate(c, fetch_count, perform_fts);
     if (rc != SQLITE_OK) return SQLITE_NOMEM;
@@ -698,9 +729,9 @@ static int vMemorySearchCursorFilter (sqlite3_vtab_cursor *cur, int idxNum, cons
     printf("=================================\n");
     for (int i = 0; i < c->count; i++) {
         double rank = c->rank[i];
-        sqlite3_int64 hash = c->hash[i];
+        uint64_t hash = c->hash[i];
         sqlite3_int64 seq = c->seq[i];
-        printf("%3d %.3f %20lld %2lld\n", i, rank, (long long)hash, (long long)seq);
+        printf("%3d %.3f %016llx %2lld\n", i, rank, (unsigned long long)hash, (long long)seq);
     }
     printf("=================================\n");
     #endif