Skip to content

Commit 8549cb5

Browse files
andinuxclaude
andcommitted
fix: do not persist zero dimension after a zero-chunk parse
A parse producing no chunks (e.g. whitespace-only content) reached the dimension persistence block with ctx->dimension still 0, writing dimension=0 to dbmem_settings and latching dimension_saved. Later real embeddings then updated the dimension in memory only, so a reopened connection that only searches saw dimension=0 and reported that no content has been indexed. Persist the dimension only when at least one real embedding was computed (chunks_added > 0), which guarantees ctx->dimension is set. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
1 parent 1b16f43 commit 8549cb5

2 files changed

Lines changed: 37 additions & 1 deletion

File tree

src/sqlite-memory.c

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3016,7 +3016,9 @@ static int dbmem_process_buffer (dbmem_context *ctx, const char *buffer, int64_t
30163016
rc = dbmem_database_add_vault_sentinel(ctx);
30173017
}
30183018

3019-
if (rc == SQLITE_OK && !ctx->dimension_saved) {
3019+
// persist the dimension only after a real embedding established it:
3020+
// a zero-chunk parse would latch dimension=0 and block the real write
3021+
if (rc == SQLITE_OK && ctx->chunks_added > 0 && !ctx->dimension_saved) {
30203022
// make sure to serialize dimension
30213023
dbmem_settings_write_int(db, DBMEM_SETTINGS_KEY_DIMENSION, ctx->dimension);
30223024
ctx->dimension_saved = true;

test/unittest.c

Lines changed: 34 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5407,6 +5407,39 @@ TEST(sqlite_memory_zero_chunk_content_marks_processed_with_sentinel) {
54075407
sqlite3_close(db);
54085408
}
54095409

5410+
TEST(sqlite_memory_zero_chunk_first_add_does_not_persist_zero_dimension) {
5411+
sqlite3 *db = open_test_db();
5412+
ASSERT(db != NULL);
5413+
5414+
dbmem_provider_t prov = { .init = dummy_init, .compute = dummy_compute, .free = dummy_free };
5415+
int rc = sqlite3_memory_register_provider(db, "dummy", &prov);
5416+
ASSERT_EQ(rc, SQLITE_OK);
5417+
5418+
sqlite3_int64 result = 0;
5419+
rc = exec_get_int(db, "SELECT memory_set_model('dummy', 'test-model');", &result);
5420+
ASSERT_EQ(rc, SQLITE_OK);
5421+
5422+
// a zero-chunk first add must not latch dimension=0 into dbmem_settings
5423+
rc = exec_get_int(db, "SELECT memory_add_content('docs/blank.md', ' ' || char(10) || char(9));", &result);
5424+
ASSERT_EQ(rc, SQLITE_OK);
5425+
5426+
sqlite3_int64 count = -1;
5427+
rc = exec_get_int(db, "SELECT COUNT(*) FROM dbmem_settings WHERE key = 'dimension';", &count);
5428+
ASSERT_EQ(rc, SQLITE_OK);
5429+
ASSERT_EQ(count, 0);
5430+
5431+
// the first real embedding persists the provider dimension
5432+
rc = exec_get_int(db, "SELECT memory_add_content('docs/real.md', '# Title' || char(10) || 'Real body text.');", &result);
5433+
ASSERT_EQ(rc, SQLITE_OK);
5434+
5435+
sqlite3_int64 dimension = -1;
5436+
rc = exec_get_int(db, "SELECT value FROM dbmem_settings WHERE key = 'dimension';", &dimension);
5437+
ASSERT_EQ(rc, SQLITE_OK);
5438+
ASSERT_EQ(dimension, 4);
5439+
5440+
sqlite3_close(db);
5441+
}
5442+
54105443
#ifndef DBMEM_OMIT_LOCAL_ENGINE
54115444
TEST(sqlite_local_logger_ignores_stale_user_data) {
54125445
dbmem_logger(GGML_LOG_LEVEL_WARN, "ignored warning", (void *)1);
@@ -5637,6 +5670,7 @@ int main(int argc, char *argv[]) {
56375670
RUN_TEST(sqlite_memory_defer_embeddings_requires_save_content);
56385671
RUN_TEST(sqlite_memory_embed_pending_embeds_deferred_content_in_batches);
56395672
RUN_TEST(sqlite_memory_zero_chunk_content_marks_processed_with_sentinel);
5673+
RUN_TEST(sqlite_memory_zero_chunk_first_add_does_not_persist_zero_dimension);
56405674

56415675
#ifndef DBMEM_OMIT_REMOTE_ENGINE
56425676
RUN_TEST(sqlite_set_model_releases_previous_engine_on_class_switch);

0 commit comments

Comments
 (0)