Add memory leak tests for open/close cycles in C API and engine

Author: sphildreth

bindings/python/tests/test_memory_leak.py

@@ -0,0 +1,35 @@
+import os
+import psutil
+import gc
+import pytest
+import decentdb
+
+def test_engine_open_close_memory_leak(tmp_path):
+    db_path = str(tmp_path / "test_py_leak.ddb")
+    process = psutil.Process(os.getpid())
+    
+    db = decentdb.connect(db_path)
+    cur = db.cursor()
+    cur.execute("CREATE TABLE leak_test(id int, data text)")
+    for i in range(1000):
+        # 1000 rows with 1KB text each ~ 1MB data
+        cur.execute("INSERT INTO leak_test VALUES (?, ?)", (i, "a" * 1000))
+    db.commit()
+    db.close()
+        
+    gc.collect()
+    mem_before = process.memory_info().rss
+    
+    for _ in range(500):
+        db = decentdb.connect(db_path)
+        cur = db.cursor()
+        cur.execute("SELECT * FROM leak_test")
+        cur.fetchall()
+        db.close()
+        
+    gc.collect()
+    mem_after = process.memory_info().rss
+    diff = mem_after - mem_before
+    
+    print(f"Memory before: {mem_before}, after: {mem_after}, diff: {diff}")
+    assert diff < 10000000, f"Memory leak detected! Diff: {diff} bytes"

src/c_api.nim

@@ -501,6 +501,12 @@ proc decentdb_open*(path: cstring, options: cstring): pointer {.exportc, cdecl,
   GC_ref(handle)
   return cast[pointer](handle)
 
+when defined(linux) and not defined(android):
+  import dynlib
+  var 
+    mallocTrimCached: proc(pad: csize_t): cint {.cdecl.}
+    mallocTrimChecked: bool = false
+
 proc decentdb_close*(p: pointer): cint {.exportc, cdecl, dynlib.} =
   if p == nil: return 0
   let handle = cast[DbHandle](p)
@@ -509,6 +515,19 @@ proc decentdb_close*(p: pointer): cint {.exportc, cdecl, dynlib.} =
     discard closeDb(handle.db)
 
   GC_unref(handle)
+
+  when defined(linux) and not defined(android):
+    if not mallocTrimChecked:
+      mallocTrimChecked = true
+      let lib = loadLib("libc.so.6")
+      if lib != nil:
+        let sym = symAddr(lib, "malloc_trim")
+        if sym != nil:
+          mallocTrimCached = cast[proc(pad: csize_t): cint {.cdecl.}](sym)
+    
+    if mallocTrimCached != nil:
+      discard mallocTrimCached(0)
+
   return 0
 
 proc decentdb_checkpoint*(p: pointer): cint {.exportc, cdecl, dynlib.} =

tests/nim/test_capi_open_close_leak.nim

@@ -0,0 +1,37 @@
+import unittest
+import os
+import ../../src/c_api
+
+proc makeTempDb(name: string): string =
+  let path = getTempDir() / name
+  if fileExists(path):
+    removeFile(path)
+  if fileExists(path & ".wal"):
+    removeFile(path & ".wal")
+  path
+
+suite "C API open/close memory leak":
+  test "opening and closing via C API should not leak memory":
+    let path = makeTempDb("decentdb_capi_leak_test.db")
+    
+    # Warm up
+    for i in 1..5:
+      let db = decentdb_open(path.cstring, "".cstring)
+      require(db != nil)
+      discard decentdb_close(db)
+      
+    GC_fullCollect()
+    let initMem = getOccupiedMem()
+    
+    for i in 1..1000:
+      let db = decentdb_open(path.cstring, "".cstring)
+      doAssert(db != nil)
+      discard decentdb_close(db)
+      
+    GC_fullCollect()
+    let finalMem = getOccupiedMem()
+    
+    let diff = int(finalMem) - int(initMem)
+    echo "C API Initial Mem: ", initMem, " Final Mem: ", finalMem, " Diff: ", diff
+    require(diff < 1000000)
+

tests/nim/test_engine_open_close_leak.nim

@@ -0,0 +1,62 @@
+import unittest
+import os
+import ../../src/engine
+import ../../src/c_api
+import ../../src/errors
+
+proc makeTempDb(name: string): string =
+  let path = getTempDir() / name
+  if fileExists(path): removeFile(path)
+  if fileExists(path & ".wal"): removeFile(path & ".wal")
+  path
+
+suite "Memory leak tests for open/close cycles":
+  test "opening and closing db connections many times via core engine should not leak memory":
+    let path = makeTempDb("decentdb_engine_leak_test.db")
+    
+    # Warm up
+    block:
+      for i in 1..5:
+        let dbRes = openDb(path)
+        require(dbRes.ok)
+        discard dbRes.value.closeDb()
+      
+    GC_fullCollect()
+    let initMem = getOccupiedMem()
+    
+    block:
+      for i in 1..2000:
+        let dbRes = openDb(path)
+        doAssert(dbRes.ok)
+        discard dbRes.value.closeDb()
+      
+    GC_fullCollect()
+    let finalMem = getOccupiedMem()
+    let diff = int(finalMem) - int(initMem)
+    echo "Engine Initial Mem: ", initMem, " Final Mem: ", finalMem, " Diff: ", diff
+    require(diff < 200000)
+
+  test "opening and closing db connections many times via C API should not leak memory":
+    let path = makeTempDb("decentdb_capi_leak_test.db")
+    
+    # Warm up
+    block:
+      for i in 1..5:
+        let db = decentdb_open(path.cstring, "".cstring)
+        require(db != nil)
+        discard decentdb_close(db)
+      
+    GC_fullCollect()
+    let initMem = getOccupiedMem()
+    
+    block:
+      for i in 1..2000:
+        let db = decentdb_open(path.cstring, "".cstring)
+        doAssert(db != nil)
+        discard decentdb_close(db)
+      
+    GC_fullCollect()
+    let finalMem = getOccupiedMem()
+    let diff = int(finalMem) - int(initMem)
+    echo "C API Initial Mem: ", initMem, " Final Mem: ", finalMem, " Diff: ", diff
+    require(diff < 200000)

tests/nim/test_open_close_memory_leak.nim

@@ -0,0 +1,62 @@
+import unittest
+import os
+import ../../src/engine
+import ../../src/c_api
+import ../../src/errors
+
+proc makeTempDb(name: string): string =
+  let path = getTempDir() / name
+  if fileExists(path): removeFile(path)
+  if fileExists(path & ".wal"): removeFile(path & ".wal")
+  path
+
+suite "Memory leak tests for open/close cycles":
+  test "opening and closing db connections many times via core engine should not leak memory":
+    let path = makeTempDb("decentdb_engine_leak_test.db")
+    
+    # Warm up / initialize anything that might do lazy init
+    for i in 1..5:
+      let dbRes = openDb(path)
+      require(dbRes.ok)
+      discard dbRes.value.closeDb()
+      
+    GC_fullCollect()
+    let initMem = getOccupiedMem()
+    
+    for i in 1..2000:
+      let dbRes = openDb(path)
+      doAssert(dbRes.ok)
+      discard dbRes.value.closeDb()
+      
+    GC_fullCollect()
+    let finalMem = getOccupiedMem()
+    
+    let diff = int(finalMem) - int(initMem)
+    echo "Engine Initial Mem: ", initMem, " Final Mem: ", finalMem, " Diff: ", diff
+    # A tiny bit of fluctuation might happen, restrict it strictly
+    require(diff < 200000)
+
+  test "opening and closing db connections many times via C API should not leak memory":
+    let path = makeTempDb("decentdb_capi_leak_test.db")
+    
+    # Warm up
+    for i in 1..5:
+      let db = decentdb_open(path.cstring, "".cstring)
+      require(db != nil)
+      discard decentdb_close(db)
+      
+    GC_fullCollect()
+    let initMem = getOccupiedMem()
+    
+    for i in 1..2000:
+      let db = decentdb_open(path.cstring, "".cstring)
+      doAssert(db != nil)
+      discard decentdb_close(db)
+      
+    GC_fullCollect()
+    let finalMem = getOccupiedMem()
+    
+    let diff = int(finalMem) - int(initMem)
+    echo "C API Initial Mem: ", initMem, " Final Mem: ", finalMem, " Diff: ", diff
+    require(diff < 200000)
+

tests/repro_py_leak.py

@@ -0,0 +1,29 @@
+import os
+import psutil
+import gc
+import decentdb
+
+db_path = "test_py_leak.ddb"
+if os.path.exists(db_path):
+    os.remove(db_path)
+if os.path.exists(db_path + ".wal"):
+    os.remove(db_path + ".wal")
+
+process = psutil.Process(os.getpid())
+
+# warm up
+for i in range(5):
+    db = decentdb.connect(db_path)
+    db.close()
+
+gc.collect()
+mem_before = process.memory_info().rss
+
+for i in range(10000):
+    db = decentdb.connect(db_path)
+    db.close()
+
+gc.collect()
+mem_after = process.memory_info().rss
+
+print(f"Memory before: {mem_before}, after: {mem_after}, diff: {mem_after - mem_before}")