From 0e712a61b2533ea51446a5ae4bf27b48a2a5bd70 Mon Sep 17 00:00:00 2001 From: Tim Besard Date: Sun, 3 May 2026 19:03:24 +0200 Subject: [PATCH 1/8] Test cleanup: drop module wrappers, use mktempdir consistently. The procedural-style smoke tests in `dbi.jl` and `cur.jl` previously created `testdb/` in CWD and only cleaned up if the surrounding `try/finally` reached the `rm`. Wraps them in `mktempdir` so failures don't leak the directory. Drops the `module LMDB_*` wrappers in favour of plain `@testset`. The modules were a workaround for `struct` re-definition that doesn't apply when the file is included once per test run; `@testset` provides sufficient scope and matches the style of `test/dict.jl`. Co-Authored-By: Claude Opus 4.7 (1M context) --- test/common.jl | 14 ++- test/cur.jl | 218 +++++++++++++++++---------------- test/dbi.jl | 282 +++++++++++++++++++++---------------------- test/dupsort.jl | 112 ++++++++--------- test/env.jl | 285 ++++++++++++++++++++++++++++---------------- test/integration.jl | 182 ++++++++++++++-------------- test/liblmdb.jl | 102 ++++++++-------- 7 files changed, 637 insertions(+), 558 deletions(-) diff --git a/test/common.jl b/test/common.jl index 0683c80..7f00247 100644 --- a/test/common.jl +++ b/test/common.jl @@ -1,9 +1,13 @@ using LMDB using Test - @test LMDB.version()[1] >= v"0.9.15" +@testset "common" begin - # LMDBError - ex = LMDBError(Cint(0)) - @test_throws LMDBError throw(ex) - @test ex.code == 0 +@test LMDB.version()[1] >= v"0.9.15" + +# LMDBError +ex = LMDBError(Cint(0)) +@test_throws LMDBError throw(ex) +@test ex.code == 0 + +end # @testset "common" diff --git a/test/cur.jl b/test/cur.jl index c4c2a84..8a35b65 100644 --- a/test/cur.jl +++ b/test/cur.jl @@ -1,15 +1,14 @@ -module LMDB_CUR - using LMDB - using Test +using LMDB +using Test - const dbname = "testdb" - key = 10 - val = "key value is " +@testset "Cursor" begin - # Create dir - mkdir(dbname) +key = 10 +val = "key value is " - # Procedural style +# Procedural style + block style smoke test, exercising cursor put!/walk +# round-trip and the parent accessors. +mktempdir() do dbname env = create() try open(env, dbname) @@ -52,122 +51,121 @@ module LMDB_CUR end end end +end + +# Cursor positioning + walk primitives. +mktempdir() do dir + environment(dir) do env + start(env) do txn + open(txn) do dbi + LMDB.put!(txn, dbi, "a", "1") + LMDB.put!(txn, dbi, "b", "2") + LMDB.put!(txn, dbi, "c", "3") + + LMDB.open(txn, dbi) do cur + @test LMDB.seek!(cur, String) == "a" + @test LMDB.value(cur, String) == "1" + @test LMDB.key(cur, String) == "a" + @test LMDB.item(cur, String, String) == ("a" => "1") + + @test LMDB.next!(cur, String) == "b" + @test LMDB.value(cur, String) == "2" + + @test LMDB.seek_last!(cur, String) == "c" + @test LMDB.prev!(cur, String) == "b" + + @test LMDB.seek!(cur, "a", String) == "a" + @test LMDB.seek!(cur, "missing", String) === nothing + + @test LMDB.seek_range!(cur, "ab", String) == "b" + @test LMDB.seek_range!(cur, "z", String) === nothing + + # walk over everything + ks = String[] + LMDB.walk(cur) do k_ref, _ + push!(ks, read(LMDB.MDBValueIO(k_ref[]), String)) + end + @test ks == ["a", "b", "c"] + + # walk from a starting key + ks2 = String[] + LMDB.walk(cur; from="b") do k_ref, _ + push!(ks2, read(LMDB.MDBValueIO(k_ref[]), String)) + end + @test ks2 == ["b", "c"] - # Remove db dir - rm(dbname, recursive=true) - - # Cursor positioning + walk primitives. - mktempdir() do dir - environment(dir) do env - start(env) do txn - open(txn) do dbi - LMDB.put!(txn, dbi, "a", "1") - LMDB.put!(txn, dbi, "b", "2") - LMDB.put!(txn, dbi, "c", "3") - - LMDB.open(txn, dbi) do cur - @test LMDB.seek!(cur, String) == "a" - @test LMDB.value(cur, String) == "1" - @test LMDB.key(cur, String) == "a" - @test LMDB.item(cur, String, String) == ("a" => "1") - - @test LMDB.next!(cur, String) == "b" - @test LMDB.value(cur, String) == "2" - - @test LMDB.seek_last!(cur, String) == "c" - @test LMDB.prev!(cur, String) == "b" - - @test LMDB.seek!(cur, "a", String) == "a" - @test LMDB.seek!(cur, "missing", String) === nothing - - @test LMDB.seek_range!(cur, "ab", String) == "b" - @test LMDB.seek_range!(cur, "z", String) === nothing - - # walk over everything - ks = String[] - LMDB.walk(cur) do k_ref, _ - push!(ks, read(LMDB.MDBValueIO(k_ref[]), String)) - end - @test ks == ["a", "b", "c"] - - # walk from a starting key - ks2 = String[] - LMDB.walk(cur; from="b") do k_ref, _ - push!(ks2, read(LMDB.MDBValueIO(k_ref[]), String)) - end - @test ks2 == ["b", "c"] - - # walk from a key past the last entry — no callbacks. - ks3 = String[] - LMDB.walk(cur; from="z") do k_ref, _ - push!(ks3, read(LMDB.MDBValueIO(k_ref[]), String)) - end - @test isempty(ks3) - - # typed walk: each ref decoded via read(MDBValueIO, K) - # / read(MDBValueIO, V). - kv = Pair{String, String}[] - LMDB.walk(cur, String, String) do k, v - push!(kv, k => v) - end - @test kv == ["a" => "1", "b" => "2", "c" => "3"] - - # typed walk respects the false-stops contract. - seen = Pair{String, String}[] - LMDB.walk(cur, String, String) do k, v - push!(seen, k => v) - k == "b" ? false : nothing - end - @test seen == ["a" => "1", "b" => "2"] + # walk from a key past the last entry — no callbacks. + ks3 = String[] + LMDB.walk(cur; from="z") do k_ref, _ + push!(ks3, read(LMDB.MDBValueIO(k_ref[]), String)) end + @test isempty(ks3) + + # typed walk: each ref decoded via read(MDBValueIO, K) + # / read(MDBValueIO, V). + kv = Pair{String, String}[] + LMDB.walk(cur, String, String) do k, v + push!(kv, k => v) + end + @test kv == ["a" => "1", "b" => "2", "c" => "3"] + + # typed walk respects the false-stops contract. + seen = Pair{String, String}[] + LMDB.walk(cur, String, String) do k, v + push!(seen, k => v) + k == "b" ? false : nothing + end + @test seen == ["a" => "1", "b" => "2"] end end end end +end - # seek!/next! on an empty database returns nothing. - mktempdir() do dir - environment(dir) do env - start(env) do txn - open(txn) do dbi - LMDB.open(txn, dbi) do cur - @test LMDB.seek!(cur, String) === nothing - @test LMDB.seek_last!(cur, String) === nothing - @test LMDB.seek!(cur, "x", String) === nothing - @test LMDB.seek_range!(cur, "x", String) === nothing - - ks = String[] - LMDB.walk(cur) do k_ref, _ - push!(ks, read(LMDB.MDBValueIO(k_ref[]), String)) - end - @test isempty(ks) +# seek!/next! on an empty database returns nothing. +mktempdir() do dir + environment(dir) do env + start(env) do txn + open(txn) do dbi + LMDB.open(txn, dbi) do cur + @test LMDB.seek!(cur, String) === nothing + @test LMDB.seek_last!(cur, String) === nothing + @test LMDB.seek!(cur, "x", String) === nothing + @test LMDB.seek_range!(cur, "x", String) === nothing + + ks = String[] + LMDB.walk(cur) do k_ref, _ + push!(ks, read(LMDB.MDBValueIO(k_ref[]), String)) end + @test isempty(ks) end end end end +end - # Cursor.delete!: removes the entry the cursor is on; LMDB advances to - # the next entry. Throws on an unpositioned cursor (EINVAL), unlike the - # txn-based `delete!(txn, dbi, key)` which is Bool-returning on - # MDB_NOTFOUND. - mktempdir() do dir - environment(dir) do env - start(env) do txn - open(txn) do dbi - LMDB.put!(txn, dbi, "a", "1") - LMDB.put!(txn, dbi, "b", "2") - - LMDB.open(txn, dbi) do cur - @test LMDB.seek!(cur, "a", String) == "a" - LMDB.delete!(cur) # removes "a", cursor now on "b" - LMDB.delete!(cur) # removes "b" - @test_throws LMDBError LMDB.delete!(cur) # no live entry - end - @test LMDB.tryget(txn, dbi, "a", String) === nothing - @test LMDB.tryget(txn, dbi, "b", String) === nothing +# Cursor.delete!: removes the entry the cursor is on; LMDB advances to +# the next entry. Throws on an unpositioned cursor (EINVAL), unlike the +# txn-based `delete!(txn, dbi, key)` which is Bool-returning on +# MDB_NOTFOUND. +mktempdir() do dir + environment(dir) do env + start(env) do txn + open(txn) do dbi + LMDB.put!(txn, dbi, "a", "1") + LMDB.put!(txn, dbi, "b", "2") + + LMDB.open(txn, dbi) do cur + @test LMDB.seek!(cur, "a", String) == "a" + LMDB.delete!(cur) # removes "a", cursor now on "b" + LMDB.delete!(cur) # removes "b" + @test_throws LMDBError LMDB.delete!(cur) # no live entry end + @test LMDB.tryget(txn, dbi, "a", String) === nothing + @test LMDB.tryget(txn, dbi, "b", String) === nothing end end end end + +end # @testset "Cursor" diff --git a/test/dbi.jl b/test/dbi.jl index 17c9631..0bdb87c 100644 --- a/test/dbi.jl +++ b/test/dbi.jl @@ -1,170 +1,162 @@ -module LMDB_DBI - using LMDB - using Test +using LMDB +using Test - const dbname = "testdb" - key = 10 - val = "key value is " +@testset "DBI" begin - # Create dir - mkdir(dbname) - try +key = 10 +val = "key value is " - # Procedural style - env = create() - try - open(env, dbname) - txn = start(env) - dbi = open(txn) - put!(txn, dbi, key+1, val*string(key+1)) - put!(txn, dbi, key, val*string(key)) - put!(txn, dbi, key+2, key+2) - put!(txn, dbi, key+3, [key, key+1, key+2]) - @test isopen(txn) - commit(txn) - @test !isopen(txn) - close(env, dbi) - @test !isopen(dbi) - finally - close(env) - end - @test !isopen(env) - - # Block style - create() do env - set!(env, LMDB.MDB_NOSYNC) - open(env, dbname) - start(env) do txn - open(txn, flags = Cuint(LMDB.MDB_REVERSEKEY)) do dbi - k = key - value = get(txn, dbi, k, String) - println("Got value for key $(k): $(value)") - @test value == val*string(k) - delete!(txn, dbi, k) - k += 1 - value = get(txn, dbi, k, String) - println("Got value for key $(k): $(value)") - @test value == val*string(k) - delete!(txn, dbi, k, value) - @test_throws LMDBError get(txn, dbi, k, String) - k += 1 - value = get(txn, dbi, k, Int) - println("Got value for key $(k): $(value)") - @test value == k - k += 1 - value = get(txn, dbi, k, Vector{Int}) - println("Got value for key $(k): $(value)") - @test value == [key, key+1, key+2] - end +# Procedural style + block style smoke test, exercising String, Int, and +# Vector{Int} round-trips through put!/get/delete!. +mktempdir() do dbname + env = create() + try + open(env, dbname) + txn = start(env) + dbi = open(txn) + put!(txn, dbi, key+1, val*string(key+1)) + put!(txn, dbi, key, val*string(key)) + put!(txn, dbi, key+2, key+2) + put!(txn, dbi, key+3, [key, key+1, key+2]) + @test isopen(txn) + commit(txn) + @test !isopen(txn) + close(env, dbi) + @test !isopen(dbi) + finally + close(env) + end + @test !isopen(env) + + # Block style + create() do env + set!(env, LMDB.MDB_NOSYNC) + open(env, dbname) + start(env) do txn + open(txn, flags = Cuint(LMDB.MDB_REVERSEKEY)) do dbi + k = key + value = get(txn, dbi, k, String) + @test value == val*string(k) + delete!(txn, dbi, k) + k += 1 + value = get(txn, dbi, k, String) + @test value == val*string(k) + delete!(txn, dbi, k, value) + @test_throws LMDBError get(txn, dbi, k, String) + k += 1 + value = get(txn, dbi, k, Int) + @test value == k + k += 1 + value = get(txn, dbi, k, Vector{Int}) + @test value == [key, key+1, key+2] end end - - finally - rm(dbname, recursive=true) end +end - # tryget / get-with-default / stat(txn, dbi) — fresh env so the entry - # count is deterministic. - mktempdir() do dir - environment(dir) do env - start(env) do txn - open(txn) do dbi - LMDB.put!(txn, dbi, "k1", "v1") - LMDB.put!(txn, dbi, "k2", "v2") - - @test LMDB.tryget(txn, dbi, "k1", String) == "v1" - @test LMDB.tryget(txn, dbi, "missing", String) === nothing - @test get(txn, dbi, "k2", String, "fallback") == "v2" - @test get(txn, dbi, "missing", String, "fallback") == "fallback" - - s = LMDB.stat(txn, dbi) - @test s isa NamedTuple - @test s.entries == 2 - @test s.psize > 0 - end +# tryget / get-with-default / stat(txn, dbi) — fresh env so the entry +# count is deterministic. +mktempdir() do dir + environment(dir) do env + start(env) do txn + open(txn) do dbi + LMDB.put!(txn, dbi, "k1", "v1") + LMDB.put!(txn, dbi, "k2", "v2") + + @test LMDB.tryget(txn, dbi, "k1", String) == "v1" + @test LMDB.tryget(txn, dbi, "missing", String) === nothing + @test get(txn, dbi, "k2", String, "fallback") == "v2" + @test get(txn, dbi, "missing", String, "fallback") == "fallback" + + s = LMDB.stat(txn, dbi) + @test s isa NamedTuple + @test s.entries == 2 + @test s.psize > 0 end end end +end - # put_reserved!: callback-style MDB_RESERVE write. - mktempdir() do dir - environment(dir) do env - start(env) do txn - open(txn) do dbi - # Write a 16-byte value where bytes 0..7 are a UInt64 - # header and bytes 8..15 are payload. The buffer hands - # back is the LMDB-allocated mmap page; we fill it - # in place — no intermediate Vector. - LMDB.put_reserved!(txn, dbi, "framed", 16) do buf - @test buf isa Vector{UInt8} - @test length(buf) == 16 - unsafe_store!(Ptr{UInt64}(pointer(buf)), - htol(UInt64(0xdeadbeef))) - for i in 1:8 - buf[8 + i] = UInt8(i) - end - end - raw = LMDB.tryget(txn, dbi, "framed", Vector{UInt8}) - @test length(raw) == 16 - @test ltoh(reinterpret(UInt64, raw[1:8])[1]) == - UInt64(0xdeadbeef) - @test raw[9:16] == UInt8[1, 2, 3, 4, 5, 6, 7, 8] - - # Return value: whatever the callback returns. - rv = LMDB.put_reserved!(txn, dbi, "rv", 4) do buf - fill!(buf, 0xab) - :sentinel +# put_reserved!: callback-style MDB_RESERVE write. +mktempdir() do dir + environment(dir) do env + start(env) do txn + open(txn) do dbi + # Write a 16-byte value where bytes 0..7 are a UInt64 + # header and bytes 8..15 are payload. The buffer hands + # back is the LMDB-allocated mmap page; we fill it + # in place — no intermediate Vector. + LMDB.put_reserved!(txn, dbi, "framed", 16) do buf + @test buf isa Vector{UInt8} + @test length(buf) == 16 + unsafe_store!(Ptr{UInt64}(pointer(buf)), + htol(UInt64(0xdeadbeef))) + for i in 1:8 + buf[8 + i] = UInt8(i) end - @test rv === :sentinel end + raw = LMDB.tryget(txn, dbi, "framed", Vector{UInt8}) + @test length(raw) == 16 + @test ltoh(reinterpret(UInt64, raw[1:8])[1]) == + UInt64(0xdeadbeef) + @test raw[9:16] == UInt8[1, 2, 3, 4, 5, 6, 7, 8] + + # Return value: whatever the callback returns. + rv = LMDB.put_reserved!(txn, dbi, "rv", 4) do buf + fill!(buf, 0xab) + :sentinel + end + @test rv === :sentinel end end end +end - # delete!: Bool-returning, idempotent on MDB_NOTFOUND. - mktempdir() do dir - environment(dir) do env - start(env) do txn - open(txn) do dbi - LMDB.put!(txn, dbi, "k1", "v1") - LMDB.put!(txn, dbi, "k2", "v2") - - # Present key → true, returns and entry is gone. - @test LMDB.delete!(txn, dbi, "k1") === true - @test LMDB.tryget(txn, dbi, "k1", String) === nothing - - # Missing key → false, no exception. - @test LMDB.delete!(txn, dbi, "ghost") === false - @test LMDB.delete!(txn, dbi, "k1") === false # already gone - - # Idempotent: a second delete on the same key is a no-op. - @test LMDB.delete!(txn, dbi, "k2") === true - @test LMDB.delete!(txn, dbi, "k2") === false - end +# delete!: Bool-returning, idempotent on MDB_NOTFOUND. +mktempdir() do dir + environment(dir) do env + start(env) do txn + open(txn) do dbi + LMDB.put!(txn, dbi, "k1", "v1") + LMDB.put!(txn, dbi, "k2", "v2") + + # Present key → true, returns and entry is gone. + @test LMDB.delete!(txn, dbi, "k1") === true + @test LMDB.tryget(txn, dbi, "k1", String) === nothing + + # Missing key → false, no exception. + @test LMDB.delete!(txn, dbi, "ghost") === false + @test LMDB.delete!(txn, dbi, "k1") === false # already gone + + # Idempotent: a second delete on the same key is a no-op. + @test LMDB.delete!(txn, dbi, "k2") === true + @test LMDB.delete!(txn, dbi, "k2") === false end end end +end - # replace! / pop! - mktempdir() do dir - environment(dir) do env - start(env) do txn - open(txn) do dbi - # replace! on a missing key returns nothing and creates the entry. - @test LMDB.replace!(txn, dbi, "k", "v1") === nothing - @test LMDB.tryget(txn, dbi, "k", String) == "v1" - - # replace! on an existing key returns the old value. - @test LMDB.replace!(txn, dbi, "k", "v2") == "v1" - @test LMDB.tryget(txn, dbi, "k", String) == "v2" - - # pop! returns the value and deletes. - @test LMDB.pop!(txn, dbi, "k", String) == "v2" - @test LMDB.tryget(txn, dbi, "k", String) === nothing - # pop! on a missing key returns nothing. - @test LMDB.pop!(txn, dbi, "k", String) === nothing - end +# replace! / pop! +mktempdir() do dir + environment(dir) do env + start(env) do txn + open(txn) do dbi + # replace! on a missing key returns nothing and creates the entry. + @test LMDB.replace!(txn, dbi, "k", "v1") === nothing + @test LMDB.tryget(txn, dbi, "k", String) == "v1" + + # replace! on an existing key returns the old value. + @test LMDB.replace!(txn, dbi, "k", "v2") == "v1" + @test LMDB.tryget(txn, dbi, "k", String) == "v2" + + # pop! returns the value and deletes. + @test LMDB.pop!(txn, dbi, "k", String) == "v2" + @test LMDB.tryget(txn, dbi, "k", String) === nothing + # pop! on a missing key returns nothing. + @test LMDB.pop!(txn, dbi, "k", String) === nothing end end end end + +end # @testset "DBI" diff --git a/test/dupsort.jl b/test/dupsort.jl index 2ba2bb1..2304a41 100644 --- a/test/dupsort.jl +++ b/test/dupsort.jl @@ -1,60 +1,62 @@ -module LMDB_DupSort - using LMDB - using Test - - # DupSort: a single key can hold multiple sorted values. Verify the - # dup-aware cursor ops navigate within and across keys correctly, and - # that delete!(txn, dbi, key, val) removes one specific dup. - mktempdir() do dir - environment(dir) do env - start(env) do txn - open(txn, flags = MDB_DUPSORT) do dbi - LMDB.put!(txn, dbi, "k1", "a") - LMDB.put!(txn, dbi, "k1", "b") - LMDB.put!(txn, dbi, "k1", "c") - LMDB.put!(txn, dbi, "k2", "x") - LMDB.put!(txn, dbi, "k2", "y") - - LMDB.open(txn, dbi) do cur - # Position at first entry; walk through k1's dups. - @test LMDB.seek!(cur, String) == "k1" - @test LMDB.value(cur, String) == "a" - @test LMDB.next_dup!(cur, String) == "b" - @test LMDB.next_dup!(cur, String) == "c" - @test LMDB.next_dup!(cur, String) === nothing # no more dups - - # Jump to next key, skipping any remaining dups (none here). - @test LMDB.next_nodup!(cur, String) == "k2" - @test LMDB.value(cur, String) == "x" - @test LMDB.next_dup!(cur, String) == "y" - - # Reset to first dup of current key. - @test LMDB.seek_first_dup!(cur, String) == "x" - @test LMDB.seek_last_dup!(cur, String) == "y" - @test LMDB.prev_dup!(cur, String) == "x" - - # prev_nodup! moves back to the last dup of the previous key. - @test LMDB.prev_nodup!(cur, String) == "k1" - @test LMDB.value(cur, String) == "c" - end - - # Count dups for k1 via cursor count(). - LMDB.open(txn, dbi) do cur - @test LMDB.seek!(cur, "k1", String) == "k1" - @test count(cur) == 3 - end - - # Dup-aware delete: delete!(txn, dbi, key, val) removes only - # that one duplicate. - LMDB.delete!(txn, dbi, "k1", "b") - LMDB.open(txn, dbi) do cur - @test LMDB.seek!(cur, "k1", String) == "k1" - @test LMDB.value(cur, String) == "a" - @test LMDB.next_dup!(cur, String) == "c" # "b" is gone - @test LMDB.next_dup!(cur, String) === nothing - end +using LMDB +using Test + +@testset "DupSort" begin + +# DupSort: a single key can hold multiple sorted values. Verify the +# dup-aware cursor ops navigate within and across keys correctly, and +# that delete!(txn, dbi, key, val) removes one specific dup. +mktempdir() do dir + environment(dir) do env + start(env) do txn + open(txn, flags = MDB_DUPSORT) do dbi + LMDB.put!(txn, dbi, "k1", "a") + LMDB.put!(txn, dbi, "k1", "b") + LMDB.put!(txn, dbi, "k1", "c") + LMDB.put!(txn, dbi, "k2", "x") + LMDB.put!(txn, dbi, "k2", "y") + + LMDB.open(txn, dbi) do cur + # Position at first entry; walk through k1's dups. + @test LMDB.seek!(cur, String) == "k1" + @test LMDB.value(cur, String) == "a" + @test LMDB.next_dup!(cur, String) == "b" + @test LMDB.next_dup!(cur, String) == "c" + @test LMDB.next_dup!(cur, String) === nothing # no more dups + + # Jump to next key, skipping any remaining dups (none here). + @test LMDB.next_nodup!(cur, String) == "k2" + @test LMDB.value(cur, String) == "x" + @test LMDB.next_dup!(cur, String) == "y" + + # Reset to first dup of current key. + @test LMDB.seek_first_dup!(cur, String) == "x" + @test LMDB.seek_last_dup!(cur, String) == "y" + @test LMDB.prev_dup!(cur, String) == "x" + + # prev_nodup! moves back to the last dup of the previous key. + @test LMDB.prev_nodup!(cur, String) == "k1" + @test LMDB.value(cur, String) == "c" + end + + # Count dups for k1 via cursor count(). + LMDB.open(txn, dbi) do cur + @test LMDB.seek!(cur, "k1", String) == "k1" + @test count(cur) == 3 + end + + # Dup-aware delete: delete!(txn, dbi, key, val) removes only + # that one duplicate. + LMDB.delete!(txn, dbi, "k1", "b") + LMDB.open(txn, dbi) do cur + @test LMDB.seek!(cur, "k1", String) == "k1" + @test LMDB.value(cur, String) == "a" + @test LMDB.next_dup!(cur, String) == "c" # "b" is gone + @test LMDB.next_dup!(cur, String) === nothing end end end end end + +end # @testset "DupSort" diff --git a/test/env.jl b/test/env.jl index 49a13ae..0696586 100644 --- a/test/env.jl +++ b/test/env.jl @@ -1,125 +1,204 @@ -module LMDB_Env - using LMDB - using Test - - const dbname = "testdb" - - # Open environemnt - env = create() - @test env.handle != C_NULL - @test env[:Readers] == 126 - @test env[:KeySize] == 511 - @test env[:Flags] == 0 - - # Manipulate flags - @test !isflagset(env[:Flags], Cuint(LMDB.MDB_NOSYNC)) - set!(env, LMDB.MDB_NOSYNC) - @test isflagset(env[:Flags], Cuint(LMDB.MDB_NOSYNC)) - unset!(env, LMDB.MDB_NOSYNC) - @test !isflagset(env[:Flags], Cuint(LMDB.MDB_NOSYNC)) - - # Parameters - @test (env[:Readers] = 100) == 100 - @test (env[:MapSize] = 1000^2) == 1000^2 - @test (env[:DBs] = 10) == 10 - @test env[:Readers] == 100 - - # Map sizes >4 GiB must not silently truncate (issue #38). - big_mapsize = Csize_t(5) * 1024 * 1024 * 1024 - @test (env[:MapSize] = big_mapsize) == big_mapsize - - # :Flags setindex! used to fall through to a warn(...) that no longer - # exists, throwing UndefVarError (issue #24). - env[:Flags] = LMDB.MDB_NOSYNC - @test isflagset(env[:Flags], Cuint(LMDB.MDB_NOSYNC)) - unset!(env, LMDB.MDB_NOSYNC) - - # Unknown options used to be silently swallowed. - @test_throws ArgumentError env[:Bogus] = 1 - - # open db - isdir(dbname) || mkdir(dbname) +using LMDB +using Test + +@testset "Environment" begin + +# Open environment +env = create() +@test env.handle != C_NULL +@test env[:Readers] == 126 +@test env[:KeySize] == 511 +@test env[:Flags] == 0 + +# Manipulate flags +@test !isflagset(env[:Flags], Cuint(LMDB.MDB_NOSYNC)) +set!(env, LMDB.MDB_NOSYNC) +@test isflagset(env[:Flags], Cuint(LMDB.MDB_NOSYNC)) +unset!(env, LMDB.MDB_NOSYNC) +@test !isflagset(env[:Flags], Cuint(LMDB.MDB_NOSYNC)) + +# Parameters +@test (env[:Readers] = 100) == 100 +@test (env[:MapSize] = 1000^2) == 1000^2 +@test (env[:DBs] = 10) == 10 +@test env[:Readers] == 100 + +# MapSize must accept values that don't fit in Cuint (#38, PR #37, #40). +big = Csize_t(8) * 1024^3 # 8 GiB +@test (env[:MapSize] = big) == big + +# Setting :Flags via setindex! used to fall through to a warning (#24). +@test !isflagset(env[:Flags], Cuint(LMDB.MDB_NOSYNC)) +env[:Flags] = LMDB.MDB_NOSYNC +@test isflagset(env[:Flags], Cuint(LMDB.MDB_NOSYNC)) +unset!(env, LMDB.MDB_NOSYNC) + +# Unknown options error instead of silently warning + returning bogus values. +@test_throws ArgumentError env[:Bogus] = 1 +@test_throws ArgumentError env[:Bogus] + +# Open a DB on the env, then close it. +mktempdir() do dir + ret = open(env, dir) + @test ret[1] == 0 + + # stat(env) returns the main DB's stats; before any puts, there are + # no entries and a positive page size. + s = stat(env) + @test s isa NamedTuple + @test s.psize > 0 + @test s.entries == 0 + + # Close environment + close(env) + @test !isopen(env) + + # do block + create() do env + set!(env, LMDB.MDB_NOSYNC) + open(env, dir) + @test isopen(env) + end +end + +# High-level Environment(path; ...) constructor. +mktempdir() do dir + big = Csize_t(8) * 1024^3 + env = Environment(dir; mapsize = big, maxreaders = 42, maxdbs = 4, + flags = MDB_NOSYNC | MDB_NOTLS) try - ret = open(env, dbname) - @test ret[1] == 0 + @test isopen(env) + @test env[:Readers] == 42 + @test info(env).mapsize == big + @test isflagset(env[:Flags], Cuint(MDB_NOSYNC)) + @test isflagset(env[:Flags], Cuint(MDB_NOTLS)) + finally + close(env) + end - # stat(env) returns the main DB's stats; before any puts, there are - # no entries and a positive page size. - s = stat(env) - @test s isa NamedTuple - @test s.psize > 0 - @test s.entries == 0 + # On failure during open, the Environment ctor closes the partial env. + @test_throws LMDBError Environment(joinpath(dir, "definitely_does_not_exist")) +end - # Close environment +# Finalizers: an abandoned write txn must be aborted by GC so a later +# write txn doesn't block on LMDB's exclusive write mutex. If the +# finalizer doesn't fire, `start(env)` below would deadlock. +mktempdir() do dir + env = Environment(dir) + try + # Open a write txn and let it become unreachable without commit/abort. + let txn = start(env) + @test isopen(txn) + end + GC.gc(); GC.gc() + # If the finalizer aborted the abandoned txn, this succeeds. + txn2 = start(env) + try + dbi = open(txn2) + LMDB.put!(txn2, dbi, "k", "v") + finally + commit(txn2) + end + finally close(env) - @test !isopen(env) + end +end - # do block - create() do env - set!(env, LMDB.MDB_NOSYNC) - open(env, dbname) - @test isopen(env) +# Cursor finalizer: an abandoned cursor must be cleaned up so its +# parent txn can commit. (LMDB requires cursors on a write txn to be +# closed before commit; for read txns it's safer too.) +mktempdir() do dir + env = Environment(dir) + try + start(env) do txn + dbi = open(txn) + let cur = LMDB.open(txn, dbi) + @test isopen(cur) + end # cur out of scope + GC.gc(); GC.gc() + # If the finalizer ran, we can still use the txn. + LMDB.put!(txn, dbi, "k", "v") end finally - rm(dbname, recursive=true) + close(env) + end +end + +# Cursor finalizer is safe even after its parent txn has been +# explicitly committed: write-txn cursors are freed by the txn's +# commit per `lmdb.h`, so `mdb_cursor_close` afterwards would be UB. +# The defensive check in `close(::Cursor)` skips the LMDB call once +# the parent txn handle is gone. +mktempdir() do dir + env = Environment(dir) + try + txn = start(env) + dbi = open(txn) + cur = LMDB.open(txn, dbi) + LMDB.put!(txn, dbi, "k", "v") + commit(txn) # invalidates write-txn cursors + @test !isopen(txn) + cur = nothing # drop the binding + GC.gc(); GC.gc() # finalizer should be a no-op + finally + close(env) end +end - # High-level Environment(path; ...) constructor. - mktempdir() do dir - big = Csize_t(8) * 1024^3 - env = Environment(dir; mapsize = big, maxreaders = 42, maxdbs = 4, - flags = MDB_NOSYNC | MDB_NOTLS) - try - @test isopen(env) - @test env[:Readers] == 42 - @test info(env).mapsize == big - @test isflagset(env[:Flags], Cuint(MDB_NOSYNC)) - @test isflagset(env[:Flags], Cuint(MDB_NOTLS)) - finally - close(env) +# Parent refs: env(txn) and transaction(cur) return the actual parents. +mktempdir() do dir + env = Environment(dir) + try + start(env) do txn + @test LMDB.env(txn) === env + dbi = open(txn) + LMDB.open(txn, dbi) do cur + @test LMDB.transaction(cur) === txn + end end - - # On failure during open, the Environment ctor closes the partial env. - @test_throws LMDBError Environment(joinpath(dir, "definitely_does_not_exist")) + finally + close(env) end +end - # reader_check / reader_list / copy - mktempdir() do dir - environment(dir) do env - # Fresh env: no stale readers. - @test reader_check(env) == 0 - - # reader_list always emits a header line listing slot fields. - txt = reader_list(env) - @test txt isa String - @test !isempty(txt) - - # Round-trip a copy. - start(env) do txn - open(txn) do dbi - LMDB.put!(txn, dbi, "k", "v") - end +# reader_check / reader_list / copy +mktempdir() do dir + environment(dir) do env + # Fresh env: no stale readers. + @test reader_check(env) == 0 + + # reader_list always emits a header line listing slot fields. + txt = reader_list(env) + @test txt isa String + @test !isempty(txt) + + # Round-trip a copy. + start(env) do txn + open(txn) do dbi + LMDB.put!(txn, dbi, "k", "v") end - mktempdir() do dst - copy(env, dst) - environment(dst) do env2 - start(env2) do txn - open(txn) do dbi - @test LMDB.tryget(txn, dbi, "k", String) == "v" - end + end + mktempdir() do dst + copy(env, dst) + environment(dst) do env2 + start(env2) do txn + open(txn) do dbi + @test LMDB.tryget(txn, dbi, "k", String) == "v" end end end - mktempdir() do dst - copy(env, dst; compact=true) - environment(dst) do env2 - start(env2) do txn - open(txn) do dbi - @test LMDB.tryget(txn, dbi, "k", String) == "v" - end + end + mktempdir() do dst + copy(env, dst; compact=true) + environment(dst) do env2 + start(env2) do txn + open(txn) do dbi + @test LMDB.tryget(txn, dbi, "k", String) == "v" end end end end end end + +end # @testset "Environment" diff --git a/test/integration.jl b/test/integration.jl index bd8387d..1754e79 100644 --- a/test/integration.jl +++ b/test/integration.jl @@ -1,106 +1,108 @@ -module LMDB_Integration - using LMDB - using Test +using LMDB +using Test - # cuTile-shaped framed value: 8-byte LE atime prefix, then the payload. - # Defined here at module scope to demonstrate the `Base.read(::IO, …)` - # extension contract and to regression-guard it (a downstream package - # — cuTile's DiskCache — relies on being able to plug in its own - # decoder against the abstract `IO`, without depending on - # `LMDB.MDBValueIO` in its own type signatures). - struct AtimedBlob end - const _ATIME_PREFIX = 8 +# cuTile-shaped framed value: 8-byte LE atime prefix, then the payload. +# Defined at file scope to demonstrate the `Base.read(::IO, …)` +# extension contract and to regression-guard it (a downstream package +# — cuTile's DiskCache — relies on being able to plug in its own +# decoder against the abstract `IO`, without depending on +# `LMDB.MDBValueIO` in its own type signatures). +struct AtimedBlob end +const _ATIME_PREFIX = 8 - function Base.read(io::IO, ::Type{AtimedBlob}) - bytesavailable(io) < _ATIME_PREFIX && return UInt8[] - skip(io, _ATIME_PREFIX) - return read(io, Vector{UInt8}) - end +function Base.read(io::IO, ::Type{AtimedBlob}) + bytesavailable(io) < _ATIME_PREFIX && return UInt8[] + skip(io, _ATIME_PREFIX) + return read(io, Vector{UInt8}) +end - pack_atimed(atime::UInt64, payload::Vector{UInt8}) = begin - out = Vector{UInt8}(undef, _ATIME_PREFIX + length(payload)) - GC.@preserve out unsafe_store!(Ptr{UInt64}(pointer(out)), htol(atime)) - copyto!(out, _ATIME_PREFIX + 1, payload, 1, length(payload)) - out - end +pack_atimed(atime::UInt64, payload::Vector{UInt8}) = begin + out = Vector{UInt8}(undef, _ATIME_PREFIX + length(payload)) + GC.@preserve out unsafe_store!(Ptr{UInt64}(pointer(out)), htol(atime)) + copyto!(out, _ATIME_PREFIX + 1, payload, 1, length(payload)) + out +end - # Power-user pattern: open an env via the Environment kwargs ctor, run - # a write txn through tier-2, then a read txn through a cursor walk - # using only tier-2 + raw MDB_val refs (the shape cuTile.DiskCache - # follows). Regression guard: ensures no future change breaks the - # `walk(...) do k_ref, v_ref` zero-copy idiom. - mktempdir() do dir - env = Environment(dir; - mapsize = 1 << 28, - maxreaders = 64, - flags = MDB_NOTLS | MDB_NORDAHEAD) - try - dbi, psize = start(env) do txn - d = open(txn) - (d, LMDB.stat(txn, d).psize) - end - @test psize > 0 +@testset "Integration" begin - # Populate. - start(env) do txn - for i in 1:5 - LMDB.put!(txn, dbi, "key$(i)", "value$(i)") - end +# Power-user pattern: open an env via the Environment kwargs ctor, run +# a write txn through tier-2, then a read txn through a cursor walk +# using only tier-2 + raw MDB_val refs (the shape cuTile.DiskCache +# follows). Regression guard: ensures no future change breaks the +# `walk(...) do k_ref, v_ref` zero-copy idiom. +mktempdir() do dir + env = Environment(dir; + mapsize = 1 << 28, + maxreaders = 64, + flags = MDB_NOTLS | MDB_NORDAHEAD) + try + dbi, psize = start(env) do txn + d = open(txn) + (d, LMDB.stat(txn, d).psize) + end + @test psize > 0 + + # Populate. + start(env) do txn + for i in 1:5 + LMDB.put!(txn, dbi, "key$(i)", "value$(i)") end + end - # Tier-2 read txn + cursor walk over the LMDB-owned mmap, like - # cuTile's eviction scan: zero allocations beyond the per-entry - # tuple. - entries = Tuple{String, Int}[] - start(env; flags = MDB_RDONLY) do txn - LMDB.open(txn, dbi) do cur - LMDB.walk(cur) do k_ref, v_ref - kv = k_ref[]; vv = v_ref[] - k = unsafe_string(Ptr{UInt8}(kv.mv_data), kv.mv_size) - push!(entries, (k, Int(vv.mv_size))) - end + # Tier-2 read txn + cursor walk over the LMDB-owned mmap, like + # cuTile's eviction scan: zero allocations beyond the per-entry + # tuple. + entries = Tuple{String, Int}[] + start(env; flags = MDB_RDONLY) do txn + LMDB.open(txn, dbi) do cur + LMDB.walk(cur) do k_ref, v_ref + kv = k_ref[]; vv = v_ref[] + k = unsafe_string(Ptr{UInt8}(kv.mv_data), kv.mv_size) + push!(entries, (k, Int(vv.mv_size))) end end - @test length(entries) == 5 - @test first.(entries) == ["key$i" for i in 1:5] - @test all(e -> e[2] == sizeof("value1"), entries) + end + @test length(entries) == 5 + @test first.(entries) == ["key$i" for i in 1:5] + @test all(e -> e[2] == sizeof("value1"), entries) - # tryget vs is_notfound — common cuTile-shaped read path. - start(env; flags = MDB_RDONLY) do txn - @test LMDB.tryget(txn, dbi, "key3", String) == "value3" - @test LMDB.tryget(txn, dbi, "ghost", String) === nothing - end + # tryget vs is_notfound — common cuTile-shaped read path. + start(env; flags = MDB_RDONLY) do txn + @test LMDB.tryget(txn, dbi, "key3", String) == "value3" + @test LMDB.tryget(txn, dbi, "ghost", String) === nothing + end - # Batch delete: present keys return true, missing return false, - # no exception on either path. cuTile's `_delete_batch!` uses - # the Bool to count actual evictions. - deleted = 0 - start(env) do txn - for k in ["key1", "ghost", "key3"] - LMDB.delete!(txn, dbi, k) && (deleted += 1) - end - end - @test deleted == 2 - start(env; flags = MDB_RDONLY) do txn - @test LMDB.tryget(txn, dbi, "key1", String) === nothing - @test LMDB.tryget(txn, dbi, "key2", String) == "value2" - @test LMDB.tryget(txn, dbi, "key3", String) === nothing + # Batch delete: present keys return true, missing return false, + # no exception on either path. cuTile's `_delete_batch!` uses + # the Bool to count actual evictions. + deleted = 0 + start(env) do txn + for k in ["key1", "ghost", "key3"] + LMDB.delete!(txn, dbi, k) && (deleted += 1) end + end + @test deleted == 2 + start(env; flags = MDB_RDONLY) do txn + @test LMDB.tryget(txn, dbi, "key1", String) === nothing + @test LMDB.tryget(txn, dbi, "key2", String) == "value2" + @test LMDB.tryget(txn, dbi, "key3", String) === nothing + end - # MDBValueIO extension: write an 8-byte-prefixed framed value - # and read it back via `tryget(..., AtimedBlob)`, getting the - # payload tail with one alloc + skip + copy, no slicing. - payload = Vector{UInt8}("cubin-bytes-here") - atime = UInt64(0xdeadbeefcafebabe) - start(env) do txn - LMDB.put!(txn, dbi, "framed", pack_atimed(atime, payload)) - end - start(env; flags = MDB_RDONLY) do txn - @test LMDB.tryget(txn, dbi, "framed", AtimedBlob) == payload - @test LMDB.tryget(txn, dbi, "ghost", AtimedBlob) === nothing - end - finally - close(env) + # MDBValueIO extension: write an 8-byte-prefixed framed value + # and read it back via `tryget(..., AtimedBlob)`, getting the + # payload tail with one alloc + skip + copy, no slicing. + payload = Vector{UInt8}("cubin-bytes-here") + atime = UInt64(0xdeadbeefcafebabe) + start(env) do txn + LMDB.put!(txn, dbi, "framed", pack_atimed(atime, payload)) end + start(env; flags = MDB_RDONLY) do txn + @test LMDB.tryget(txn, dbi, "framed", AtimedBlob) == payload + @test LMDB.tryget(txn, dbi, "ghost", AtimedBlob) === nothing + end + finally + close(env) end end + +end # @testset "Integration" diff --git a/test/liblmdb.jl b/test/liblmdb.jl index 931c40b..586a317 100644 --- a/test/liblmdb.jl +++ b/test/liblmdb.jl @@ -1,54 +1,56 @@ -module LMDB_LibLMDB - using LMDB - using Test - - # @checked: status-returning bindings auto-throw an LMDBError on a - # non-zero return. - mktempdir() do dir - env_ref = Ref{Ptr{LMDB.MDB_env}}(C_NULL) - LMDB.mdb_env_create(env_ref) - env = env_ref[] - try - # Opening a *file* (not a directory) when MDB_NOSUBDIR isn't - # set should fail with a non-zero status that @checked turns - # into an LMDBError. - f = touch(joinpath(dir, "not_a_dir")) - @test_throws LMDBError LMDB.mdb_env_open(env, - f, Cuint(0), LMDB.mode_t(0o644)) - finally - LMDB.mdb_env_close(env) - end +using LMDB +using Test + +@testset "liblmdb" begin + +# @checked: status-returning bindings auto-throw an LMDBError on a +# non-zero return. +mktempdir() do dir + env_ref = Ref{Ptr{LMDB.MDB_env}}(C_NULL) + LMDB.mdb_env_create(env_ref) + env = env_ref[] + try + # Opening a *file* (not a directory) when MDB_NOSUBDIR isn't + # set should fail with a non-zero status that @checked turns + # into an LMDBError. + f = touch(joinpath(dir, "not_a_dir")) + @test_throws LMDBError LMDB.mdb_env_open(env, + f, Cuint(0), LMDB.mode_t(0o644)) + finally + LMDB.mdb_env_close(env) end +end + +# unchecked_*: returns the raw Cint without throwing — caller decides. +mktempdir() do dir + env_ref = Ref{Ptr{LMDB.MDB_env}}(C_NULL) + LMDB.mdb_env_create(env_ref) + env = env_ref[] + try + LMDB.mdb_env_open(env, dir, Cuint(0), LMDB.mode_t(0o755)) + + txn_ref = Ref{Ptr{LMDB.MDB_txn}}(C_NULL) + LMDB.mdb_txn_begin(env, C_NULL, Cuint(0), txn_ref) + txn = txn_ref[] - # unchecked_*: returns the raw Cint without throwing — caller decides. - mktempdir() do dir - env_ref = Ref{Ptr{LMDB.MDB_env}}(C_NULL) - LMDB.mdb_env_create(env_ref) - env = env_ref[] - try - LMDB.mdb_env_open(env, dir, Cuint(0), LMDB.mode_t(0o755)) - - txn_ref = Ref{Ptr{LMDB.MDB_txn}}(C_NULL) - LMDB.mdb_txn_begin(env, C_NULL, Cuint(0), txn_ref) - txn = txn_ref[] - - dbi_ref = Ref{LMDB.MDB_dbi}(0) - LMDB.mdb_dbi_open(txn, C_NULL, Cuint(0), dbi_ref) - dbi = dbi_ref[] - - # Look up a key that doesn't exist — unchecked returns - # MDB_NOTFOUND, no exception. - key = "missing" - val_ref = Ref(LMDB.MDBValue()) - ret = LMDB.unchecked_mdb_get(txn, dbi, key, val_ref) - @test ret == LMDB.MDB_NOTFOUND - - # The checked counterpart throws. - @test_throws LMDBError LMDB.mdb_get(txn, dbi, key, val_ref) - - LMDB.mdb_txn_abort(txn) - finally - LMDB.mdb_env_close(env) - end + dbi_ref = Ref{LMDB.MDB_dbi}(0) + LMDB.mdb_dbi_open(txn, C_NULL, Cuint(0), dbi_ref) + dbi = dbi_ref[] + + # Look up a key that doesn't exist — unchecked returns + # MDB_NOTFOUND, no exception. + key = "missing" + val_ref = Ref(LMDB.MDBValue()) + ret = LMDB.unchecked_mdb_get(txn, dbi, key, val_ref) + @test ret == LMDB.MDB_NOTFOUND + + # The checked counterpart throws. + @test_throws LMDBError LMDB.mdb_get(txn, dbi, key, val_ref) + + LMDB.mdb_txn_abort(txn) + finally + LMDB.mdb_env_close(env) end end + +end # @testset "liblmdb" From d1b23e4f48e23b6aeabda5f8524fe72ac66cfca5 Mon Sep 17 00:00:00 2001 From: Tim Besard Date: Sun, 3 May 2026 18:18:14 +0200 Subject: [PATCH 2/8] Update docs. --- Project.toml | 3 + docs/Manifest.toml | 181 ---------------------------------- docs/Project.toml | 3 + docs/api/index.md | 17 ---- docs/make.jl | 57 +++++++---- docs/src/api/index.md | 17 ---- docs/src/index.md | 55 +++++++++-- docs/src/lib/cursors.md | 66 +++++++++++++ docs/src/lib/databases.md | 56 +++++++++++ docs/src/lib/dict.md | 48 +++++++++ docs/src/lib/environments.md | 60 +++++++++++ docs/src/lib/errors.md | 60 +++++++++++ docs/src/lib/lowlevel.md | 162 ++++++++++++++++++++++++++++++ docs/src/lib/transactions.md | 42 ++++++++ docs/src/man/cursors.md | 186 +++++++++++++++++++++++++++++++++++ docs/src/man/databases.md | 141 ++++++++++++++++++++++++++ docs/src/man/dict.md | 145 +++++++++++++++++++++++++++ docs/src/man/dupsort.md | 98 ++++++++++++++++++ docs/src/man/environments.md | 126 ++++++++++++++++++++++++ docs/src/man/essentials.md | 132 +++++++++++++++++++++++++ docs/src/man/lowlevel.md | 151 ++++++++++++++++++++++++++++ docs/src/man/transactions.md | 125 +++++++++++++++++++++++ docs/src/manual.md | 119 ---------------------- 23 files changed, 1691 insertions(+), 359 deletions(-) delete mode 100644 docs/Manifest.toml delete mode 100644 docs/api/index.md delete mode 100644 docs/src/api/index.md create mode 100644 docs/src/lib/cursors.md create mode 100644 docs/src/lib/databases.md create mode 100644 docs/src/lib/dict.md create mode 100644 docs/src/lib/environments.md create mode 100644 docs/src/lib/errors.md create mode 100644 docs/src/lib/lowlevel.md create mode 100644 docs/src/lib/transactions.md create mode 100644 docs/src/man/cursors.md create mode 100644 docs/src/man/databases.md create mode 100644 docs/src/man/dict.md create mode 100644 docs/src/man/dupsort.md create mode 100644 docs/src/man/environments.md create mode 100644 docs/src/man/essentials.md create mode 100644 docs/src/man/lowlevel.md create mode 100644 docs/src/man/transactions.md delete mode 100644 docs/src/manual.md diff --git a/Project.toml b/Project.toml index 849fb1f..ff68ab5 100644 --- a/Project.toml +++ b/Project.toml @@ -3,6 +3,9 @@ uuid = "11f193de-5e89-5f17-923a-7d207d56daf9" authors = ["Art Wild "] version = "1.0.0" +[workspace] +projects = ["docs", "test"] + [deps] CEnum = "fa961155-64e5-5f13-b03f-caf6b980ea82" LMDB_jll = "6206cf0b-f360-5984-af49-5437264c140e" diff --git a/docs/Manifest.toml b/docs/Manifest.toml deleted file mode 100644 index 8869612..0000000 --- a/docs/Manifest.toml +++ /dev/null @@ -1,181 +0,0 @@ -# This file is machine-generated - editing it directly is not advised - -[[ANSIColoredPrinters]] -git-tree-sha1 = "574baf8110975760d391c710b6341da1afa48d8c" -uuid = "a4c015fc-c6ff-483c-b24f-f7ea428134e9" -version = "0.0.1" - -[[ArgTools]] -uuid = "0dad84c5-d112-42e6-8d28-ef12dabb789f" - -[[Artifacts]] -uuid = "56f22d72-fd6d-98f1-02f0-08ddc0907c33" - -[[Base64]] -uuid = "2a0f44e3-6c83-55bd-87e4-b1978d98bd5f" - -[[CEnum]] -git-tree-sha1 = "215a9aa4a1f23fbd05b92769fdd62559488d70e9" -uuid = "fa961155-64e5-5f13-b03f-caf6b980ea82" -version = "0.4.1" - -[[Dates]] -deps = ["Printf"] -uuid = "ade2ca70-3891-5945-98fb-dc099432e06a" - -[[DocStringExtensions]] -deps = ["LibGit2"] -git-tree-sha1 = "b19534d1895d702889b219c382a6e18010797f0b" -uuid = "ffbed154-4ef7-542d-bbb7-c09d3a79fcae" -version = "0.8.6" - -[[Documenter]] -deps = ["ANSIColoredPrinters", "Base64", "Dates", "DocStringExtensions", "IOCapture", "InteractiveUtils", "JSON", "LibGit2", "Logging", "Markdown", "REPL", "Test", "Unicode"] -git-tree-sha1 = "f425293f7e0acaf9144de6d731772de156676233" -uuid = "e30172f5-a6a5-5a46-863b-614d45cd2de4" -version = "0.27.10" - -[[Downloads]] -deps = ["ArgTools", "LibCURL", "NetworkOptions"] -uuid = "f43a241f-c20a-4ad4-852c-f6b1247861c6" - -[[IOCapture]] -deps = ["Logging", "Random"] -git-tree-sha1 = "f7be53659ab06ddc986428d3a9dcc95f6fa6705a" -uuid = "b5f81e59-6552-4d32-b1f0-c071b021bf89" -version = "0.2.2" - -[[InteractiveUtils]] -deps = ["Markdown"] -uuid = "b77e0a4c-d291-57a0-90e8-8db25a27a240" - -[[JLLWrappers]] -deps = ["Preferences"] -git-tree-sha1 = "642a199af8b68253517b80bd3bfd17eb4e84df6e" -uuid = "692b3bcd-3c85-4b1f-b108-f13ce0eb3210" -version = "1.3.0" - -[[JSON]] -deps = ["Dates", "Mmap", "Parsers", "Unicode"] -git-tree-sha1 = "8076680b162ada2a031f707ac7b4953e30667a37" -uuid = "682c06a0-de6a-54ab-a142-c8b1cf79cde6" -version = "0.21.2" - -[[LMDB]] -deps = ["CEnum", "LMDB_jll", "Libdl"] -path = "/home/fgans/julia_depots/dev/LMDB" -uuid = "11f193de-5e89-5f17-923a-7d207d56daf9" -version = "0.2.0" - -[[LMDB_jll]] -deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"] -git-tree-sha1 = "fe4e275790c9a6b331f7e7e10e04a0a0f1e57123" -uuid = "6206cf0b-f360-5984-af49-5437264c140e" -version = "0.9.27+0" - -[[LibCURL]] -deps = ["LibCURL_jll", "MozillaCACerts_jll"] -uuid = "b27032c2-a3e7-50c8-80cd-2d36dbcbfd21" - -[[LibCURL_jll]] -deps = ["Artifacts", "LibSSH2_jll", "Libdl", "MbedTLS_jll", "Zlib_jll", "nghttp2_jll"] -uuid = "deac9b47-8bc7-5906-a0fe-35ac56dc84c0" - -[[LibGit2]] -deps = ["Base64", "NetworkOptions", "Printf", "SHA"] -uuid = "76f85450-5226-5b5a-8eaa-529ad045b433" - -[[LibSSH2_jll]] -deps = ["Artifacts", "Libdl", "MbedTLS_jll"] -uuid = "29816b5a-b9ab-546f-933c-edad1886dfa8" - -[[Libdl]] -uuid = "8f399da3-3557-5675-b5ff-fb832c97cbdb" - -[[Logging]] -uuid = "56ddb016-857b-54e1-b83d-db4d58db5568" - -[[Markdown]] -deps = ["Base64"] -uuid = "d6f4376e-aef5-505a-96c1-9c027394607a" - -[[MbedTLS_jll]] -deps = ["Artifacts", "Libdl"] -uuid = "c8ffd9c3-330d-5841-b78e-0817d7145fa1" - -[[Mmap]] -uuid = "a63ad114-7e13-5084-954f-fe012c677804" - -[[MozillaCACerts_jll]] -uuid = "14a3606d-f60d-562e-9121-12d972cd8159" - -[[NetworkOptions]] -uuid = "ca575930-c2e3-43a9-ace4-1e988b2c1908" - -[[Parsers]] -deps = ["Dates"] -git-tree-sha1 = "ae4bbcadb2906ccc085cf52ac286dc1377dceccc" -uuid = "69de0a69-1ddd-5017-9359-2bf0b02dc9f0" -version = "2.1.2" - -[[Pkg]] -deps = ["Artifacts", "Dates", "Downloads", "LibGit2", "Libdl", "Logging", "Markdown", "Printf", "REPL", "Random", "SHA", "Serialization", "TOML", "Tar", "UUIDs", "p7zip_jll"] -uuid = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f" - -[[Preferences]] -deps = ["TOML"] -git-tree-sha1 = "00cfd92944ca9c760982747e9a1d0d5d86ab1e5a" -uuid = "21216c6a-2e73-6563-6e65-726566657250" -version = "1.2.2" - -[[Printf]] -deps = ["Unicode"] -uuid = "de0858da-6303-5e67-8744-51eddeeeb8d7" - -[[REPL]] -deps = ["InteractiveUtils", "Markdown", "Sockets", "Unicode"] -uuid = "3fa0cd96-eef1-5676-8a61-b3b8758bbffb" - -[[Random]] -deps = ["Serialization"] -uuid = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c" - -[[SHA]] -uuid = "ea8e919c-243c-51af-8825-aaa63cd721ce" - -[[Serialization]] -uuid = "9e88b42a-f829-5b0c-bbe9-9e923198166b" - -[[Sockets]] -uuid = "6462fe0b-24de-5631-8697-dd941f90decc" - -[[TOML]] -deps = ["Dates"] -uuid = "fa267f1f-6049-4f14-aa54-33bafae1ed76" - -[[Tar]] -deps = ["ArgTools", "SHA"] -uuid = "a4e569a6-e804-4fa4-b0f3-eef7a1d5b13e" - -[[Test]] -deps = ["InteractiveUtils", "Logging", "Random", "Serialization"] -uuid = "8dfed614-e22c-5e08-85e1-65c5234f0b40" - -[[UUIDs]] -deps = ["Random", "SHA"] -uuid = "cf7118a7-6976-5b1a-9a39-7adc72f591a4" - -[[Unicode]] -uuid = "4ec0a83e-493e-50e2-b9ac-8f72acf5a8f5" - -[[Zlib_jll]] -deps = ["Libdl"] -uuid = "83775a58-1f1d-513f-b197-d71354ab007a" - -[[nghttp2_jll]] -deps = ["Artifacts", "Libdl"] -uuid = "8e850ede-7688-5339-a07c-302acd2aaf8d" - -[[p7zip_jll]] -deps = ["Artifacts", "Libdl"] -uuid = "3f19e933-33d8-53b3-aaab-bd5110c3b7a0" diff --git a/docs/Project.toml b/docs/Project.toml index c8c2f26..d6088c0 100644 --- a/docs/Project.toml +++ b/docs/Project.toml @@ -1,3 +1,6 @@ [deps] Documenter = "e30172f5-a6a5-5a46-863b-614d45cd2de4" LMDB = "11f193de-5e89-5f17-923a-7d207d56daf9" + +[compat] +Documenter = "1" diff --git a/docs/api/index.md b/docs/api/index.md deleted file mode 100644 index c528b7d..0000000 --- a/docs/api/index.md +++ /dev/null @@ -1,17 +0,0 @@ -# LMDB - -## Exported - -```@autodocs -Modules = [LMDB] -Order = [:function, :type] -Private = false -``` - -## Not Exported - -```@autodocs -Modules = [LMDB] -Order = [:function, :type] -Public = false -``` \ No newline at end of file diff --git a/docs/make.jl b/docs/make.jl index df19ce9..44ebce3 100644 --- a/docs/make.jl +++ b/docs/make.jl @@ -1,21 +1,44 @@ using Documenter, LMDB -makedocs( - modules = [LMDB], - clean = false, - format = Documenter.HTML(), - sitename = "LMDB.jl", - authors = "Art Wild, Fabian Gans", - pages = [ - "Home" => "index.md", - "Manual" => "manual.md", - "API" => [ - "Index"=>"api/index.md", - ] - ] -) +function main() + ci = get(ENV, "CI", "") == "true" -deploydocs( - repo = "github.com/wildart/LMDB.jl.git", -) + makedocs( + sitename = "LMDB.jl", + authors = "Art Wild, Fabian Gans, Tim Besard", + format = Documenter.HTML(prettyurls = ci, + edit_link = "master"), + modules = [LMDB], + checkdocs = :exports, + pages = [ + "Home" => "index.md", + "Usage" => [ + "man/essentials.md", + "man/dict.md", + "man/environments.md", + "man/transactions.md", + "man/databases.md", + "man/cursors.md", + "man/dupsort.md", + "man/lowlevel.md", + ], + "API reference" => [ + "lib/dict.md", + "lib/environments.md", + "lib/transactions.md", + "lib/databases.md", + "lib/cursors.md", + "lib/errors.md", + "lib/lowlevel.md", + ], + ], + ) + if ci + deploydocs( + repo = "github.com/wildart/LMDB.jl.git", + ) + end +end + +isinteractive() || main() diff --git a/docs/src/api/index.md b/docs/src/api/index.md deleted file mode 100644 index 31aac18..0000000 --- a/docs/src/api/index.md +++ /dev/null @@ -1,17 +0,0 @@ -# LMDB - -## Exported - -```@autodocs -Modules = [LMDB] -Order = [:function, :type] -Private = false -``` - -### Not Exported - -```@autodocs -Modules = [LMDB] -Order = [:function, :type] -Public = false -``` \ No newline at end of file diff --git a/docs/src/index.md b/docs/src/index.md index aab837e..92e4b60 100644 --- a/docs/src/index.md +++ b/docs/src/index.md @@ -1,10 +1,49 @@ # LMDB.jl -[*LMDB.jl*](https://github.com/wildart/LMDB.jl) is a [Julia](http://www.julialang.org) package for interfacing with LMDB database. - -[Lightning Memory-Mapped Database (LMDB)](http://symas.com/mdb/) is an ultra-fast, -ultra-compact key-value embedded data store developed by Symas for the OpenLDAP Project. -It uses memory-mapped files, so it has the read performance of a pure in-memory -database while still offering the persistence of standard disk-based databases, -and is only limited to the size of the virtual address space. -This module provides a Julia interface to LMDB. +*A Julia wrapper for [LMDB](http://www.lmdb.tech/doc/), the Lightning +Memory-Mapped Database.* + +LMDB is an embedded, memory-mapped, ACID key-value store developed by +Symas for OpenLDAP. It is small, fast, and persists to disk while reading +at near in-memory speeds — limited only by the size of the virtual address +space. + +```julia +using Pkg; Pkg.add("LMDB") +``` + +## Three layers of abstraction + +LMDB.jl exposes three tiers, each with a clear consumer: + +| Tier | Surface | When to use | +|------|---------|-------------| +| **3** | `LMDBDict <: AbstractDict{K,V}` | "I want a persistent `Dict`." | +| **2** | `Environment`, `Transaction`, `DBI`, `Cursor` | Julian wrappers with explicit transactions and cursors. The recommended surface for most code. | +| **1** | `LMDB.mdb_*`, `LMDB.MDB_*` | Raw `ccall` bindings + status-code constants. For power users integrating with custom data layouts or shaving allocations on hot paths. | + +A light **tier 1.5** sits between tier 1 and tier 2: `MDBValue`, `MDBArg`, +and the [`MDBValueIO`](@ref LMDB.MDBValueIO) extension point — an `IO` +view over `MDB_val` that lets custom value representations plug into +all the typed reads via `Base.read(io, T)`. + +The Usage section is organised in increasing order of complexity: start +with [Essentials](@ref) for a working example, then [Dictionary +interface](@ref) for the `LMDBDict` surface, and progress through +[Environments](@ref), [Transactions](@ref), [Databases](@ref), +[Cursors](@ref), and [Duplicate-sort databases](@ref) as you need them. +[Low-level bindings](@ref) covers the `ccall` surface for callers who need +to skip the wrappers. + +The API reference mirrors the same structure but lists every exported and +public docstring. + +## A 5-line example + +```julia +using LMDB +d = LMDBDict{String, Vector{Float32}}("/tmp/mydb") +d["alpha"] = Float32[1, 2, 3] +@show d["alpha"] +close(d) +``` diff --git a/docs/src/lib/cursors.md b/docs/src/lib/cursors.md new file mode 100644 index 0000000..575a2d9 --- /dev/null +++ b/docs/src/lib/cursors.md @@ -0,0 +1,66 @@ +# [Cursors](@id API-Cur) + +```@meta +CurrentModule = LMDB +``` + +A `Cursor` is a positioned iterator over the entries in a `DBI`. Cursors +are bound to a transaction; closing the txn invalidates the cursor. + +## Construction + +```@docs +Cursor +Base.open(::Transaction, ::DBI) +Base.close(::Cursor) +Base.isopen(::Cursor) +renew(::Transaction, ::Cursor) +transaction +database +``` + +## Navigation + +```@docs +seek! +seek_last! +seek_range! +next! +prev! +``` + +## Current-position accessors + +```@docs +key +value +item +``` + +## [Bulk walk](@id API-Cur-walk) + +```@docs +walk +``` + +## [DUPSORT navigation](@id API-Cur-DUPSORT) + +These are only meaningful when the database was opened with +`MDB_DUPSORT`. See [Duplicate-sort databases](@ref) for the data model. + +```@docs +seek_first_dup! +seek_last_dup! +next_dup! +prev_dup! +next_nodup! +prev_nodup! +``` + +## Mutation + +```@docs +Base.put!(::Cursor, ::Any, ::Any) +Base.delete!(::Cursor) +Base.count(::Cursor) +``` diff --git a/docs/src/lib/databases.md b/docs/src/lib/databases.md new file mode 100644 index 0000000..a5d9832 --- /dev/null +++ b/docs/src/lib/databases.md @@ -0,0 +1,56 @@ +# [Databases](@id API-DBI) + +```@meta +CurrentModule = LMDB +``` + +A `DBI` (database identifier) is a handle to one B-tree inside an +environment. By default an env has a single anonymous database (the +"main DB"); pass `maxdbs > 0` to `Environment` and a name to `open` to +work with multiple named sub-databases. + +## Construction + +```@docs +DBI +Base.open(::Transaction, ::String) +Base.close(::Environment, ::DBI) +Base.isopen(::DBI) +flags +drop +Base.stat(::Transaction, ::DBI) +``` + +## Reads + +```@docs +Base.get(::Transaction, ::DBI, ::Any, ::Type{T}) where T +Base.get(::Transaction, ::DBI, ::Any, ::Type{T}, ::Any) where T +tryget +``` + +`get(txn, dbi, key, T, default)` falls back to `default` if `key` is +missing — same shape as `Base.get(dict, key, default)`. + +## Writes + +```@docs +Base.put!(::Transaction, ::DBI, ::Any, ::Any) +put_reserved! +Base.delete!(::Transaction, ::DBI, ::Any) +Base.replace!(::Transaction, ::DBI, ::Any, ::Any) +Base.pop!(::Transaction, ::DBI, ::Any, ::Type) +``` + +## Write flags + +The `flags` keyword on `put!` accepts a bitwise-or of: + +| flag | meaning | +|------|---------| +| `MDB_NOOVERWRITE` | fail with `MDB_KEYEXIST` if `key` is already present | +| `MDB_NODUPDATA` | (DUPSORT) fail if `(key, val)` pair already present | +| `MDB_APPEND` | append at the end; only valid if the new key sorts after every existing key | +| `MDB_RESERVE` | preferred via [`put_reserved!`](@ref) | + +See also the [DUPSORT-only ops](@ref API-Cur-DUPSORT) on the cursor surface. diff --git a/docs/src/lib/dict.md b/docs/src/lib/dict.md new file mode 100644 index 0000000..edd9777 --- /dev/null +++ b/docs/src/lib/dict.md @@ -0,0 +1,48 @@ +# [Dictionary interface](@id API-Dict) + +The tier-3 surface: a single `AbstractDict{K,V}` over an LMDB environment. + +```@meta +CurrentModule = LMDB +``` + +## `LMDBDict` + +```@docs +LMDBDict +``` + +`LMDBDict` is `<: AbstractDict{K,V}`, so it transparently picks up +`Base`'s generic methods on top of the lookup/mutation primitives: + +- **Reads.** `getindex`, `haskey`, `get`, `get!`, `length`, `isempty`, + `iterate`, `keys`, `values`, `pairs`. All defined on `Base`-side + signatures and dispatched into LMDB; `getindex` and `pop!` throw + `KeyError` on miss to match `Base.Dict`. +- **Writes.** `setindex!`, `delete!`, `pop!`, `empty!`. `delete!` + silently no-ops on a missing key (matching `Base.delete!`'s "if any" + contract). +- **Generic.** Everything `AbstractDict` derives — `merge!`, `merge`, + `mergewith!`, `filter!`, `filter`, `==`, `isequal`, `hash`, + `in(::Pair, d)`, `copy(d)` — applies for free. + +`LMDBDict` iterates in lexicographic key order, which is stronger than +`Base.Dict`'s no-order promise. + +## Lifecycle + +`close(::LMDBDict)` closes the underlying env (and the default DBI). +Idempotent — also called from the finalizer. + +## Prefix-scan helpers + +LMDB-namespaced extensions for hierarchical-key schemes that don't fit +the polymorphic `AbstractDict` contract: + +```@docs +LMDB.scan +LMDB.scan_keys +LMDB.scan_values +LMDB.list_dirs +LMDB.valuesize +``` diff --git a/docs/src/lib/environments.md b/docs/src/lib/environments.md new file mode 100644 index 0000000..a7addb8 --- /dev/null +++ b/docs/src/lib/environments.md @@ -0,0 +1,60 @@ +# [Environments](@id API-Env) + +```@meta +CurrentModule = LMDB +``` + +An `Environment` wraps an LMDB env handle (`Ptr{MDB_env}`). It is the +top of the handle hierarchy — every transaction, database, and cursor +ultimately lives inside one env. + +## Construction + +```@docs +Environment +Environment(::AbstractString) +create +environment +``` + +## Lifecycle + +```@docs +Base.open(::Environment, ::String) +Base.close(::Environment) +Base.isopen(::Environment) +sync +path +``` + +## Configuration + +`Environment` exposes its tunables through `getindex` / `setindex!` with +symbol keys (`:Flags`, `:Readers`, `:MapSize`, `:DBs`, `:KeySize`): + +```@docs +Base.setindex!(::Environment, ::Integer, ::Symbol) +Base.getindex(::Environment, ::Symbol) +set! +unset! +``` + +## Inspection + +```@docs +info +Base.stat(::Environment) +``` + +## Backup + +```@docs +Base.copy(::Environment, ::AbstractString) +``` + +## Reader management + +```@docs +reader_check +reader_list +``` diff --git a/docs/src/lib/errors.md b/docs/src/lib/errors.md new file mode 100644 index 0000000..5164e3d --- /dev/null +++ b/docs/src/lib/errors.md @@ -0,0 +1,60 @@ +# [Errors](@id API-Errors) + +```@meta +CurrentModule = LMDB +``` + +Every LMDB-internal error surfaces as an `LMDBError` whose `code` field is +the raw status `Cint` returned by the underlying binding. Status-code +matchers cover the three most common branches; less common codes can be +matched against `LMDB.MDB_*` constants directly. + +## Exception type + +```@docs +LMDBError +``` + +## Status-code matchers + +```@docs +is_notfound +is_keyexist +is_map_full +``` + +## Error helpers + +```@docs +errormsg +``` + +## Status constants + +The full set of LMDB status codes is exposed as `LMDB.MDB_*`. The +commonly-needed ones are exported: + +| constant | meaning | +|----------|---------| +| `MDB_NOTFOUND` | key not present | +| `MDB_KEYEXIST` | key (or duplicate) already present, with `MDB_NOOVERWRITE`/`MDB_NODUPDATA` | +| `MDB_MAP_FULL` | environment's `MapSize` exhausted | + +The rest (`MDB_PAGE_NOTFOUND`, `MDB_CORRUPTED`, `MDB_PANIC`, +`MDB_VERSION_MISMATCH`, `MDB_INVALID`, `MDB_DBS_FULL`, `MDB_READERS_FULL`, +`MDB_TLS_FULL`, `MDB_TXN_FULL`, `MDB_CURSOR_FULL`, `MDB_PAGE_FULL`, +`MDB_MAP_RESIZED`, `MDB_INCOMPATIBLE`, `MDB_BAD_RSLOT`, `MDB_BAD_TXN`, +`MDB_BAD_VALSIZE`, `MDB_BAD_DBI`) live under the `LMDB.` prefix. + +## Where errors come from at each tier + +- **Tier 1.** Bindings wrapped by `@checked` auto-throw `LMDBError`; + the `unchecked_*` companion returns the raw `Cint` so the caller can + branch on `MDB_NOTFOUND`/`MDB_KEYEXIST`/etc. +- **Tier 2.** Handle methods that wrap status-returning bindings let + `LMDBError` propagate. `tryget` and `get(..., default)` swallow + `MDB_NOTFOUND` and return `nothing`/`default`. `delete!(txn, dbi, key)` + likewise swallows `MDB_NOTFOUND` and returns `false`. +- **Tier 3.** Missing keys produce `KeyError` (matching `Base.Dict`). + `pop!(d)` on an empty dict throws `ArgumentError`. Other LMDB errors + propagate as `LMDBError`. diff --git a/docs/src/lib/lowlevel.md b/docs/src/lib/lowlevel.md new file mode 100644 index 0000000..a56b484 --- /dev/null +++ b/docs/src/lib/lowlevel.md @@ -0,0 +1,162 @@ +# [Low-level bindings](@id API-LowLevel) + +```@meta +CurrentModule = LMDB +``` + +The tier-1 surface is a flat namespace of `ccall` bindings (`LMDB.mdb_*`), +opaque handle types (`LMDB.MDB_env`, `LMDB.MDB_txn`, `LMDB.MDB_cursor`), +plain structs (`LMDB.MDB_val`, `LMDB.MDB_stat`, `LMDB.MDB_envinfo`), the +cursor-op `@cenum` (`LMDB.MDB_cursor_op`), and `LMDB.MDB_*` flag/status +constants. + +Everything is public-but-unexported: refer to it as `LMDB.mdb_env_create`, +`LMDB.MDB_NOTLS`, `LMDB.MDB_val`. The bindings in this section auto-throw +on a non-zero status; for callers that need to inspect the raw status +code, an `unchecked_*` companion is paired with each. + +## The auto-throw convention + +Every status-returning binding in `liblmdb.jl` is paired with an +`unchecked_*` companion at definition time. Use the bare name when any +error should propagate (the common case); use `unchecked_*` when you +need to inspect the raw `Cint` yourself — e.g. distinguishing +`MDB_NOTFOUND` from a real error: + +```julia +val_ref = Ref(LMDB.MDB_val(zero(Csize_t), C_NULL)) +ret = LMDB.unchecked_mdb_get(txn, dbi, key, val_ref) +ret == LMDB.MDB_NOTFOUND && return nothing +ret == 0 || throw(LMDB.LMDBError(ret)) +``` + +Bindings that return non-status data (`mdb_strerror`, `mdb_version`, +`mdb_txn_id`, `mdb_cmp`, `mdb_dcmp`, `mdb_env_get_maxkeysize`, +`mdb_env_get_userctx`, `mdb_cursor_txn`, `mdb_cursor_dbi`) and +`Cvoid`-returning ones (`mdb_env_close`, `mdb_dbi_close`, +`mdb_txn_abort`, `mdb_txn_reset`, `mdb_cursor_close`) are left bare — +there is nothing to check. + +## Customisation point: `MDBValueIO` + +`tryget` / `get` / `key` / `value` / `item` / typed `walk` / `pop!` / +`replace!` all funnel through `read(::MDBValueIO, T)` to decode an +`MDB_val` into a Julia value. Define a `Base.read` method on +`MDBValueIO` to plug in a custom representation — see [Cursors](@ref) +for a worked example. + +```@docs +MDBValueIO +``` + +## Helpers + +```@docs +isflagset +version +``` + +## Raw bindings + +The bindings are listed below by topic. Every name in this section is +reachable as `LMDB.`; status-returning ones additionally expose +`LMDB.unchecked_`. + +### Types + +```julia +LMDB.MDB_env # opaque +LMDB.MDB_txn # opaque +LMDB.MDB_cursor # opaque +LMDB.MDB_dbi # = Cuint +LMDB.MDB_val # struct { mv_size::Csize_t; mv_data::Ptr{Cvoid} } +LMDB.MDB_stat # struct (page sizes, depth, leaf/branch/overflow page counts, entries) +LMDB.MDB_envinfo # struct (mapaddr, mapsize, last_pgno, last_txnid, maxreaders, numreaders) +LMDB.MDB_cursor_op # @cenum: MDB_FIRST … MDB_PREV_MULTIPLE (19 variants) +``` + +### Environment + +```julia +mdb_env_create +mdb_env_open +mdb_env_close +mdb_env_copy mdb_env_copy2 +mdb_env_copyfd mdb_env_copyfd2 +mdb_env_stat mdb_env_info +mdb_env_sync +mdb_env_set_flags mdb_env_get_flags +mdb_env_get_path mdb_env_get_fd +mdb_env_set_mapsize +mdb_env_set_maxreaders mdb_env_get_maxreaders +mdb_env_set_maxdbs +mdb_env_get_maxkeysize +mdb_env_set_userctx mdb_env_get_userctx +mdb_env_set_assert +``` + +### Transaction + +```julia +mdb_txn_begin +mdb_txn_env +mdb_txn_id +mdb_txn_commit +mdb_txn_abort +mdb_txn_reset +mdb_txn_renew +``` + +### Database (DBI) + +```julia +mdb_dbi_open +mdb_dbi_close +mdb_dbi_flags +mdb_drop +mdb_stat +mdb_set_compare mdb_set_dupsort +mdb_set_relfunc mdb_set_relctx +``` + +### Data access + +```julia +mdb_get +mdb_put +mdb_del +``` + +### Cursor + +```julia +mdb_cursor_open +mdb_cursor_close +mdb_cursor_renew +mdb_cursor_txn +mdb_cursor_dbi +mdb_cursor_get +mdb_cursor_put +mdb_cursor_del +mdb_cursor_count +``` + +### Comparators / readers / version + +```julia +mdb_cmp mdb_dcmp +mdb_reader_list +mdb_reader_check +mdb_version +mdb_strerror +``` + +## Constants + +| group | constants | +|------|-----------| +| Env flags | `MDB_FIXEDMAP`, `MDB_NOSUBDIR`, `MDB_NOSYNC`, `MDB_RDONLY`, `MDB_NOMETASYNC`, `MDB_WRITEMAP`, `MDB_MAPASYNC`, `MDB_NOTLS`, `MDB_NOLOCK`, `MDB_NORDAHEAD`, `MDB_NOMEMINIT` | +| DB flags | `MDB_REVERSEKEY`, `MDB_DUPSORT`, `MDB_INTEGERKEY`, `MDB_DUPFIXED`, `MDB_INTEGERDUP`, `MDB_REVERSEDUP`, `MDB_CREATE` | +| Write flags | `MDB_NOOVERWRITE`, `MDB_NODUPDATA`, `MDB_CURRENT`, `MDB_RESERVE`, `MDB_APPEND`, `MDB_APPENDDUP`, `MDB_MULTIPLE` | +| Copy flag | `MDB_CP_COMPACT` | +| Status codes | `MDB_SUCCESS=0`, `MDB_KEYEXIST`, `MDB_NOTFOUND`, `MDB_PAGE_NOTFOUND`, `MDB_CORRUPTED`, `MDB_PANIC`, `MDB_VERSION_MISMATCH`, `MDB_INVALID`, `MDB_MAP_FULL`, `MDB_DBS_FULL`, `MDB_READERS_FULL`, `MDB_TLS_FULL`, `MDB_TXN_FULL`, `MDB_CURSOR_FULL`, `MDB_PAGE_FULL`, `MDB_MAP_RESIZED`, `MDB_INCOMPATIBLE`, `MDB_BAD_RSLOT`, `MDB_BAD_TXN`, `MDB_BAD_VALSIZE`, `MDB_BAD_DBI` | diff --git a/docs/src/lib/transactions.md b/docs/src/lib/transactions.md new file mode 100644 index 0000000..ef952b3 --- /dev/null +++ b/docs/src/lib/transactions.md @@ -0,0 +1,42 @@ +# [Transactions](@id API-Txn) + +```@meta +CurrentModule = LMDB +``` + +Every database operation runs inside a transaction. Transactions may be +read-only (`MDB_RDONLY`) or read-write; an environment supports many +concurrent readers but only one writer at a time. + +## Construction + +```@docs +Transaction +start +``` + +## Lifecycle + +```@docs +commit +abort +Base.isopen(::Transaction) +env +``` + +## Read-only reuse + +For read-only transactions, the txn handle can be parked across requests +to skip the begin/abort cost: + +```@docs +Base.reset(::Transaction) +renew(::Transaction) +``` + +## Sub-transactions + +Pass `parent = txn` to [`start`](@ref) to nest a child write transaction +inside an open write transaction. The child sees the parent's uncommitted +state; on `commit` the child's changes are folded into the parent, on +`abort` they are discarded. diff --git a/docs/src/man/cursors.md b/docs/src/man/cursors.md new file mode 100644 index 0000000..6d148ef --- /dev/null +++ b/docs/src/man/cursors.md @@ -0,0 +1,186 @@ +# Cursors + +```@meta +CurrentModule = LMDB +``` + +A `Cursor` is a positioned iterator over a `DBI`. Use it for ordered +scans, range queries, and any time you want to amortise the per-lookup +overhead of `mdb_get` across many keys. + +## Opening a cursor + +```julia +start(env; flags = MDB_RDONLY) do txn + open(txn) do dbi + open(txn, dbi) do cur + # use cur + end + end +end +``` + +A cursor is bound to its transaction; closing the txn invalidates the +cursor. The cursor's finalizer is idempotent, so a still-open cursor +is reclaimed when GC visits it. + +## Navigation + +Each navigation function repositions the cursor and returns the new +key, or `nothing` if the move would step past the end: + +```julia +seek!(cur) # MDB_FIRST — first entry +seek_last!(cur) # MDB_LAST — last entry +seek!(cur, key) # MDB_SET_KEY — exact key match +seek_range!(cur, key) # MDB_SET_RANGE — smallest key ≥ `key` +next!(cur) # MDB_NEXT +prev!(cur) # MDB_PREV +``` + +Each accepts an optional key-type parameter `T` (default `Vector{UInt8}`): + +```julia +seek!(cur, String) # decode the resulting key as String +seek_range!(cur, "users/", String) +``` + +## Reading at the current position + +```julia +key(cur, K) # current key, decoded as K +value(cur, V) # current value, decoded as V +item(cur, K, V) # Pair{K, V} +``` + +The defaults are `K = V = Vector{UInt8}`. + +```julia +seek_range!(cur, "users/", String) === nothing && return +@show key(cur, String), value(cur, String) +``` + +## Range scans + +A typical pattern for "all keys with a given prefix": + +```julia +prefix = "users/" +start(env; flags = MDB_RDONLY) do txn + open(txn) do dbi + open(txn, dbi) do cur + k = seek_range!(cur, prefix, String) + while k !== nothing && startswith(k, prefix) + v = value(cur, String) + handle(k, v) + k = next!(cur, String) + end + end + end +end +``` + +For the same pattern *one level up* (already wrapped, returns a +`Vector{Pair}`), use [`LMDB.scan(d; prefix)`](@ref LMDB.scan) on an +`LMDBDict`. + +## [Bulk walk — zero-copy iteration](@id man-cur-walk) + +`walk` runs a callback over every entry the cursor visits. It exists in +two shapes: + +```julia +# Untyped — receives Ref{MDB_val} pairs (zero-copy, mmap pointers) +walk(cur) do k_ref, v_ref + kv = k_ref[]; vv = v_ref[] + # kv.mv_data / vv.mv_data are mmap pointers, valid in this scope + do_something(kv.mv_size, vv.mv_size) +end + +# Typed — runs each ref through `read(MDBValueIO, K)` / `read(MDBValueIO, V)` +walk(cur, String, Vector{UInt8}) do k::String, v::Vector{UInt8} + println(k, " => ", length(v), " bytes") +end +``` + +Pass `from = key` to start at the smallest entry `≥ key` (i.e. +`MDB_SET_RANGE`); the default is to start at `MDB_FIRST`. + +The callback can return `false` to stop iteration; any other return +(including `nothing`) continues. + +The untyped form is the right tool when you want to inspect raw byte +sizes, copy slices, or feed a custom decoder — the data pointers are +into LMDB's mmap and are valid only inside the callback (and only for +the surrounding txn). The typed form is the iteration analogue of +`tryget(..., T)` and works for any `T` for which `Base.read(io::IO, +::Type{T})` (or `Base.read(io::LMDB.MDBValueIO, ::Type{T})`) is +defined (see [Custom value decoding](@ref)). + +## Cursor mutation + +Inside a write transaction, a cursor can put or delete at its current +position: + +```julia +put!(cur, key, val) +put!(cur, key, val; flags = MDB_NOOVERWRITE) +delete!(cur) +delete!(cur; flags = MDB_NODUPDATA) +``` + +`count(cur)` returns the number of duplicate values for the current +key (1 in non-DUPSORT databases). + +## Custom value decoding + +`tryget` / `get` / `key` / `value` / `item` / typed `walk` all funnel +through `Base.read(io::IO, ::Type{T})` against an +[`MDBValueIO`](@ref LMDB.MDBValueIO). The defaults cover Base's +primitive numeric types (`Int8`/…/`Float64`, `Bool`, `Char`, `Ptr`), +`String`, and (added by this package) `Vector{E}` for any bitstype `E`. + +For everything else — including `isbitstype` structs and framed +values — define a single `Base.read` method on the abstract `IO`: + +```julia +struct PrefixedBlob end + +function Base.read(io::IO, ::Type{PrefixedBlob}) + bytesavailable(io) < 8 && return UInt8[] + skip(io, 8) + return read(io, Vector{UInt8}) +end + +# now usable everywhere a value-type parameter is accepted: +LMDB.tryget(txn, dbi, key, PrefixedBlob) +walk(cur, String, PrefixedBlob) do k, blob + handle(k, blob) +end +``` + +`MDBValueIO <: IO` so all the usual `Base` IO primitives — `position`, +`seek`, `skip`, `read(io, n::Integer)`, `read(io, T)`, `read!(io, A)`, +`bytesavailable`, `eof` — work as expected. This makes structured +framed-value decoders read like any other Julia binary parser, and is +the analogue of heed's `BytesDecode<'txn>` trait — but expressed +through Julia's existing IO extension point rather than a bespoke +trait, so the same decoder works against any byte source. + +## Reset and renew + +For long-running readers, opening one cursor per snapshot can be +expensive. Park the txn with [`reset`](@ref Base.reset(::LMDB.Transaction)) +and refresh both the txn and the cursor with `renew(txn, cur)`: + +```julia +txn = start(env; flags = MDB_RDONLY) +cur = open(txn, dbi) +while running + ... # use cur + reset(txn) + renew(txn) + renew(txn, cur) +end +abort(txn) +``` diff --git a/docs/src/man/databases.md b/docs/src/man/databases.md new file mode 100644 index 0000000..9e08697 --- /dev/null +++ b/docs/src/man/databases.md @@ -0,0 +1,141 @@ +# Databases + +```@meta +CurrentModule = LMDB +``` + +A `DBI` is a handle to one B-tree inside an environment. By default an +env has a single anonymous database (the "main DB"); pass `maxdbs > 0` +to `Environment` to support multiple named sub-databases. + +## Opening a DBI + +```julia +dbi = open(txn) # main (unnamed) DB +dbi = open(txn, "users") # named sub-DB; needs maxdbs >= 1 +dbi = open(txn, "edges"; flags = MDB_CREATE | MDB_DUPSORT) +``` + +The do-block form closes the DBI on the way out: + +```julia +open(txn, "users") do dbi + put!(txn, dbi, "1", "Ada") +end +``` + +In practice you'll rarely *want* to close a DBI handle explicitly: the +env owns it, and `mdb_dbi_close` is documented as rarely useful. The +env's finalizer cascades through any open DBI handles. + +## DBI flags + +`flags` accepts a bitwise-or of: + +| flag | meaning | +|------|---------| +| `MDB_CREATE` | create the named DB if it doesn't exist | +| `MDB_REVERSEKEY` | compare keys back-to-front (suffix-sorted) | +| `MDB_INTEGERKEY` | keys are native-endian integers, sorted numerically | +| `MDB_DUPSORT` | allow multiple values per key, sorted; see [Duplicate-sort databases](@ref) | +| `MDB_DUPFIXED` | (DUPSORT) all duplicates have the same byte size | +| `MDB_INTEGERDUP` | (DUPSORT) duplicates are native-endian integers | +| `MDB_REVERSEDUP` | (DUPSORT) compare duplicates back-to-front | + +## Reads + +Every read takes a value-type parameter `T`. The default tier-2 forms +are: + +```julia +get(txn, dbi, key, T) # throws LMDBError(MDB_NOTFOUND) on miss +tryget(txn, dbi, key, T) # nothing on miss +get(txn, dbi, key, T, default) # default on miss +``` + +`T` is anything `read(::LMDB.MDBValueIO, ::Type{T})` knows how to +decode — `String`, `Vector{E}` for any bitstype `E`, or any bitstype +scalar: + +```julia +tryget(txn, dbi, "name", String) # → Union{String, Nothing} +tryget(txn, dbi, key, Vector{Float32}) # → Union{Vector{Float32}, Nothing} +tryget(txn, dbi, key, UInt64) # → Union{UInt64, Nothing} +``` + +`tryget` is the workhorse: it inspects the raw status code and +swallows `MDB_NOTFOUND` cheaply, without throwing. + +## Writes + +```julia +put!(txn, dbi, key, val) +put!(txn, dbi, key, val; flags = MDB_NOOVERWRITE) +delete!(txn, dbi, key) # → Bool: true if removed +delete!(txn, dbi, key, val) # DUPSORT: delete one specific dup +replace!(txn, dbi, key, val) # atomic put-and-return-old +pop!(txn, dbi, key, T) # atomic get-and-delete +``` + +Useful write flags: + +| flag | meaning | +|------|---------| +| `MDB_NOOVERWRITE` | fail with `MDB_KEYEXIST` if `key` is already present | +| `MDB_NODUPDATA` | (DUPSORT) fail if the `(key, val)` pair already exists | +| `MDB_APPEND` | append; only valid if the new key sorts after every existing key — *much* faster for sorted bulk loads | + +```julia +# Bulk import in sorted order: +start(env) do txn + open(txn) do dbi + for (k, v) in sorted_pairs + put!(txn, dbi, k, v; flags = MDB_APPEND) + end + end +end +``` + +`replace!` and `pop!` perform the read-modify pair inside the same +transaction — no time-of-check / time-of-use gap. + +## `put_reserved!` — write directly into the mmap + +When the value is large or assembled from multiple sources, you can +skip the intermediate `Vector{UInt8}` round-trip and write straight +into the LMDB-allocated page: + +```julia +put_reserved!(txn, dbi, key, sizeof(header) + length(payload)) do buf + unsafe_store!(Ptr{Header}(pointer(buf)), header) + copyto!(buf, sizeof(header) + 1, payload, 1, length(payload)) +end +``` + +`buf` is an `unsafe_wrap` over the LMDB write buffer; it is **only +valid inside the callback** (and only inside the surrounding write +txn). Don't escape it. + +`put_reserved!` is the equivalent of heed's `Database::put_reserved`. +It is incompatible with DUPSORT. + +## Stats + +```julia +s = stat(txn, dbi) +@show s.entries, s.depth, s.leaf_pages, s.psize + +# rough on-disk byte count: +live = (s.branch_pages + s.leaf_pages + s.overflow_pages) * s.psize +``` + +## Dropping a database + +```julia +drop(txn, dbi) # empty the DB (handle still valid) +drop(txn, dbi; delete = true) # delete the DB and close the handle +``` + +For named sub-DBs, `delete = true` removes the entry from the env's +main DB. For the main DB itself, `delete = true` is treated as +`delete = false` (LMDB cannot delete its own root). diff --git a/docs/src/man/dict.md b/docs/src/man/dict.md new file mode 100644 index 0000000..a331fb0 --- /dev/null +++ b/docs/src/man/dict.md @@ -0,0 +1,145 @@ +# Dictionary interface + +```@meta +CurrentModule = LMDB +``` + +`LMDBDict{K,V}` is a persistent `AbstractDict{K,V}` backed by a single +LMDB environment + the default DBI. It is the simplest way to use LMDB +from Julia: open it, treat it like a `Dict`, close it. + +## Construction + +```julia +d = LMDBDict{String, Vector{Float32}}("/tmp/mydb") +``` + +`LMDBDict(path)` without explicit type parameters defaults to +`LMDBDict{String, Vector{UInt8}}`. The path is the directory LMDB will +manage (it must exist; LMDB will create the data files inside it). + +Constructor keyword arguments: + +| kwarg | default | meaning | +|-------|---------|---------| +| `readonly` | `false` | open with `MDB_RDONLY` | +| `rdahead` | `false` | unset `MDB_NORDAHEAD` (LMDB's default is to read-ahead; LMDB.jl turns it off because cold-page workloads pay for it) | +| `mapsize` | LMDB default (10 MiB) | virtual map size in bytes; the on-disk file may be much smaller | +| `readers` | LMDB default | max concurrent reader slots | +| `dbs` | LMDB default | max named sub-databases | + +`MDB_NOTLS` is always set, so a single thread can hold multiple read +transactions. This is required for any interleaved read pattern (e.g. +calling `length(d)` mid-iteration) and for read txns shared between +tasks. + +## Storing and retrieving + +Anything that round-trips through the package's `MDB_val` glue and +[`MDBValueIO`](@ref LMDB.MDBValueIO) works as a value type — `String`, +`Vector{T}` for any bitstype `T`, and any bitstype scalar (`Int`, +`Float32`, `(Int, UInt32)` `Tuple`, …). + +```julia +d = LMDBDict{String, Float64}("/tmp/scores") +d["alpha"] = 1.5 +d["beta"] = 2.0 + +@show d["alpha"] # 1.5 +@show get(d, "missing", -1.0) # -1.0 +@show haskey(d, "alpha") # true +@show length(d) # 2 +``` + +Missing keys throw `KeyError`, exactly like `Base.Dict`: + +```julia-repl +julia> d["nonexistent"] +ERROR: KeyError: key "nonexistent" not found +``` + +## Iteration + +```julia +for (k, v) in d + println(k, " => ", v) +end +``` + +Iteration is in **lexicographic key order** — strictly stronger than +`Base.Dict`'s no-order promise. Each `for` loop opens a fresh read +transaction; the txn is committed on normal exit and aborted (via the +`Transaction` finalizer) on early break or throw. + +`keys(d)`, `values(d)`, `pairs(d)` are lazy. `collect(d)` materialises +a `Vector{Pair{K,V}}`. + +## Mutations + +```julia +d["x"] = 42 +delete!(d, "x") # silent no-op if missing +pop!(d, "x") # throws KeyError if missing +pop!(d, "x", default) # returns default if missing +empty!(d) # drops every entry +``` + +`delete!` matches `Base.delete!`'s "if any" contract: it returns `d` and +silently no-ops when the key isn't present. + +Generic `AbstractDict` operations all kick in for free: + +```julia +merge!(d, Dict("a" => 1, "b" => 2)) +filter!(((k, v),) -> v > 0, d) +``` + +## Prefix-scoped scans + +For hierarchical key schemes (e.g. `"users/123/name"`), LMDB's +lexicographic order makes prefix scans cheap — they're a single +`MDB_SET_RANGE` plus iteration until the prefix stops matching. + +```julia +d = LMDBDict{String, String}("/tmp/tree") +d["users/1/name"] = "Ada" +d["users/2/name"] = "Bob" +d["users/2/email"] = "bob@example.com" +d["other"] = "skip" + +LMDB.scan_keys(d, prefix = "users/") +# 3-element Vector{String}: +# "users/1/name" +# "users/2/email" +# "users/2/name" + +LMDB.scan(d, prefix = "users/2/") +# 2-element Vector{Pair{String,String}}: +# "users/2/email" => "bob@example.com" +# "users/2/name" => "Bob" +``` + +For directory-style listings — leaf keys appear as-is, anything with +the separator after the prefix collapses to its first segment: + +```julia +LMDB.list_dirs(d, prefix = "") # ["other", "users/"] +LMDB.list_dirs(d, prefix = "users/") # ["users/1/", "users/2/"] +``` + +`LMDB.valuesize(d; prefix)` sums byte sizes — useful for quick storage +audits without `stat`. + +## When to drop down + +Reach for the explicit tier-2 surface (next chapters) when: + +- you need **multi-key atomicity**: a single transaction grouping more + than one `put!`/`delete!`, +- you want to **stream** without eagerly building a `Vector{Pair{K,V}}` + (see [Cursors](@ref) and `walk`), +- you need **multiple named databases** in one env (LMDBDict only + exposes the default unnamed DB), +- you're using `MDB_DUPSORT` (multiple values per key — see + [Duplicate-sort databases](@ref)), +- or you want zero-copy reads against the mmap. diff --git a/docs/src/man/dupsort.md b/docs/src/man/dupsort.md new file mode 100644 index 0000000..8f44f3c --- /dev/null +++ b/docs/src/man/dupsort.md @@ -0,0 +1,98 @@ +# Duplicate-sort databases + +```@meta +CurrentModule = LMDB +``` + +By default each key in an LMDB database has a single value. Opening a +DB with `MDB_DUPSORT` instead allows **multiple values per key**, kept +in sorted order. This is LMDB's answer to inverted indexes, +many-to-many edges, time-series buckets, and any "key → set of values" +pattern. + +```julia +env = Environment("/tmp/edges"; mapsize = 1 << 30, maxdbs = 1) +start(env) do txn + dbi = open(txn, "edges"; flags = MDB_CREATE | MDB_DUPSORT) + put!(txn, dbi, "a", "b") + put!(txn, dbi, "a", "c") + put!(txn, dbi, "a", "d") + put!(txn, dbi, "b", "c") +end +``` + +`(a, b)`, `(a, c)`, `(a, d)`, `(b, c)` are all distinct entries. +Putting the same `(key, val)` pair twice silently no-ops (or raises +`MDB_KEYEXIST` if `MDB_NODUPDATA` is set). + +## Why DUPSORT instead of value packing + +A common alternative is to pack a list into a single value +(`key -> [v1, v2, v3]`) and read-modify-write on each update. +DUPSORT wins when: + +- you want `O(log n)` insert/delete of a single value (vs. rewriting + the whole list), +- you want sorted access to values without sorting in-process, +- the per-key cardinality is large enough that value-packing pages + blow past `MDB_MAXKEYSIZE` or LMDB's overflow-page threshold, +- you want range queries within a key's values + (`seek_range!` style). + +It loses if you need **fast aggregate reads of every value at a key** — +that's where `MDB_DUPFIXED` (fixed-size duplicates) shines because +the values are stored contiguously and can be returned in batches. + +## Navigation + +DUPSORT layers an extra dimension on the cursor: the cursor is +positioned at a `(key, value)` pair, and you can navigate either +across keys or across values within the current key. + +```julia +seek!(cur, "a") # position at (a, b) — the first dup +seek_first_dup!(cur) # value of first dup of current key +next_dup!(cur) # next dup of current key → (a, c) +next_dup!(cur) # → (a, d) +next_dup!(cur) # nothing — out of dups for "a" +next_nodup!(cur) # skip to next key, first dup → (b, c) +``` + +| function | LMDB op | step within key | step across keys | +|----------|---------|-----------------|------------------| +| `next!` | `MDB_NEXT` | yes (next dup) | yes (next key when dups exhausted) | +| `prev!` | `MDB_PREV` | yes | yes | +| `next_dup!` | `MDB_NEXT_DUP` | yes | no — `nothing` past last dup | +| `prev_dup!` | `MDB_PREV_DUP` | yes | no | +| `next_nodup!` | `MDB_NEXT_NODUP` | jump out | yes (first dup of next key) | +| `prev_nodup!` | `MDB_PREV_NODUP` | jump out | yes (first dup of previous key) | +| `seek_first_dup!` | `MDB_FIRST_DUP` | first dup of current key | – | +| `seek_last_dup!` | `MDB_LAST_DUP` | last dup of current key | – | + +`count(cur)` returns the number of duplicates at the current key. + +## Deleting a single duplicate + +```julia +delete!(txn, dbi, "a", "c") # → true; only (a, c) is removed +delete!(txn, dbi, "a") # → true; removes ALL dups of "a" +``` + +The two-argument `delete!` removes every value at `key`. The +three-argument form removes one specific `(key, val)` pair, leaving +the rest of the dups intact. + +## Useful flag combinations + +- `MDB_DUPSORT` alone: variable-size duplicates, sorted by full byte + comparison. +- `MDB_DUPSORT | MDB_DUPFIXED`: every duplicate has the same byte + size; LMDB stores them as a packed array per key. Required for the + `MDB_GET_MULTIPLE` / `MDB_NEXT_MULTIPLE` cursor ops (reachable from + the tier-1 surface). +- `MDB_DUPSORT | MDB_DUPFIXED | MDB_INTEGERDUP`: values are + native-endian integers; sorted numerically. +- `MDB_DUPSORT | MDB_REVERSEDUP`: values compared back-to-front. + +`MDB_RESERVE` (and therefore `put_reserved!`) is **not** valid against +a DUPSORT database — LMDB rejects it. diff --git a/docs/src/man/environments.md b/docs/src/man/environments.md new file mode 100644 index 0000000..4211c85 --- /dev/null +++ b/docs/src/man/environments.md @@ -0,0 +1,126 @@ +# Environments + +```@meta +CurrentModule = LMDB +``` + +An `Environment` corresponds to a single LMDB directory on disk and to +the in-process memory map of that directory. Every transaction, +database handle, and cursor lives inside one env. + +## Creating and opening + +The simplest path is the one-call constructor — it `create`s the +handle, applies optional configuration, and `open`s the directory in a +single call: + +```julia +env = Environment("/tmp/mydb"; mapsize = 1 << 30, # 1 GiB virtual map + maxreaders = 510, + maxdbs = 8, + flags = MDB_NOTLS) +``` + +If anything fails between `create` and a successful `open`, the +partially constructed env is closed before rethrowing. + +The split form is also available, mirroring the LMDB C API: + +```julia +env = create() +env[:MapSize] = 1 << 30 +env[:Readers] = 510 +env[:DBs] = 8 +open(env, "/tmp/mydb"; flags = MDB_NOTLS) +``` + +The `[:Flags]`/`[:Readers]`/`[:MapSize]`/`[:DBs]` keys map directly to +`mdb_env_set_flags` / `mdb_env_set_maxreaders` / `mdb_env_set_mapsize` +/ `mdb_env_set_maxdbs`. `set!` / `unset!` flip individual flag bits +after the env is open. + +`getindex` exposes a few read-only views: `env[:Flags]`, +`env[:Readers]`, and `env[:KeySize]` (the maximum key length, fixed at +compile time of the bundled `LMDB_jll`). + +The do-block constructor `environment(f, path; flags, mode)` opens the +env, calls `f(env)`, and closes the env on the way out: + +```julia +environment("/tmp/mydb"; flags = MDB_NOTLS) do env + # use env +end +``` + +## Common environment flags + +`flags` accepts a bitwise-or of: + +| flag | meaning | +|------|---------| +| `MDB_RDONLY` | open the env in read-only mode | +| `MDB_NOSUBDIR` | `path` is a single file, not a directory | +| `MDB_NOSYNC` | don't `fsync` on commit (faster, less durable) | +| `MDB_NOMETASYNC` | `fsync` data but not metadata | +| `MDB_WRITEMAP` | mmap as writable; faster but requires more discipline (no torn writes from other processes) | +| `MDB_NOMEMINIT` | skip zero-init of new pages | +| `MDB_NOTLS` | drop thread-local reader slots — needed for multiple read txns on one thread | +| `MDB_NORDAHEAD` | turn off OS-level read-ahead — better for cold-page workloads | +| `MDB_NOLOCK` | the caller takes responsibility for locking | + +`MDB_RDONLY` can only be set at `open` time — calling `set!(env, +MDB_RDONLY)` on an open env will return `EINVAL`. + +## Sizing the map + +The `mapsize` is a *virtual* limit on the env's address space, not the +on-disk size. A typical pattern is to pick a generous power of two +(say, 1 GiB or 8 GiB) up front; the on-disk file grows incrementally as +data is written. + +If a write txn would exceed `mapsize`, LMDB returns `MDB_MAP_FULL` +(catchable via [`is_map_full(::LMDBError)`](@ref is_map_full)). The +remedy is to close the env, raise `mapsize`, and reopen — no rewrite +of the database is needed. + +## Inspection + +```julia +ei = info(env) +@show ei.mapsize, ei.last_pgno, ei.numreaders + +s = stat(env) +@show s.psize, s.depth, s.entries +``` + +[`info`](@ref) and [`stat`](@ref Base.stat(::LMDB.Environment)) both +return `NamedTuple`s; see their docstrings for the field layout. + +## Backup + +[`copy(env, path)`](@ref Base.copy(::LMDB.Environment, ::AbstractString)) +takes a hot, transactionally consistent snapshot of the environment to +another directory. With `compact = true`, free-space pages are omitted +and the destination is approximately the size of the live data set: + +```julia +copy(env, "/backup/mydb-snapshot"; compact = true) +``` + +There is also a file-descriptor variant — `copy(env, fd)` — for +streaming the snapshot to a pipe or socket. + +## Reader management + +Each open read transaction occupies one reader slot. If a process +crashes without releasing its txns, the slots remain reserved until the +env is closed. `reader_check` reaps such stale slots and returns the +count of slots cleared: + +```julia +n = reader_check(env) +@info "reaped $n stale readers" +``` + +`reader_list(env)` returns a human-readable dump of every active slot +(PID, thread, txn id) for diagnosing reader-table contention. diff --git a/docs/src/man/essentials.md b/docs/src/man/essentials.md new file mode 100644 index 0000000..8432f28 --- /dev/null +++ b/docs/src/man/essentials.md @@ -0,0 +1,132 @@ +# Essentials + +```@meta +CurrentModule = LMDB +``` + +After importing LMDB.jl, you can immediately query the bundled library: + +```julia-repl +julia> using LMDB + +julia> LMDB.version() +(v"0.9.33", "LMDB 0.9.33: (May 21, 2024)") +``` + +## A complete example + +The easiest entry point is the [`LMDBDict`](@ref), a persistent +`AbstractDict{K,V}` backed by a single LMDB environment: + +```julia +using LMDB + +d = LMDBDict{String, Vector{Float32}}("/tmp/mydb") +d["alpha"] = Float32[1, 2, 3] +d["beta/x"] = Float32[10, 11] + +@show d["alpha"] # [1.0, 2.0, 3.0] +@show haskey(d, "alpha") # true +@show length(d) # 2 + +for (k, v) in d + @show k, v +end + +close(d) +``` + +Behind the scenes this opens an `Environment` with `MDB_NOTLS` (so +multiple read transactions can coexist on a single thread) and a single +default `DBI`. Type conversions happen automatically — anything the +`MDBValue` constructor accepts (`String`, `Vector{T}` of bitstype `T`, +or any bitstype scalar) can be stored. + +## The three tiers + +LMDB.jl is organised in layers. The same database can be accessed at any +of them, depending on what you need: + +``` +┌──────────────────────────────────────────────────────────────────┐ +│ Tier 3 — LMDBDict <: AbstractDict{K,V} │ +│ "I want a persistent Dict." │ +├──────────────────────────────────────────────────────────────────┤ +│ Tier 2 — Environment, Transaction, DBI, Cursor │ +│ Julian wrappers with finalizers and parent refs. │ +│ The recommended surface for most code. │ +├──────────────────────────────────────────────────────────────────┤ +│ Tier 1.5 — MDBValue, MDBArg, MDBValueIO │ +│ cconvert/unsafe_convert glue between Julia values │ +│ and `Ptr{MDB_val}` arguments, and an `IO` view over │ +│ `MDB_val` for typed reads via `Base.read(io, T)`. │ +├──────────────────────────────────────────────────────────────────┤ +│ Tier 1 — mdb_*, MDB_*, unchecked_mdb_* │ +│ @ccall bindings; status-returning ones auto-throw │ +│ and have `unchecked_*` companions. │ +└──────────────────────────────────────────────────────────────────┘ +``` + +The recommended progression is: + +1. Start with [Dictionary interface](@ref) (tier 3) until you need + transactional grouping or zero-copy reads. +2. Drop to [Environments](@ref) → [Transactions](@ref) → + [Databases](@ref) → [Cursors](@ref) (tier 2) for explicit lifetimes + and fine-grained control. +3. Reach for [Low-level bindings](@ref) (tier 1) only when integrating + with a custom data layout or when the wrappers introduce overhead + you can't afford. + +## Resource lifecycle + +Each tier-2 handle type wraps a raw LMDB pointer in a `mutable struct` +with a finalizer: + +| handle | finalizer | parent ref | +|--------|-----------|------------| +| `Environment` | `close` (`mdb_env_close`) | – | +| `Transaction` | `abort` (`mdb_txn_abort`) | `Environment` | +| `Cursor` | `close` (`mdb_cursor_close`) | `Transaction`, `DBI` | +| `LMDBDict` | `close` env + dbi | – | + +Parent references pin the lifetime: a `Cursor` keeps its `Transaction` +alive, which keeps its `Environment` alive. All cleanup operations +(`close`, `commit`, `abort`) are idempotent — calling them twice (or +on a never-opened handle) is a silent no-op. This means abandoned write +txns (e.g. from a `for … break` over an `LMDBDict`, or any error path) +are eventually reclaimed when GC runs. + +For most call sites the do-block constructors are the simplest correct +shape: + +```julia +environment("/tmp/mydb"; flags = MDB_NOTLS) do env + start(env) do txn + open(txn) do dbi + put!(txn, dbi, "k", "v") + end + end # commits on success, aborts on throw +end # closes env +``` + +## Errors + +Every LMDB-internal error surfaces as an `LMDBError`: + +```julia +try + LMDB.get(txn, dbi, "missing", String) +catch e + e isa LMDBError && is_notfound(e) || rethrow() + # treat as missing +end +``` + +Common branches have helpers (`is_notfound`, `is_keyexist`, +`is_map_full`); rarer codes can be matched against `LMDB.MDB_*` +constants directly. See [Errors](@ref API-Errors) for the full list. + +For the dominant "missing key" case, prefer the no-throw paths: +[`tryget(txn, dbi, key, T)`](@ref tryget) returns `nothing` on miss, +and `get(txn, dbi, key, T, default)` falls back to `default`. diff --git a/docs/src/man/lowlevel.md b/docs/src/man/lowlevel.md new file mode 100644 index 0000000..2108955 --- /dev/null +++ b/docs/src/man/lowlevel.md @@ -0,0 +1,151 @@ +# Low-level bindings + +```@meta +CurrentModule = LMDB +``` + +The tier-1 surface is the raw `ccall` interface to `liblmdb`. It is +public-but-unexported: refer to it as `LMDB.mdb_env_create`, +`LMDB.MDB_NOTLS`, `LMDB.MDB_val`. Use this layer when you need to +integrate with a custom data layout, branch on a status code that the +tier-2 wrappers don't surface, or skip allocations on a hot path. + +For the full inventory, see [the API reference](@ref API-LowLevel). What +follows is a tour of how the layer is shaped. + +## The auto-throwing convention + +Every status-returning binding is paired with an `unchecked_*` +companion at definition time: + +```julia +LMDB.mdb_env_open(env, path, flags, mode) # auto-throws on non-zero +LMDB.unchecked_mdb_env_open(env, path, flags, mode) # returns the raw Cint +``` + +Use the bare name when any error should propagate (the common case). +Use the `unchecked_*` companion when you need to inspect the raw status +yourself — e.g. distinguishing `MDB_NOTFOUND` from a real error: + +```julia +val_ref = Ref(LMDB.MDB_val(zero(Csize_t), C_NULL)) +ret = LMDB.unchecked_mdb_get(txn, dbi, key, val_ref) +ret == LMDB.MDB_NOTFOUND && return nothing +ret == 0 || throw(LMDB.LMDBError(ret)) +return read(LMDB.MDBValueIO(val_ref[]), T) +``` + +This is exactly the pattern [`tryget`](@ref) uses internally. + +Bindings that don't return a status (`mdb_strerror`, `mdb_version`, +`mdb_txn_id`, `mdb_cmp`, `mdb_dcmp`, `mdb_env_get_maxkeysize`, +`mdb_cursor_txn`, `mdb_cursor_dbi`) and `Cvoid`-returning ones +(`mdb_env_close`, `mdb_dbi_close`, `mdb_txn_abort`, `mdb_txn_reset`, +`mdb_cursor_close`) are left bare — there is nothing to check. + +## ccall glue: passing values to `Ptr{MDB_val}` + +LMDB exchanges keys and values through a `Ptr{MDB_val}` argument: a +two-field struct of `(size, data_ptr)` plus an out-pointer for the +ccall to fill in. LMDB.jl ships `Base.cconvert` overloads on +`Ptr{MDB_val}` that route any of `String`, `AbstractArray` (with +bitstype element type), `Base.RefValue` over a bitstype, any bitstype +scalar, or a pre-built `Ref{MDB_val}` (used as an out-param) into a +self-rooted argument that `ccall`'s automatic `GC.@preserve` keeps alive +across the call. Callers never need to write `Ref(...)` or +`GC.@preserve` for input arguments. + +```julia +import LMDB + +env_ref = Ref{Ptr{LMDB.MDB_env}}(C_NULL) +LMDB.mdb_env_create(env_ref) # auto-throws +env = env_ref[] +LMDB.mdb_env_set_mapsize(env, Csize_t(1 << 30)) +LMDB.mdb_env_open(env, "/tmp/mydb", + LMDB.MDB_NOTLS | LMDB.MDB_NORDAHEAD, + Cushort(0o644)) + +txn_ref = Ref{Ptr{LMDB.MDB_txn}}() +LMDB.mdb_txn_begin(env, C_NULL, Cuint(0), txn_ref) +txn = txn_ref[] + +dbi_ref = Ref{LMDB.MDB_dbi}() +LMDB.mdb_dbi_open(txn, C_NULL, Cuint(0), dbi_ref) +dbi = dbi_ref[] + +LMDB.mdb_put(txn, dbi, "key", "value", Cuint(0)) # cconvert handles strings +LMDB.mdb_txn_commit(txn) +LMDB.mdb_env_close(env) +``` + +## Decoding `MDB_val`: the [`MDBValueIO`](@ref) extension point + +A successful read populates a `Ref{MDB_val}` whose `mv_data` points +into the LMDB-owned mmap. `MDBValueIO` is a thin `IO` view over that +buffer; `Base.read(io, T)` decodes it into a Julia value of type `T`. + +The package ships these defaults: + +| `T` | behaviour | +|-----|-----------| +| `String` | one `unsafe_string` over the remaining bytes | +| `Vector{E}` for bitstype `E` | one alloc + `unsafe_copyto!`; the buffer is Julia-owned | +| any bitstype scalar `T` | one `unsafe_load` of `sizeof(T)` bytes — zero allocations | + +Custom representations are added by overloading `Base.read` on the +abstract `IO` (the idiomatic Julia form — keeps the decoder portable +to other byte sources): + +```julia +struct AtimedBlob end +function Base.read(io::IO, ::Type{AtimedBlob}) + bytesavailable(io) < 8 && return UInt8[] + skip(io, 8) + return read(io, Vector{UInt8}) +end + +LMDB.tryget(txn, dbi, key, AtimedBlob) # skip 8-byte prefix, copy tail +``` + +For an `isbitstype` struct `T`, the standard one-liner is enough: + +```julia +Base.read(io::IO, ::Type{T}) = read!(io, Ref{T}())[] +``` + +This is the analogue of heed's `BytesDecode<'txn>` trait. Every typed +read in tier 2 — `tryget`, `get`, `key`, `value`, `item`, typed `walk`, +`pop!`, `replace!` — funnels through `read(::MDBValueIO, T)`, so a +single method opt-in is enough to make a custom representation usable +across the package. Because `MDBValueIO <: IO`, all the standard +`Base` IO primitives (`position`, `seek`, `skip`, `read(io)`, +`read(io, n::Integer)`, `read!(io, A)`, `bytesavailable`, `eof`) work +out of the box, which makes structured framed-value decoders read +exactly like any other Julia parser. + +## Memory ownership rules + +- The `mv_data` pointer of an `MDB_val` produced by a *read* is into + LMDB's mmap. It is **valid only for the producing transaction's + lifetime** — copy out anything you want to retain past commit. + The default `Vector{E}` and `String` `read(::MDBValueIO, T)` methods + always copy; custom decoders are responsible for doing the same. +- The `mv_data` pointer of an `MDB_val` produced by a `MDB_RESERVE` + *write* points into the LMDB write buffer and is **valid only inside + the surrounding write transaction**. [`put_reserved!`](@ref) wraps + this; don't escape its `buf` argument. + +## Unwrapped LMDB features + +A few LMDB features are reachable only through tier 1 because the +tier-2 surface deliberately doesn't include them: + +- **Custom comparators.** `LMDB.mdb_set_compare` / + `LMDB.mdb_set_dupsort` accept a `MDB_cmp_func` callback. Use + `@cfunction` to lift a Julia function into the right C signature. +- **`mdb_set_relfunc` / `mdb_set_relctx`.** Used by + `MDB_FIXEDMAP`-style relocations; rarely needed. +- **`MDB_GET_MULTIPLE` / `MDB_NEXT_MULTIPLE` cursor ops.** Reachable by + passing the constant directly to `LMDB.mdb_cursor_get`. Useful with + `MDB_DUPFIXED` databases for batched reads. diff --git a/docs/src/man/transactions.md b/docs/src/man/transactions.md new file mode 100644 index 0000000..901e08b --- /dev/null +++ b/docs/src/man/transactions.md @@ -0,0 +1,125 @@ +# Transactions + +```@meta +CurrentModule = LMDB +``` + +Every LMDB operation runs inside a transaction. Transactions are either +**read-only** (any number can run concurrently) or **read-write** (one +at a time per environment). + +## Starting a transaction + +```julia +txn = start(env) # read-write +txn = start(env; flags = MDB_RDONLY) # read-only +``` + +LMDB can hold one writer plus an unlimited number of readers +concurrently. Read txns do not block writers and vice versa. + +The do-block form is the recommended shape — it commits on normal +return and aborts on throw: + +```julia +result = start(env) do txn + open(txn) do dbi + put!(txn, dbi, "k", "v") + tryget(txn, dbi, "k", String) + end +end # commits if no throw +``` + +## Commit / abort + +`commit(txn)` writes the txn's modifications to disk and frees the +handle. `abort(txn)` discards them. Both are idempotent — calling them +twice (or on a never-started txn) is a silent no-op. `Transaction`'s +finalizer calls `abort`, so an abandoned write txn eventually releases +LMDB's exclusive write mutex. + +After `commit` or `abort`, the txn (and any cursors created against it) +must not be used. Continuing to call `mdb_*` against a freed handle is +undefined behaviour. + +## Read-only transactions + +Read-only txns are cheap to start and stop, but for tight loops the +[`reset`](@ref Base.reset(::LMDB.Transaction)) / [`renew`](@ref renew) +pair is even cheaper: + +```julia +txn = start(env; flags = MDB_RDONLY) +for batch in batches + open(txn) do dbi + for k in batch + v = tryget(txn, dbi, k, String) + handle(k, v) + end + end + reset(txn) # release the reader slot but keep the handle + renew(txn) # acquire a fresh slot — sees newly-committed writes +end +abort(txn) +``` + +`reset` is only valid on read-only txns; `renew` fetches a new snapshot +of the database. Without `renew`, the parked txn would not see writes +committed in the meantime. + +## Sub-transactions + +A read-write txn can spawn a child write txn that sees the parent's +uncommitted state. `commit` on the child folds its changes into the +parent; `abort` discards them, but the parent continues: + +```julia +start(env) do parent + open(parent) do dbi + put!(parent, dbi, "before", "1") + try + start(env; parent = parent) do child + put!(child, dbi, "during", "2") + error("oops") # abort propagates + end + catch + end + # "before" survives; "during" was rolled back + @assert tryget(parent, dbi, "during", String) === nothing + end +end +``` + +LMDB does not support nested *read-only* txns — pass a write txn as the +parent. + +## Reader slots + +Each open read txn occupies one reader slot. The default `maxreaders` +is small (126); raise it via `Environment(...; maxreaders = N)` for +high-concurrency read workloads, or call [`reader_check(env)`](@ref) to +reap slots left behind by crashed processes. + +Aggressive `for … break` over an `LMDBDict` without GC pressure can +pile up read txns; the explicit +[`walk(f, cur)`](@ref API-Cur-walk) form inside an `open(txn) do …` +block is leak-free. + +## Picking flags + +The most common patterns: + +```julia +# Hot read path — many small lookups, no writes +start(env; flags = MDB_RDONLY) do txn ... end + +# Bulk import — single transaction across many writes (atomic, fast) +start(env) do txn ... end + +# Long-running reader (e.g. background scrubber) — reset + renew loop +txn = start(env; flags = MDB_RDONLY) +while running + ... + reset(txn); renew(txn) +end +``` diff --git a/docs/src/manual.md b/docs/src/manual.md deleted file mode 100644 index b137aae..0000000 --- a/docs/src/manual.md +++ /dev/null @@ -1,119 +0,0 @@ -### Working with the database - -First, an LMDB environment needs to be created. The `create` function creates an `Environment` object that contains a DB environment handle. -```julia -env = create() -``` -Before opening the environment, you can set its parameters. -Environment parameters are set with the `put!` function, which accepts: -* `Environment` object -* `option` symbol which indicates parameter name, and -* parameter `value`. - -```julia -env[:DBs] = 2 -``` - -Environment parameters can be read with the `get` function: -```julia -env[:Readers] -``` - -Next, an environment must be opened using `open` function that takes as a parameter the path to the directory where database files reside. Make sure that the database directory exists and is writable. -```julia -open(env, "./testdb") -``` - -After opening the environment, create a transaction with the `start` function. It creates a new transaction and returns a `Transaction` object. -```julia -txn = start(env) -``` - -Next step, you need to open database using the `open` function, which takes the transaction as an argument. -```julia -dbi = open(txn) -``` - -Put key-value pair into the database with the `put!` function: -```julia -put!(txn, dbi, "key", "val") -``` - -Commit all the operations of a transaction into the database. The transaction and its cursors must not be used afterwards, because its handle has been freed. -```julia -commit(txn) -``` - -When you have finished working with the database, close it with a `close` call: -```julia -close(env, dbi) -``` - -After you finished working with the environment, it has to be closed to free resources: -```julia -close(env) -``` - - -### Complete example -```julia -env = create() # create new db environment -try - open(env, "./testdb") # open db environment !!! `testdb` must exist !!! - txn = start(env) # start new transaction - dbi = open(txn) # open database - try - put!(txn, dbi, "key", "val") # add key-value pair - commit(txn) # commit transaction - finally - close(env, dbi) # close db - end -finally - close(env) # close environment -end -``` - -### Dict-like data access - -If you don't want to deal directly with the C interface it is possible to use the `LMDBDict` type that implements parts of Julia's Dictionary interface, except iteration. One can construct a database connected to the folder `"mydb"` the following way: - -````julia -d = LMDBDict{String, Vector{Float64}}("mydb") -```` - -Note that to avoid repeated opening and closing of the database, the dict wraps a few C Pointers. They might be used by different threads, but should not be copied to other processes. The pointers will be freed when the object is gc-ed or manually when `close(d)` is called. -One can use this like a normal Julia dict: - -````julia -d["aa"] = [1.0, 2.0, 3.0] - -d["ab"] = 1.0:2.0:10.0 - -d["bb"] = [-1, -2, -3, -4] -```` - -Type conversions happen automatically if necessary. Keys, values and key-value pairs can be retrieved through `keys`, `values`, and `collect` respectively. -All these functions take an additional `prefix` argument. - -````julia -collect(d) -```` -```` -3-element Vector{Pair{String, Vector{Float64}}}: - "aa" => [1.0, 2.0, 3.0] - "ab" => [1.0, 3.0, 5.0, 7.0, 9.0] - "bb" => [-1.0, -2.0, -3.0, -4.0] -```` - -````julia -keys(d, prefix="a") -```` -```` -2-element Vector{String}: - "aa" - "ab" -```` - - - - From e9815f7f98c4857b574315df3045e5d395ccfc28 Mon Sep 17 00:00:00 2001 From: Tim Besard Date: Sun, 3 May 2026 19:06:36 +0200 Subject: [PATCH 3/8] Rename tiers to High-level abstractions / Julia API / C API. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Replaces the numbered "tier 1 / 1.5 / 2 / 3" naming throughout source, docs, and README with the explicit names of the three layers. The README is rewritten in this naming and the three stale references (`ms_entries` → `entries`, `mbd_unpack` → `read(MDBValueIO, T)`, `Cushort(0o644)` → `LMDB.mode_t(0o644)`) are corrected. Co-Authored-By: Claude Opus 4.7 (1M context) --- README.md | 79 ++++++++++++++++++++------------------ docs/src/index.md | 24 ++++++------ docs/src/lib/dict.md | 2 +- docs/src/lib/errors.md | 12 +++--- docs/src/lib/lowlevel.md | 2 +- docs/src/man/databases.md | 3 +- docs/src/man/dict.md | 2 +- docs/src/man/dupsort.md | 2 +- docs/src/man/essentials.md | 26 ++++++------- docs/src/man/lowlevel.md | 18 ++++----- src/LMDB.jl | 26 +++++++------ src/checked.jl | 2 +- src/cur.jl | 2 +- test/integration.jl | 6 +-- 14 files changed, 105 insertions(+), 101 deletions(-) diff --git a/README.md b/README.md index 6a25e87..aae4b69 100644 --- a/README.md +++ b/README.md @@ -1,9 +1,9 @@ # LMDB.jl -Julia bindings for [LMDB](http://www.lmdb.tech/doc/), the -Lightning Memory-Mapped Database — an embedded, memory-mapped, ACID -key-value store developed by Symas for OpenLDAP. It is small, fast, and -persists to disk while reading at near in-memory speeds. +Julia bindings for [LMDB](http://www.lmdb.tech/doc/), the Lightning +Memory-Mapped Database — an embedded, memory-mapped, ACID key-value +store developed by Symas for OpenLDAP. Small, fast, persisted to disk, +and reads at near in-memory speeds. ```julia using Pkg; Pkg.add("LMDB") @@ -11,25 +11,26 @@ using Pkg; Pkg.add("LMDB") ## Three layers -LMDB.jl exposes three tiers, each with a clear consumer: - -``` -Tier 3 LMDBDict — `AbstractDict` over a single LMDB file. -Tier 2 Environment, … — Julian wrappers: handles, txns, cursors, dicts. -Tier 1 mdb_*, MDB_* — raw bindings + status-code constants. -``` - -Tier 3 is the easy mode. Tier 2 is what most users want. Tier 1 is for -power users who need to integrate with custom data layouts or skip -allocations on hot paths — its functions auto-throw on non-zero status, -and an `unchecked_*` companion is available for callers that need to -inspect the raw status code. - -### Tier 3 — `LMDBDict` - -`LMDBDict{K,V} <: AbstractDict{K,V}`, so the standard library does most -of the work — `merge!`, `filter!`, `pairs`, `==`, `hash`, `keys`, -`values`, lazy iteration — all come for free: +LMDB.jl exposes the same database through three layers, each with a +clear consumer: + +- **High-level abstractions** — `LMDBDict <: AbstractDict`, an + `AbstractDict{K,V}` over a single LMDB file. Standard library + machinery (`merge!`, `filter!`, `pairs`, iteration, …) works out + of the box. Reach for this when you want a persistent `Dict`. +- **Julia API** — `Environment`, `Transaction`, `DBI`, `Cursor`. Julian + wrappers around handles, transactions, and cursors, with finalizers, + `do`-block forms, and typed reads through the + [`MDBValueIO`](https://en.wikipedia.org/wiki/Memory-mapped_file) + extension point. The recommended surface for most code that needs + explicit transactions. +- **C API** — `LMDB.mdb_*` and `LMDB.MDB_*`. Raw `ccall` bindings and + status-code constants. Status-returning functions auto-throw + `LMDBError` on a non-zero return; an `unchecked_*` companion is + available where the caller needs to inspect the raw status (for + example, branching on `MDB_NOTFOUND`). + +### High-level abstractions — `LMDBDict` ```julia using LMDB @@ -50,10 +51,10 @@ close(d) ``` Constructor kwargs: `mapsize`, `readers`, `dbs`, `readonly`, `rdahead`. -The env is opened with `MDB_NOTLS` so multiple read txns can coexist on -one thread — needed for interleaved reads or task-parallel access. +The env is opened with `MDB_NOTLS` so multiple read txns can coexist +on a single thread. -### Tier 2 — explicit env / txn / cursor +### Julia API — explicit env / txn / cursor ```julia using LMDB @@ -66,18 +67,18 @@ try put!(txn, dbi, "k1", "hello") put!(txn, dbi, "k2", [1.0, 2.0, 3.0]) - @show LMDB.tryget(txn, dbi, "k1", String) # "hello" + @show LMDB.tryget(txn, dbi, "k1", String) @show LMDB.get(txn, dbi, "missing", String, "default") - @show LMDB.stat(txn, dbi).ms_entries # 2 + @show LMDB.stat(txn, dbi).entries end end - # Cursor walk: zero-copy access to raw MDB_val refs. + # Cursor walk over the LMDB-owned mmap (zero-copy access). start(env; flags = MDB_RDONLY) do txn open(txn) do dbi open(txn, dbi) do cur - LMDB.walk(cur) do k_ref, v_ref - println(LMDB.mbd_unpack(String, k_ref)) + LMDB.walk(cur, String, String) do k, v + println(k, " => ", v) end end end @@ -87,20 +88,25 @@ finally end ``` -Status-code matchers are in `LMDBError`: +The package decodes `String`, `Vector{T}` for any bitstype `T`, and +the primitive numeric types out of the box. To plug in a custom +representation, define a single `Base.read(io::IO, ::Type{T})` method; +it will be picked up by `tryget` / `get` / `walk(f, cur, K, V)` and +the cursor accessors `key`/`value`/`item`. Status-code matchers live +on `LMDBError`: ```julia try LMDB.get(txn, dbi, "missing", String) catch e e isa LMDBError && LMDB.is_notfound(e) || rethrow() - # … + # treat as missing end ``` -### Tier 1 — raw bindings +### C API — raw bindings -The bindings are `LMDB.mdb_*`; constants like `LMDB.MDB_NOTLS`, +The bindings are `LMDB.mdb_*`; constants like `LMDB.MDB_NOTLS` and `LMDB.MDB_NOTFOUND` are public-but-unexported. Status-returning bindings have an auto-throwing default and an `unchecked_*` companion: @@ -114,7 +120,7 @@ LMDB.mdb_env_set_maxreaders(env, Cuint(510)) LMDB.mdb_env_set_mapsize(env, Csize_t(1 << 30)) LMDB.mdb_env_open(env, "/tmp/mydb", LMDB.MDB_NOTLS | LMDB.MDB_NORDAHEAD, - Cushort(0o644)) + LMDB.mode_t(0o644)) # Inspect the raw status code (e.g. for MDB_NOTFOUND): ret = LMDB.unchecked_mdb_get(txn, dbi, key, val_ref) @@ -126,4 +132,3 @@ ret == 0 || throw(LMDB.LMDBError(ret)) - LMDB upstream: - LMDB API docs: -- Julia LMDB.jl issues / PRs: this repository. diff --git a/docs/src/index.md b/docs/src/index.md index 92e4b60..8fa0a11 100644 --- a/docs/src/index.md +++ b/docs/src/index.md @@ -14,18 +14,18 @@ using Pkg; Pkg.add("LMDB") ## Three layers of abstraction -LMDB.jl exposes three tiers, each with a clear consumer: - -| Tier | Surface | When to use | -|------|---------|-------------| -| **3** | `LMDBDict <: AbstractDict{K,V}` | "I want a persistent `Dict`." | -| **2** | `Environment`, `Transaction`, `DBI`, `Cursor` | Julian wrappers with explicit transactions and cursors. The recommended surface for most code. | -| **1** | `LMDB.mdb_*`, `LMDB.MDB_*` | Raw `ccall` bindings + status-code constants. For power users integrating with custom data layouts or shaving allocations on hot paths. | - -A light **tier 1.5** sits between tier 1 and tier 2: `MDBValue`, `MDBArg`, -and the [`MDBValueIO`](@ref LMDB.MDBValueIO) extension point — an `IO` -view over `MDB_val` that lets custom value representations plug into -all the typed reads via `Base.read(io, T)`. +LMDB.jl exposes three layers, each with a clear consumer: + +| Layer | Surface | When to use | +|-------|---------|-------------| +| **High-level abstractions** | `LMDBDict <: AbstractDict{K,V}` | "I want a persistent `Dict`." | +| **Julia API** | `Environment`, `Transaction`, `DBI`, `Cursor` | Julian wrappers with explicit transactions and cursors. The recommended surface for most code. | +| **C API** | `LMDB.mdb_*`, `LMDB.MDB_*` | Raw `ccall` bindings + status-code constants. For power users integrating with custom data layouts or shaving allocations on hot paths. | + +The C API also includes `MDBValue`, `MDBArg`, and the +[`MDBValueIO`](@ref LMDB.MDBValueIO) extension point — an `IO` view +over `MDB_val` that lets custom value representations plug into all the +typed reads via `Base.read(io, T)`. The Usage section is organised in increasing order of complexity: start with [Essentials](@ref) for a working example, then [Dictionary diff --git a/docs/src/lib/dict.md b/docs/src/lib/dict.md index edd9777..adc6901 100644 --- a/docs/src/lib/dict.md +++ b/docs/src/lib/dict.md @@ -1,6 +1,6 @@ # [Dictionary interface](@id API-Dict) -The tier-3 surface: a single `AbstractDict{K,V}` over an LMDB environment. +The high-level abstraction: a single `AbstractDict{K,V}` over an LMDB environment. ```@meta CurrentModule = LMDB diff --git a/docs/src/lib/errors.md b/docs/src/lib/errors.md index 5164e3d..3ab0960 100644 --- a/docs/src/lib/errors.md +++ b/docs/src/lib/errors.md @@ -46,15 +46,15 @@ The rest (`MDB_PAGE_NOTFOUND`, `MDB_CORRUPTED`, `MDB_PANIC`, `MDB_MAP_RESIZED`, `MDB_INCOMPATIBLE`, `MDB_BAD_RSLOT`, `MDB_BAD_TXN`, `MDB_BAD_VALSIZE`, `MDB_BAD_DBI`) live under the `LMDB.` prefix. -## Where errors come from at each tier +## Where errors come from at each layer -- **Tier 1.** Bindings wrapped by `@checked` auto-throw `LMDBError`; +- **C API.** Bindings wrapped by `@checked` auto-throw `LMDBError`; the `unchecked_*` companion returns the raw `Cint` so the caller can branch on `MDB_NOTFOUND`/`MDB_KEYEXIST`/etc. -- **Tier 2.** Handle methods that wrap status-returning bindings let +- **Julia API.** Handle methods that wrap status-returning bindings let `LMDBError` propagate. `tryget` and `get(..., default)` swallow `MDB_NOTFOUND` and return `nothing`/`default`. `delete!(txn, dbi, key)` likewise swallows `MDB_NOTFOUND` and returns `false`. -- **Tier 3.** Missing keys produce `KeyError` (matching `Base.Dict`). - `pop!(d)` on an empty dict throws `ArgumentError`. Other LMDB errors - propagate as `LMDBError`. +- **High-level abstractions.** Missing keys produce `KeyError` + (matching `Base.Dict`). `pop!(d)` on an empty dict throws + `ArgumentError`. Other LMDB errors propagate as `LMDBError`. diff --git a/docs/src/lib/lowlevel.md b/docs/src/lib/lowlevel.md index a56b484..afbaded 100644 --- a/docs/src/lib/lowlevel.md +++ b/docs/src/lib/lowlevel.md @@ -4,7 +4,7 @@ CurrentModule = LMDB ``` -The tier-1 surface is a flat namespace of `ccall` bindings (`LMDB.mdb_*`), +The C API is a flat namespace of `ccall` bindings (`LMDB.mdb_*`), opaque handle types (`LMDB.MDB_env`, `LMDB.MDB_txn`, `LMDB.MDB_cursor`), plain structs (`LMDB.MDB_val`, `LMDB.MDB_stat`, `LMDB.MDB_envinfo`), the cursor-op `@cenum` (`LMDB.MDB_cursor_op`), and `LMDB.MDB_*` flag/status diff --git a/docs/src/man/databases.md b/docs/src/man/databases.md index 9e08697..1743f2e 100644 --- a/docs/src/man/databases.md +++ b/docs/src/man/databases.md @@ -44,8 +44,7 @@ env's finalizer cascades through any open DBI handles. ## Reads -Every read takes a value-type parameter `T`. The default tier-2 forms -are: +Every read takes a value-type parameter `T`. The default forms are: ```julia get(txn, dbi, key, T) # throws LMDBError(MDB_NOTFOUND) on miss diff --git a/docs/src/man/dict.md b/docs/src/man/dict.md index a331fb0..0fbad0b 100644 --- a/docs/src/man/dict.md +++ b/docs/src/man/dict.md @@ -132,7 +132,7 @@ audits without `stat`. ## When to drop down -Reach for the explicit tier-2 surface (next chapters) when: +Reach for the explicit Julia API (next chapters) when: - you need **multi-key atomicity**: a single transaction grouping more than one `put!`/`delete!`, diff --git a/docs/src/man/dupsort.md b/docs/src/man/dupsort.md index 8f44f3c..86ce0e9 100644 --- a/docs/src/man/dupsort.md +++ b/docs/src/man/dupsort.md @@ -89,7 +89,7 @@ the rest of the dups intact. - `MDB_DUPSORT | MDB_DUPFIXED`: every duplicate has the same byte size; LMDB stores them as a packed array per key. Required for the `MDB_GET_MULTIPLE` / `MDB_NEXT_MULTIPLE` cursor ops (reachable from - the tier-1 surface). + the C API). - `MDB_DUPSORT | MDB_DUPFIXED | MDB_INTEGERDUP`: values are native-endian integers; sorted numerically. - `MDB_DUPSORT | MDB_REVERSEDUP`: values compared back-to-front. diff --git a/docs/src/man/essentials.md b/docs/src/man/essentials.md index 8432f28..48a80ce 100644 --- a/docs/src/man/essentials.md +++ b/docs/src/man/essentials.md @@ -42,45 +42,43 @@ default `DBI`. Type conversions happen automatically — anything the `MDBValue` constructor accepts (`String`, `Vector{T}` of bitstype `T`, or any bitstype scalar) can be stored. -## The three tiers +## The three layers LMDB.jl is organised in layers. The same database can be accessed at any of them, depending on what you need: ``` ┌──────────────────────────────────────────────────────────────────┐ -│ Tier 3 — LMDBDict <: AbstractDict{K,V} │ +│ High-level abstractions — LMDBDict <: AbstractDict{K,V} │ │ "I want a persistent Dict." │ ├──────────────────────────────────────────────────────────────────┤ -│ Tier 2 — Environment, Transaction, DBI, Cursor │ +│ Julia API — Environment, Transaction, DBI, Cursor │ │ Julian wrappers with finalizers and parent refs. │ │ The recommended surface for most code. │ ├──────────────────────────────────────────────────────────────────┤ -│ Tier 1.5 — MDBValue, MDBArg, MDBValueIO │ -│ cconvert/unsafe_convert glue between Julia values │ -│ and `Ptr{MDB_val}` arguments, and an `IO` view over │ -│ `MDB_val` for typed reads via `Base.read(io, T)`. │ -├──────────────────────────────────────────────────────────────────┤ -│ Tier 1 — mdb_*, MDB_*, unchecked_mdb_* │ +│ C API — mdb_*, MDB_*, unchecked_mdb_* │ │ @ccall bindings; status-returning ones auto-throw │ -│ and have `unchecked_*` companions. │ +│ and have `unchecked_*` companions. `MDBValue`, │ +│ `MDBArg`, and `MDBValueIO` glue Julia values to │ +│ `Ptr{MDB_val}` and let custom decoders plug in via │ +│ `Base.read(io, T)`. │ └──────────────────────────────────────────────────────────────────┘ ``` The recommended progression is: -1. Start with [Dictionary interface](@ref) (tier 3) until you need +1. Start with the [Dictionary interface](@ref) until you need transactional grouping or zero-copy reads. 2. Drop to [Environments](@ref) → [Transactions](@ref) → - [Databases](@ref) → [Cursors](@ref) (tier 2) for explicit lifetimes + [Databases](@ref) → [Cursors](@ref) for explicit lifetimes and fine-grained control. -3. Reach for [Low-level bindings](@ref) (tier 1) only when integrating +3. Reach for the [Low-level bindings](@ref) only when integrating with a custom data layout or when the wrappers introduce overhead you can't afford. ## Resource lifecycle -Each tier-2 handle type wraps a raw LMDB pointer in a `mutable struct` +Each Julia-API handle type wraps a raw LMDB pointer in a `mutable struct` with a finalizer: | handle | finalizer | parent ref | diff --git a/docs/src/man/lowlevel.md b/docs/src/man/lowlevel.md index 2108955..7e87898 100644 --- a/docs/src/man/lowlevel.md +++ b/docs/src/man/lowlevel.md @@ -4,11 +4,11 @@ CurrentModule = LMDB ``` -The tier-1 surface is the raw `ccall` interface to `liblmdb`. It is +The C API is the raw `ccall` interface to `liblmdb`. It is public-but-unexported: refer to it as `LMDB.mdb_env_create`, `LMDB.MDB_NOTLS`, `LMDB.MDB_val`. Use this layer when you need to integrate with a custom data layout, branch on a status code that the -tier-2 wrappers don't surface, or skip allocations on a hot path. +Julia API doesn't surface, or skip allocations on a hot path. For the full inventory, see [the API reference](@ref API-LowLevel). What follows is a tour of how the layer is shaped. @@ -64,7 +64,7 @@ env = env_ref[] LMDB.mdb_env_set_mapsize(env, Csize_t(1 << 30)) LMDB.mdb_env_open(env, "/tmp/mydb", LMDB.MDB_NOTLS | LMDB.MDB_NORDAHEAD, - Cushort(0o644)) + LMDB.mode_t(0o644)) txn_ref = Ref{Ptr{LMDB.MDB_txn}}() LMDB.mdb_txn_begin(env, C_NULL, Cuint(0), txn_ref) @@ -115,10 +115,10 @@ Base.read(io::IO, ::Type{T}) = read!(io, Ref{T}())[] ``` This is the analogue of heed's `BytesDecode<'txn>` trait. Every typed -read in tier 2 — `tryget`, `get`, `key`, `value`, `item`, typed `walk`, -`pop!`, `replace!` — funnels through `read(::MDBValueIO, T)`, so a -single method opt-in is enough to make a custom representation usable -across the package. Because `MDBValueIO <: IO`, all the standard +read in the Julia API — `tryget`, `get`, `key`, `value`, `item`, typed +`walk`, `pop!`, `replace!` — funnels through `read(::MDBValueIO, T)`, +so a single method opt-in is enough to make a custom representation +usable across the package. Because `MDBValueIO <: IO`, all the standard `Base` IO primitives (`position`, `seek`, `skip`, `read(io)`, `read(io, n::Integer)`, `read!(io, A)`, `bytesavailable`, `eof`) work out of the box, which makes structured framed-value decoders read @@ -138,8 +138,8 @@ exactly like any other Julia parser. ## Unwrapped LMDB features -A few LMDB features are reachable only through tier 1 because the -tier-2 surface deliberately doesn't include them: +A few LMDB features are reachable only through the C API because the +Julia API deliberately doesn't include them: - **Custom comparators.** `LMDB.mdb_set_compare` / `LMDB.mdb_set_dupsort` accept a `MDB_cmp_func` callback. Use diff --git a/src/LMDB.jl b/src/LMDB.jl index 601cdde..46a0c64 100644 --- a/src/LMDB.jl +++ b/src/LMDB.jl @@ -23,25 +23,25 @@ export # commonly-needed write flags MDB_NOOVERWRITE, MDB_NODUPDATA, MDB_APPEND, MDB_RESERVE, - # tier 2 — environment + # Julia API — environment Environment, create, environment, sync, set!, unset!, info, stat, path, isopen, isflagset, reader_check, reader_list, - # tier 2 — transaction + # Julia API — transaction Transaction, start, abort, commit, reset, renew, - # tier 2 — database (DBI) - DBI, drop, get, put!, delete!, tryget, replace!, + # Julia API — database (DBI) + DBI, drop, get, put!, put_reserved!, delete!, tryget, replace!, - # tier 2 — cursor + # Julia API — cursor Cursor, count, transaction, database, seek!, seek_last!, seek_range!, next!, prev!, key, value, item, walk, seek_first_dup!, seek_last_dup!, next_dup!, prev_dup!, next_nodup!, prev_nodup!, - # tier 3 + # High-level abstractions LMDBDict # --------------------------------------------------------------------------- @@ -100,15 +100,15 @@ done after `close(env)` without rewriting the database). is_map_full # --------------------------------------------------------------------------- -# Tier 1 — raw bindings, types, constants. Public-but-unexported. +# C API — raw bindings, types, constants. Public-but-unexported. # # Every status-returning binding has a `@checked` wrapper (auto-throws) and an # `unchecked_*` companion (returns the raw `Cint` for callers that need to # inspect it, e.g. branching on `MDB_NOTFOUND`). # # Use as `LMDB.mdb_env_create`, `LMDB.MDB_NOTLS`, `LMDB.MDB_val`. Mostly -# relevant to power users; tier 2 (`Environment`, `Transaction`, …) is the -# recommended surface. +# relevant to power users; the Julia API (`Environment`, `Transaction`, …) is +# the recommended surface. # --------------------------------------------------------------------------- include("checked.jl") @@ -123,13 +123,15 @@ is_keyexist(err::LMDBError) = err.code == MDB_KEYEXIST is_map_full(err::LMDBError) = err.code == MDB_MAP_FULL # --------------------------------------------------------------------------- -# Tier 1.5 — ccall glue. +# ccall glue between the C API and the Julia API: `MDBValue`, `MDBArg`, and +# the `MDBValueIO <: IO` extension point used to plug in custom decoders via +# `Base.read(io, T)`. # --------------------------------------------------------------------------- include("common.jl") # --------------------------------------------------------------------------- -# Tier 2 — Julian wrappers around the raw bindings. +# Julia API — Julian wrappers around the raw bindings. # --------------------------------------------------------------------------- include("env.jl") @@ -138,7 +140,7 @@ include("dbi.jl") include("cur.jl") # --------------------------------------------------------------------------- -# Tier 3 — high-level convenience. +# High-level abstractions — `LMDBDict`. # --------------------------------------------------------------------------- include("dicts.jl") diff --git a/src/checked.jl b/src/checked.jl index 3d2bcd7..5535a51 100644 --- a/src/checked.jl +++ b/src/checked.jl @@ -1,4 +1,4 @@ -# Applied to a tier-1 binding that returns an LMDB status code (`Cint`). +# Applied to a C-API binding that returns an LMDB status code (`Cint`). # Emits two functions: # # * `(...)` — same name, throws `LMDBError` on a non-zero diff --git a/src/cur.jl b/src/cur.jl index 0d8477d..9623c74 100644 --- a/src/cur.jl +++ b/src/cur.jl @@ -328,7 +328,7 @@ end walk(f, cur::Cursor, ::Type{K}, ::Type{V}=K; from = nothing) Typed overload of `walk` mirroring the `tryget(txn, dbi, key, T)` / -`key(cur, T)` / `seek!(cur, key, T)` shape used elsewhere in tier-2. +`key(cur, T)` / `seek!(cur, key, T)` shape used elsewhere in the Julia API. Decodes each key and value through `read(::MDBValueIO, K)` / `read(::MDBValueIO, V)` before passing them to `f(k::K, v::V)`. Same stop contract as the raw form: `f` returning `false` halts iteration. diff --git a/test/integration.jl b/test/integration.jl index 1754e79..e7c9ac6 100644 --- a/test/integration.jl +++ b/test/integration.jl @@ -26,8 +26,8 @@ end @testset "Integration" begin # Power-user pattern: open an env via the Environment kwargs ctor, run -# a write txn through tier-2, then a read txn through a cursor walk -# using only tier-2 + raw MDB_val refs (the shape cuTile.DiskCache +# a write txn through the Julia API, then a read txn through a cursor walk +# using only the Julia API + raw MDB_val refs (the shape cuTile.DiskCache # follows). Regression guard: ensures no future change breaks the # `walk(...) do k_ref, v_ref` zero-copy idiom. mktempdir() do dir @@ -49,7 +49,7 @@ mktempdir() do dir end end - # Tier-2 read txn + cursor walk over the LMDB-owned mmap, like + # Julia-API read txn + cursor walk over the LMDB-owned mmap, like # cuTile's eviction scan: zero allocations beyond the per-entry # tuple. entries = Tuple{String, Int}[] From b9b6e697c374e9a0e91c6b3964cb6a0770a30cab Mon Sep 17 00:00:00 2001 From: Tim Besard Date: Sun, 3 May 2026 20:24:42 +0200 Subject: [PATCH 4/8] Modernize CI, docs, and README for JuliaDatabases transfer. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - CI.yml: bump to checkout@v4, setup-julia@v2, julia-actions/cache@v2, codecov-action@v5; matrix is lts/1/pre/nightly × ubuntu/macOS/windows; drop coveralls; add concurrency group. - Documenter.yml: pin julia-buildpkg@v1 / julia-docdeploy@v1, add julia setup + cache, set the permissions block expected by docdeploy. - CompatHelper.yml: standard nightly cron. - README.md: add the conventional CI / codecov / docs-stable / docs-dev badge row. - docs/make.jl: enable doctests via DocMeta.setdocmeta! and doctest=true. --- .github/workflows/CI.yml | 6 ++++ .github/workflows/CompatHelper.yml | 46 ++++++++++++++++++++++++++++++ .github/workflows/Documenter.yml | 17 +++++++++-- README.md | 5 ++++ docs/make.jl | 3 ++ 5 files changed, 74 insertions(+), 3 deletions(-) create mode 100644 .github/workflows/CompatHelper.yml diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index fb6a755..7f2bc80 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -4,6 +4,11 @@ on: branches: [main, master] tags: ["*"] pull_request: + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: ${{ startsWith(github.ref, 'refs/heads/') }} + jobs: test: name: Julia ${{ matrix.version }} - ${{ matrix.os }} - ${{ matrix.arch }} - ${{ github.event_name }} @@ -38,3 +43,4 @@ jobs: with: files: lcov.info token: ${{ secrets.CODECOV_TOKEN }} + fail_ci_if_error: false diff --git a/.github/workflows/CompatHelper.yml b/.github/workflows/CompatHelper.yml new file mode 100644 index 0000000..866c0bc --- /dev/null +++ b/.github/workflows/CompatHelper.yml @@ -0,0 +1,46 @@ +name: CompatHelper +on: + schedule: + - cron: 0 0 * * * + workflow_dispatch: + +permissions: + contents: write + pull-requests: write + +jobs: + CompatHelper: + runs-on: ubuntu-latest + steps: + - name: Check if Julia is already available in the PATH + id: julia_in_path + run: which julia + continue-on-error: true + - name: Install Julia, but only if it is not already available in the PATH + uses: julia-actions/setup-julia@v2 + with: + version: '1' + arch: ${{ runner.arch }} + if: steps.julia_in_path.outcome != 'success' + - name: "Add the General registry via Git" + run: | + import Pkg + ENV["JULIA_PKG_SERVER"] = "" + Pkg.Registry.add("General") + shell: julia --color=yes {0} + - name: "Install CompatHelper" + run: | + import Pkg + name = "CompatHelper" + uuid = "aa819f21-2bde-4658-8897-bab36330d9b7" + version = "3" + Pkg.add(; name, uuid, version) + shell: julia --color=yes {0} + - name: "Run CompatHelper" + run: | + import CompatHelper + CompatHelper.main() + shell: julia --color=yes {0} + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + COMPATHELPER_PRIV: ${{ secrets.DOCUMENTER_KEY }} diff --git a/.github/workflows/Documenter.yml b/.github/workflows/Documenter.yml index 424c69e..fc788b2 100644 --- a/.github/workflows/Documenter.yml +++ b/.github/workflows/Documenter.yml @@ -2,17 +2,28 @@ name: Documenter on: push: branches: [main, master] - tags: [v*] + tags: ['*'] pull_request: +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: ${{ startsWith(github.ref, 'refs/heads/') }} + jobs: Documenter: name: Documentation runs-on: ubuntu-latest + permissions: + contents: write + statuses: write steps: - uses: actions/checkout@v4 - - uses: julia-actions/julia-buildpkg@latest - - uses: julia-actions/julia-docdeploy@latest + - uses: julia-actions/setup-julia@v2 + with: + version: '1' + - uses: julia-actions/cache@v2 + - uses: julia-actions/julia-buildpkg@v1 + - uses: julia-actions/julia-docdeploy@v1 env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} DOCUMENTER_KEY: ${{ secrets.DOCUMENTER_KEY }} diff --git a/README.md b/README.md index aae4b69..89e17fa 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,10 @@ # LMDB.jl +[![CI](https://github.com/maleadt/LMDB.jl/actions/workflows/CI.yml/badge.svg)](https://github.com/maleadt/LMDB.jl/actions/workflows/CI.yml) +[![codecov](https://codecov.io/gh/maleadt/LMDB.jl/branch/master/graph/badge.svg)](https://codecov.io/gh/maleadt/LMDB.jl) +[![Stable](https://img.shields.io/badge/docs-stable-blue.svg)](https://maleadt.github.io/LMDB.jl/stable) +[![Dev](https://img.shields.io/badge/docs-dev-blue.svg)](https://maleadt.github.io/LMDB.jl/dev) + Julia bindings for [LMDB](http://www.lmdb.tech/doc/), the Lightning Memory-Mapped Database — an embedded, memory-mapped, ACID key-value store developed by Symas for OpenLDAP. Small, fast, persisted to disk, diff --git a/docs/make.jl b/docs/make.jl index 44ebce3..7a61441 100644 --- a/docs/make.jl +++ b/docs/make.jl @@ -1,5 +1,7 @@ using Documenter, LMDB +DocMeta.setdocmeta!(LMDB, :DocTestSetup, :(using LMDB); recursive = true) + function main() ci = get(ENV, "CI", "") == "true" @@ -10,6 +12,7 @@ function main() edit_link = "master"), modules = [LMDB], checkdocs = :exports, + doctest = true, pages = [ "Home" => "index.md", "Usage" => [ From 1d8ed183b842679ad0e04a7bde3ade055b0e5864 Mon Sep 17 00:00:00 2001 From: Tim Besard Date: Thu, 21 May 2026 23:07:01 +0200 Subject: [PATCH 5/8] Point docs at JuliaDatabases/LMDB.jl, edit_link to main. --- docs/make.jl | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/docs/make.jl b/docs/make.jl index 7a61441..83b1c39 100644 --- a/docs/make.jl +++ b/docs/make.jl @@ -8,8 +8,9 @@ function main() makedocs( sitename = "LMDB.jl", authors = "Art Wild, Fabian Gans, Tim Besard", + repo = Documenter.Remotes.GitHub("JuliaDatabases", "LMDB.jl"), format = Documenter.HTML(prettyurls = ci, - edit_link = "master"), + edit_link = "main"), modules = [LMDB], checkdocs = :exports, doctest = true, @@ -39,7 +40,7 @@ function main() if ci deploydocs( - repo = "github.com/wildart/LMDB.jl.git", + repo = "github.com/JuliaDatabases/LMDB.jl.git", ) end end From c3a771d29bbb68936d040d66233d786438b27fee Mon Sep 17 00:00:00 2001 From: Tim Besard Date: Fri, 22 May 2026 10:14:36 +0200 Subject: [PATCH 6/8] Improve docs. --- README.md | 58 +++++++++++---------------- docs/src/index.md | 48 +++++++++++----------- docs/src/lib/dict.md | 29 +++++++------- docs/src/lib/environments.md | 6 +-- docs/src/lib/errors.md | 27 +++++++------ docs/src/lib/lowlevel.md | 12 +++--- docs/src/man/cursors.md | 30 +++++++------- docs/src/man/databases.md | 18 ++++----- docs/src/man/dict.md | 17 ++++---- docs/src/man/dupsort.md | 14 +++---- docs/src/man/environments.md | 24 ++++++----- docs/src/man/essentials.md | 77 +++++++++++++++--------------------- docs/src/man/lowlevel.md | 54 ++++++++++++------------- docs/src/man/transactions.md | 27 ++++++------- res/wrap.jl | 4 +- src/LMDB.jl | 24 +++++------ src/common.jl | 15 ++++--- src/cur.jl | 10 ++--- src/dbi.jl | 12 +++--- src/dicts.jl | 7 ++-- src/txn.jl | 2 +- test/integration.jl | 8 ++-- 22 files changed, 248 insertions(+), 275 deletions(-) diff --git a/README.md b/README.md index 89e17fa..f29613e 100644 --- a/README.md +++ b/README.md @@ -1,41 +1,36 @@ # LMDB.jl -[![CI](https://github.com/maleadt/LMDB.jl/actions/workflows/CI.yml/badge.svg)](https://github.com/maleadt/LMDB.jl/actions/workflows/CI.yml) -[![codecov](https://codecov.io/gh/maleadt/LMDB.jl/branch/master/graph/badge.svg)](https://codecov.io/gh/maleadt/LMDB.jl) -[![Stable](https://img.shields.io/badge/docs-stable-blue.svg)](https://maleadt.github.io/LMDB.jl/stable) -[![Dev](https://img.shields.io/badge/docs-dev-blue.svg)](https://maleadt.github.io/LMDB.jl/dev) +[![CI](https://github.com/JuliaDatabases/LMDB.jl/actions/workflows/CI.yml/badge.svg)](https://github.com/JuliaDatabases/LMDB.jl/actions/workflows/CI.yml) +[![codecov](https://codecov.io/gh/JuliaDatabases/LMDB.jl/branch/main/graph/badge.svg)](https://codecov.io/gh/JuliaDatabases/LMDB.jl) +[![Stable](https://img.shields.io/badge/docs-stable-blue.svg)](https://JuliaDatabases.github.io/LMDB.jl/stable) +[![Dev](https://img.shields.io/badge/docs-dev-blue.svg)](https://JuliaDatabases.github.io/LMDB.jl/dev) Julia bindings for [LMDB](http://www.lmdb.tech/doc/), the Lightning -Memory-Mapped Database — an embedded, memory-mapped, ACID key-value -store developed by Symas for OpenLDAP. Small, fast, persisted to disk, -and reads at near in-memory speeds. +Memory-Mapped Database: An embedded, memory-mapped, ACID key-value store +developed by Symas for OpenLDAP. Small, fast, persisted to disk, and reads at +near in-memory speeds. ```julia using Pkg; Pkg.add("LMDB") ``` -## Three layers +## Using LMDB.jl -LMDB.jl exposes the same database through three layers, each with a -clear consumer: +LMDB.jl exposes the same database through three surfaces: -- **High-level abstractions** — `LMDBDict <: AbstractDict`, an +- **High-level interface**: `LMDBDict <: AbstractDict`, an `AbstractDict{K,V}` over a single LMDB file. Standard library machinery (`merge!`, `filter!`, `pairs`, iteration, …) works out of the box. Reach for this when you want a persistent `Dict`. -- **Julia API** — `Environment`, `Transaction`, `DBI`, `Cursor`. Julian - wrappers around handles, transactions, and cursors, with finalizers, - `do`-block forms, and typed reads through the - [`MDBValueIO`](https://en.wikipedia.org/wiki/Memory-mapped_file) - extension point. The recommended surface for most code that needs +- **Julia wrappers**: `Environment`, `Transaction`, `DBI`, `Cursor`. + Julia-shaped wrappers around handles, transactions, and cursors, + with finalizers, `do`-block forms, etc. Use this when you want explicit transactions. -- **C API** — `LMDB.mdb_*` and `LMDB.MDB_*`. Raw `ccall` bindings and - status-code constants. Status-returning functions auto-throw - `LMDBError` on a non-zero return; an `unchecked_*` companion is - available where the caller needs to inspect the raw status (for - example, branching on `MDB_NOTFOUND`). +- **C API**: `LMDB.mdb_*` and `LMDB.MDB_*`. Raw `ccall` bindings and + status-code constants. Use this when the Julia wrappers don't expose + a particular API or you want to inspect status codes directly. -### High-level abstractions — `LMDBDict` +### `LMDBDict` ```julia using LMDB @@ -55,11 +50,7 @@ end close(d) ``` -Constructor kwargs: `mapsize`, `readers`, `dbs`, `readonly`, `rdahead`. -The env is opened with `MDB_NOTLS` so multiple read txns can coexist -on a single thread. - -### Julia API — explicit env / txn / cursor +### Julia wrappers ```julia using LMDB @@ -93,12 +84,11 @@ finally end ``` -The package decodes `String`, `Vector{T}` for any bitstype `T`, and -the primitive numeric types out of the box. To plug in a custom -representation, define a single `Base.read(io::IO, ::Type{T})` method; -it will be picked up by `tryget` / `get` / `walk(f, cur, K, V)` and -the cursor accessors `key`/`value`/`item`. Status-code matchers live -on `LMDBError`: +The package decodes `String`, `Vector{T}` for any bitstype `T`, and the +primitive numeric types out of the box. To plug in a custom representation, +define a `Base.read(io::IO, ::Type{T})` method; it will be picked up by `tryget` +/ `get` / `walk(f, cur, K, V)` and the cursor accessors `key`/`value`/`item`. +Status-code matchers live on `LMDBError`: ```julia try @@ -109,7 +99,7 @@ catch e end ``` -### C API — raw bindings +### C API bindings The bindings are `LMDB.mdb_*`; constants like `LMDB.MDB_NOTLS` and `LMDB.MDB_NOTFOUND` are public-but-unexported. Status-returning diff --git a/docs/src/index.md b/docs/src/index.md index 8fa0a11..5556271 100644 --- a/docs/src/index.md +++ b/docs/src/index.md @@ -12,31 +12,29 @@ space. using Pkg; Pkg.add("LMDB") ``` -## Three layers of abstraction - -LMDB.jl exposes three layers, each with a clear consumer: - -| Layer | Surface | When to use | -|-------|---------|-------------| -| **High-level abstractions** | `LMDBDict <: AbstractDict{K,V}` | "I want a persistent `Dict`." | -| **Julia API** | `Environment`, `Transaction`, `DBI`, `Cursor` | Julian wrappers with explicit transactions and cursors. The recommended surface for most code. | -| **C API** | `LMDB.mdb_*`, `LMDB.MDB_*` | Raw `ccall` bindings + status-code constants. For power users integrating with custom data layouts or shaving allocations on hot paths. | - -The C API also includes `MDBValue`, `MDBArg`, and the -[`MDBValueIO`](@ref LMDB.MDBValueIO) extension point — an `IO` view -over `MDB_val` that lets custom value representations plug into all the -typed reads via `Base.read(io, T)`. - -The Usage section is organised in increasing order of complexity: start -with [Essentials](@ref) for a working example, then [Dictionary -interface](@ref) for the `LMDBDict` surface, and progress through -[Environments](@ref), [Transactions](@ref), [Databases](@ref), -[Cursors](@ref), and [Duplicate-sort databases](@ref) as you need them. -[Low-level bindings](@ref) covers the `ccall` surface for callers who need -to skip the wrappers. - -The API reference mirrors the same structure but lists every exported and -public docstring. +## Using LMDB.jl + +LMDB.jl exposes the same database through three surfaces: + +| Surface | What it offers | When to use | +|---------|----------------|-------------| +| **High-level interface** | `LMDBDict <: AbstractDict{K,V}` | When you want a persistent `Dict`. | +| **Julia wrappers** | `Environment`, `Transaction`, `DBI`, `Cursor` | When you want explicit transactions and cursors with Julia-shaped wrappers. | +| **C API** | `LMDB.mdb_*`, `LMDB.MDB_*` | Raw `ccall` bindings and status-code constants, for custom data layouts or when you want to skip allocations on hot paths. | + +`MDBValue`, `MDBArg`, and the [`MDBValueIO`](@ref LMDB.MDBValueIO) +type sit between the C API and the Julia wrappers. `MDBValueIO` is an +`IO` view over `MDB_val`; defining `Base.read(io, T)` on it is how you +teach the typed reads about a custom value type. + +The Usage section starts simple and gets more involved: [Essentials](@ref) +has a working example, [Dictionary interface](@ref) covers `LMDBDict`, +and [Environments](@ref), [Transactions](@ref), [Databases](@ref), +[Cursors](@ref), and [Duplicate-sort databases](@ref) cover the wrappers. +[Low-level bindings](@ref) is the raw `ccall` surface. + +The API reference follows the same structure and lists every exported +and public docstring. ## A 5-line example diff --git a/docs/src/lib/dict.md b/docs/src/lib/dict.md index adc6901..8ad44af 100644 --- a/docs/src/lib/dict.md +++ b/docs/src/lib/dict.md @@ -1,6 +1,6 @@ # [Dictionary interface](@id API-Dict) -The high-level abstraction: a single `AbstractDict{K,V}` over an LMDB environment. +The high-level interface: a single `AbstractDict{K,V}` over an LMDB environment. ```@meta CurrentModule = LMDB @@ -12,19 +12,20 @@ CurrentModule = LMDB LMDBDict ``` -`LMDBDict` is `<: AbstractDict{K,V}`, so it transparently picks up -`Base`'s generic methods on top of the lookup/mutation primitives: - -- **Reads.** `getindex`, `haskey`, `get`, `get!`, `length`, `isempty`, - `iterate`, `keys`, `values`, `pairs`. All defined on `Base`-side - signatures and dispatched into LMDB; `getindex` and `pop!` throw - `KeyError` on miss to match `Base.Dict`. -- **Writes.** `setindex!`, `delete!`, `pop!`, `empty!`. `delete!` - silently no-ops on a missing key (matching `Base.delete!`'s "if any" - contract). -- **Generic.** Everything `AbstractDict` derives — `merge!`, `merge`, - `mergewith!`, `filter!`, `filter`, `==`, `isequal`, `hash`, - `in(::Pair, d)`, `copy(d)` — applies for free. +`LMDBDict <: AbstractDict{K,V}`, so it picks up `Base`'s generic +methods on top of the lookup/mutation primitives. + +Reads (`getindex`, `haskey`, `get`, `get!`, `length`, `isempty`, +`iterate`, `keys`, `values`, `pairs`) are dispatched into LMDB. +`getindex` and `pop!` throw `KeyError` on miss to match `Base.Dict`. + +Writes (`setindex!`, `delete!`, `pop!`, `empty!`) likewise. `delete!` +silently no-ops on a missing key, matching `Base.delete!`'s "if any" +contract. + +Everything `AbstractDict` derives — `merge!`, `merge`, `mergewith!`, +`filter!`, `filter`, `==`, `isequal`, `hash`, `in(::Pair, d)`, +`copy(d)` — comes along for free. `LMDBDict` iterates in lexicographic key order, which is stronger than `Base.Dict`'s no-order promise. diff --git a/docs/src/lib/environments.md b/docs/src/lib/environments.md index a7addb8..e8189f2 100644 --- a/docs/src/lib/environments.md +++ b/docs/src/lib/environments.md @@ -4,9 +4,9 @@ CurrentModule = LMDB ``` -An `Environment` wraps an LMDB env handle (`Ptr{MDB_env}`). It is the -top of the handle hierarchy — every transaction, database, and cursor -ultimately lives inside one env. +An `Environment` wraps an LMDB env handle (`Ptr{MDB_env}`). It sits at +the top of the handle hierarchy: every transaction, database, and +cursor lives inside one env. ## Construction diff --git a/docs/src/lib/errors.md b/docs/src/lib/errors.md index 3ab0960..fa80507 100644 --- a/docs/src/lib/errors.md +++ b/docs/src/lib/errors.md @@ -46,15 +46,18 @@ The rest (`MDB_PAGE_NOTFOUND`, `MDB_CORRUPTED`, `MDB_PANIC`, `MDB_MAP_RESIZED`, `MDB_INCOMPATIBLE`, `MDB_BAD_RSLOT`, `MDB_BAD_TXN`, `MDB_BAD_VALSIZE`, `MDB_BAD_DBI`) live under the `LMDB.` prefix. -## Where errors come from at each layer - -- **C API.** Bindings wrapped by `@checked` auto-throw `LMDBError`; - the `unchecked_*` companion returns the raw `Cint` so the caller can - branch on `MDB_NOTFOUND`/`MDB_KEYEXIST`/etc. -- **Julia API.** Handle methods that wrap status-returning bindings let - `LMDBError` propagate. `tryget` and `get(..., default)` swallow - `MDB_NOTFOUND` and return `nothing`/`default`. `delete!(txn, dbi, key)` - likewise swallows `MDB_NOTFOUND` and returns `false`. -- **High-level abstractions.** Missing keys produce `KeyError` - (matching `Base.Dict`). `pop!(d)` on an empty dict throws - `ArgumentError`. Other LMDB errors propagate as `LMDBError`. +## Where errors come from at each surface + +At the C API, bindings wrapped by `@checked` auto-throw `LMDBError`; +the `unchecked_*` companion returns the raw `Cint` so the caller can +branch on `MDB_NOTFOUND`/`MDB_KEYEXIST` and friends. + +At the Julia wrappers, handle methods that wrap status-returning +bindings let `LMDBError` propagate. `tryget` and `get(..., default)` +swallow `MDB_NOTFOUND` and return `nothing`/`default`; +`delete!(txn, dbi, key)` likewise swallows `MDB_NOTFOUND` and returns +`false`. + +At the high-level interface, missing keys produce `KeyError` (matching +`Base.Dict`), and `pop!(d)` on an empty dict throws `ArgumentError`. +Other LMDB errors propagate as `LMDBError`. diff --git a/docs/src/lib/lowlevel.md b/docs/src/lib/lowlevel.md index afbaded..112c47d 100644 --- a/docs/src/lib/lowlevel.md +++ b/docs/src/lib/lowlevel.md @@ -12,15 +12,15 @@ constants. Everything is public-but-unexported: refer to it as `LMDB.mdb_env_create`, `LMDB.MDB_NOTLS`, `LMDB.MDB_val`. The bindings in this section auto-throw -on a non-zero status; for callers that need to inspect the raw status -code, an `unchecked_*` companion is paired with each. +on a non-zero status. Each one is paired with an `unchecked_*` companion +for callers that need to inspect the raw status code. ## The auto-throw convention Every status-returning binding in `liblmdb.jl` is paired with an `unchecked_*` companion at definition time. Use the bare name when any error should propagate (the common case); use `unchecked_*` when you -need to inspect the raw `Cint` yourself — e.g. distinguishing +need to inspect the raw `Cint` yourself, for example to distinguish `MDB_NOTFOUND` from a real error: ```julia @@ -34,15 +34,15 @@ Bindings that return non-status data (`mdb_strerror`, `mdb_version`, `mdb_txn_id`, `mdb_cmp`, `mdb_dcmp`, `mdb_env_get_maxkeysize`, `mdb_env_get_userctx`, `mdb_cursor_txn`, `mdb_cursor_dbi`) and `Cvoid`-returning ones (`mdb_env_close`, `mdb_dbi_close`, -`mdb_txn_abort`, `mdb_txn_reset`, `mdb_cursor_close`) are left bare — +`mdb_txn_abort`, `mdb_txn_reset`, `mdb_cursor_close`) are left bare; there is nothing to check. ## Customisation point: `MDBValueIO` `tryget` / `get` / `key` / `value` / `item` / typed `walk` / `pop!` / -`replace!` all funnel through `read(::MDBValueIO, T)` to decode an +`replace!` all go through `read(::MDBValueIO, T)` to decode an `MDB_val` into a Julia value. Define a `Base.read` method on -`MDBValueIO` to plug in a custom representation — see [Cursors](@ref) +`MDBValueIO` to plug in a custom representation; see [Cursors](@ref) for a worked example. ```@docs diff --git a/docs/src/man/cursors.md b/docs/src/man/cursors.md index 6d148ef..df95220 100644 --- a/docs/src/man/cursors.md +++ b/docs/src/man/cursors.md @@ -5,7 +5,7 @@ CurrentModule = LMDB ``` A `Cursor` is a positioned iterator over a `DBI`. Use it for ordered -scans, range queries, and any time you want to amortise the per-lookup +scans, range queries, or when you want to amortise the per-lookup overhead of `mdb_get` across many keys. ## Opening a cursor @@ -109,13 +109,13 @@ Pass `from = key` to start at the smallest entry `≥ key` (i.e. The callback can return `false` to stop iteration; any other return (including `nothing`) continues. -The untyped form is the right tool when you want to inspect raw byte -sizes, copy slices, or feed a custom decoder — the data pointers are -into LMDB's mmap and are valid only inside the callback (and only for -the surrounding txn). The typed form is the iteration analogue of +Use the untyped form when you want to inspect raw byte sizes, copy +slices, or feed a custom decoder; the data pointers are into LMDB's +mmap and are valid only inside the callback (and only for the +surrounding txn). The typed form is the iteration analogue of `tryget(..., T)` and works for any `T` for which `Base.read(io::IO, ::Type{T})` (or `Base.read(io::LMDB.MDBValueIO, ::Type{T})`) is -defined (see [Custom value decoding](@ref)). +defined; see [Custom value decoding](@ref). ## Cursor mutation @@ -140,8 +140,8 @@ through `Base.read(io::IO, ::Type{T})` against an primitive numeric types (`Int8`/…/`Float64`, `Bool`, `Char`, `Ptr`), `String`, and (added by this package) `Vector{E}` for any bitstype `E`. -For everything else — including `isbitstype` structs and framed -values — define a single `Base.read` method on the abstract `IO`: +For everything else, including `isbitstype` structs and framed +values, define a single `Base.read` method on the abstract `IO`: ```julia struct PrefixedBlob end @@ -159,13 +159,13 @@ walk(cur, String, PrefixedBlob) do k, blob end ``` -`MDBValueIO <: IO` so all the usual `Base` IO primitives — `position`, -`seek`, `skip`, `read(io, n::Integer)`, `read(io, T)`, `read!(io, A)`, -`bytesavailable`, `eof` — work as expected. This makes structured -framed-value decoders read like any other Julia binary parser, and is -the analogue of heed's `BytesDecode<'txn>` trait — but expressed -through Julia's existing IO extension point rather than a bespoke -trait, so the same decoder works against any byte source. +`MDBValueIO <: IO`, so all the usual `Base` IO primitives work on it: +`position`, `seek`, `skip`, `read(io, n::Integer)`, `read(io, T)`, +`read!(io, A)`, `bytesavailable`, `eof`. Structured framed-value +decoders end up reading like any other Julia binary parser, and the +same decoder works against any byte source. This is the analogue of +heed's `BytesDecode<'txn>` trait, expressed through Julia's existing IO +extension point rather than a bespoke trait. ## Reset and renew diff --git a/docs/src/man/databases.md b/docs/src/man/databases.md index 1743f2e..399dd8a 100644 --- a/docs/src/man/databases.md +++ b/docs/src/man/databases.md @@ -62,8 +62,8 @@ tryget(txn, dbi, key, Vector{Float32}) # → Union{Vector{Float32}, Nothing tryget(txn, dbi, key, UInt64) # → Union{UInt64, Nothing} ``` -`tryget` is the workhorse: it inspects the raw status code and -swallows `MDB_NOTFOUND` cheaply, without throwing. +`tryget` is the cheap one: it inspects the raw status code and swallows +`MDB_NOTFOUND` without throwing. ## Writes @@ -95,8 +95,8 @@ start(env) do txn end ``` -`replace!` and `pop!` perform the read-modify pair inside the same -transaction — no time-of-check / time-of-use gap. +`replace!` and `pop!` do the read-modify pair inside the same +transaction, so there is no time-of-check / time-of-use gap. ## `put_reserved!` — write directly into the mmap @@ -111,12 +111,12 @@ put_reserved!(txn, dbi, key, sizeof(header) + length(payload)) do buf end ``` -`buf` is an `unsafe_wrap` over the LMDB write buffer; it is **only -valid inside the callback** (and only inside the surrounding write -txn). Don't escape it. +`buf` is an `unsafe_wrap` over the LMDB write buffer. It is only valid +inside the callback, and only inside the surrounding write txn; don't +escape it. -`put_reserved!` is the equivalent of heed's `Database::put_reserved`. -It is incompatible with DUPSORT. +`put_reserved!` is the equivalent of heed's `Database::put_reserved`, +and is incompatible with DUPSORT. ## Stats diff --git a/docs/src/man/dict.md b/docs/src/man/dict.md index 0fbad0b..8ba4ce6 100644 --- a/docs/src/man/dict.md +++ b/docs/src/man/dict.md @@ -5,8 +5,8 @@ CurrentModule = LMDB ``` `LMDBDict{K,V}` is a persistent `AbstractDict{K,V}` backed by a single -LMDB environment + the default DBI. It is the simplest way to use LMDB -from Julia: open it, treat it like a `Dict`, close it. +LMDB environment plus the default DBI. Open it, treat it like a `Dict`, +close it. ## Construction @@ -132,14 +132,13 @@ audits without `stat`. ## When to drop down -Reach for the explicit Julia API (next chapters) when: +Reach for the explicit Julia wrappers (next chapters) when: -- you need **multi-key atomicity**: a single transaction grouping more - than one `put!`/`delete!`, -- you want to **stream** without eagerly building a `Vector{Pair{K,V}}` +- you need a single transaction grouping more than one `put!`/`delete!`, +- you want to stream rather than eagerly build a `Vector{Pair{K,V}}` (see [Cursors](@ref) and `walk`), -- you need **multiple named databases** in one env (LMDBDict only - exposes the default unnamed DB), -- you're using `MDB_DUPSORT` (multiple values per key — see +- you want multiple named databases in one env (`LMDBDict` only exposes + the default unnamed DB), +- you're using `MDB_DUPSORT` for multiple values per key (see [Duplicate-sort databases](@ref)), - or you want zero-copy reads against the mmap. diff --git a/docs/src/man/dupsort.md b/docs/src/man/dupsort.md index 86ce0e9..2eb5e6e 100644 --- a/docs/src/man/dupsort.md +++ b/docs/src/man/dupsort.md @@ -5,10 +5,10 @@ CurrentModule = LMDB ``` By default each key in an LMDB database has a single value. Opening a -DB with `MDB_DUPSORT` instead allows **multiple values per key**, kept -in sorted order. This is LMDB's answer to inverted indexes, -many-to-many edges, time-series buckets, and any "key → set of values" -pattern. +DB with `MDB_DUPSORT` instead allows multiple values per key, kept in +sorted order. This is what LMDB offers for inverted indexes, +many-to-many edges, time-series buckets, and other "key → set of +values" patterns. ```julia env = Environment("/tmp/edges"; mapsize = 1 << 30, maxdbs = 1) @@ -39,9 +39,9 @@ DUPSORT wins when: - you want range queries within a key's values (`seek_range!` style). -It loses if you need **fast aggregate reads of every value at a key** — -that's where `MDB_DUPFIXED` (fixed-size duplicates) shines because -the values are stored contiguously and can be returned in batches. +It loses if you need fast aggregate reads of every value at a key. For +that case, use `MDB_DUPFIXED` (fixed-size duplicates), which stores the +values contiguously and returns them in batches. ## Navigation diff --git a/docs/src/man/environments.md b/docs/src/man/environments.md index 4211c85..9d0e466 100644 --- a/docs/src/man/environments.md +++ b/docs/src/man/environments.md @@ -10,9 +10,8 @@ database handle, and cursor lives inside one env. ## Creating and opening -The simplest path is the one-call constructor — it `create`s the -handle, applies optional configuration, and `open`s the directory in a -single call: +The one-call constructor `create`s the handle, applies any configuration, +and `open`s the directory in one go: ```julia env = Environment("/tmp/mydb"; mapsize = 1 << 30, # 1 GiB virtual map @@ -24,7 +23,7 @@ env = Environment("/tmp/mydb"; mapsize = 1 << 30, # 1 GiB virtual map If anything fails between `create` and a successful `open`, the partially constructed env is closed before rethrowing. -The split form is also available, mirroring the LMDB C API: +The split form mirrors the LMDB C API: ```julia env = create() @@ -73,15 +72,14 @@ MDB_RDONLY)` on an open env will return `EINVAL`. ## Sizing the map -The `mapsize` is a *virtual* limit on the env's address space, not the -on-disk size. A typical pattern is to pick a generous power of two -(say, 1 GiB or 8 GiB) up front; the on-disk file grows incrementally as -data is written. +`mapsize` is a *virtual* limit on the env's address space, not the +on-disk size. Pick a generous power of two (say, 1 GiB or 8 GiB) up +front; the on-disk file grows incrementally as data is written. If a write txn would exceed `mapsize`, LMDB returns `MDB_MAP_FULL` -(catchable via [`is_map_full(::LMDBError)`](@ref is_map_full)). The -remedy is to close the env, raise `mapsize`, and reopen — no rewrite -of the database is needed. +(catchable via [`is_map_full(::LMDBError)`](@ref is_map_full)). To +recover, close the env, raise `mapsize`, and reopen. The database +itself does not need rewriting. ## Inspection @@ -107,8 +105,8 @@ and the destination is approximately the size of the live data set: copy(env, "/backup/mydb-snapshot"; compact = true) ``` -There is also a file-descriptor variant — `copy(env, fd)` — for -streaming the snapshot to a pipe or socket. +A file-descriptor variant, `copy(env, fd)`, streams the snapshot to a +pipe or socket. ## Reader management diff --git a/docs/src/man/essentials.md b/docs/src/man/essentials.md index 48a80ce..b662476 100644 --- a/docs/src/man/essentials.md +++ b/docs/src/man/essentials.md @@ -38,47 +38,33 @@ close(d) Behind the scenes this opens an `Environment` with `MDB_NOTLS` (so multiple read transactions can coexist on a single thread) and a single -default `DBI`. Type conversions happen automatically — anything the -`MDBValue` constructor accepts (`String`, `Vector{T}` of bitstype `T`, -or any bitstype scalar) can be stored. - -## The three layers - -LMDB.jl is organised in layers. The same database can be accessed at any -of them, depending on what you need: - -``` -┌──────────────────────────────────────────────────────────────────┐ -│ High-level abstractions — LMDBDict <: AbstractDict{K,V} │ -│ "I want a persistent Dict." │ -├──────────────────────────────────────────────────────────────────┤ -│ Julia API — Environment, Transaction, DBI, Cursor │ -│ Julian wrappers with finalizers and parent refs. │ -│ The recommended surface for most code. │ -├──────────────────────────────────────────────────────────────────┤ -│ C API — mdb_*, MDB_*, unchecked_mdb_* │ -│ @ccall bindings; status-returning ones auto-throw │ -│ and have `unchecked_*` companions. `MDBValue`, │ -│ `MDBArg`, and `MDBValueIO` glue Julia values to │ -│ `Ptr{MDB_val}` and let custom decoders plug in via │ -│ `Base.read(io, T)`. │ -└──────────────────────────────────────────────────────────────────┘ -``` - -The recommended progression is: - -1. Start with the [Dictionary interface](@ref) until you need - transactional grouping or zero-copy reads. -2. Drop to [Environments](@ref) → [Transactions](@ref) → - [Databases](@ref) → [Cursors](@ref) for explicit lifetimes - and fine-grained control. -3. Reach for the [Low-level bindings](@ref) only when integrating - with a custom data layout or when the wrappers introduce overhead - you can't afford. +default `DBI`. Type conversion happens automatically for anything the +`MDBValue` constructor accepts: `String`, `Vector{T}` of bitstype `T`, +or any bitstype scalar. + +## Picking a surface + +LMDB.jl exposes the same database three ways, in increasing order of +control: + +- The **high-level interface** ([`LMDBDict`](@ref)) is the + `AbstractDict{K,V}` surface — start here unless you need + transactional grouping or zero-copy reads. +- The **Julia wrappers** (`Environment`, `Transaction`, `DBI`, + `Cursor`) give you explicit lifetimes and fine-grained control with + finalizers, parent refs, and `do`-block forms. Drop down to these + via [Environments](@ref) → [Transactions](@ref) → + [Databases](@ref) → [Cursors](@ref). +- The **C API** (`mdb_*`, `MDB_*`, `unchecked_mdb_*`) is the raw + `@ccall` surface. `MDBValue`, `MDBArg`, and `MDBValueIO` glue Julia + values to `Ptr{MDB_val}` and let custom decoders plug in via + `Base.read(io, T)`. Reach for the [Low-level bindings](@ref) only + when integrating with a custom data layout or when the wrappers + introduce overhead you can't afford. ## Resource lifecycle -Each Julia-API handle type wraps a raw LMDB pointer in a `mutable struct` +Each wrapper handle is a `mutable struct` around a raw LMDB pointer, with a finalizer: | handle | finalizer | parent ref | @@ -89,14 +75,13 @@ with a finalizer: | `LMDBDict` | `close` env + dbi | – | Parent references pin the lifetime: a `Cursor` keeps its `Transaction` -alive, which keeps its `Environment` alive. All cleanup operations -(`close`, `commit`, `abort`) are idempotent — calling them twice (or -on a never-opened handle) is a silent no-op. This means abandoned write -txns (e.g. from a `for … break` over an `LMDBDict`, or any error path) -are eventually reclaimed when GC runs. +alive, which keeps its `Environment` alive. `close`, `commit`, and +`abort` are idempotent: calling them twice, or on a handle that was +never opened, is a silent no-op. So an abandoned write txn — say, from +a `for … break` over an `LMDBDict`, or any error path — gets reclaimed +when GC runs. -For most call sites the do-block constructors are the simplest correct -shape: +The do-block constructors are usually what you want: ```julia environment("/tmp/mydb"; flags = MDB_NOTLS) do env @@ -125,6 +110,6 @@ Common branches have helpers (`is_notfound`, `is_keyexist`, `is_map_full`); rarer codes can be matched against `LMDB.MDB_*` constants directly. See [Errors](@ref API-Errors) for the full list. -For the dominant "missing key" case, prefer the no-throw paths: +For the usual "missing key" case, prefer the no-throw paths: [`tryget(txn, dbi, key, T)`](@ref tryget) returns `nothing` on miss, and `get(txn, dbi, key, T, default)` falls back to `default`. diff --git a/docs/src/man/lowlevel.md b/docs/src/man/lowlevel.md index 7e87898..cc9315a 100644 --- a/docs/src/man/lowlevel.md +++ b/docs/src/man/lowlevel.md @@ -6,12 +6,12 @@ CurrentModule = LMDB The C API is the raw `ccall` interface to `liblmdb`. It is public-but-unexported: refer to it as `LMDB.mdb_env_create`, -`LMDB.MDB_NOTLS`, `LMDB.MDB_val`. Use this layer when you need to +`LMDB.MDB_NOTLS`, `LMDB.MDB_val`. Reach for it when you need to integrate with a custom data layout, branch on a status code that the -Julia API doesn't surface, or skip allocations on a hot path. +Julia wrappers don't surface, or skip allocations on a hot path. For the full inventory, see [the API reference](@ref API-LowLevel). What -follows is a tour of how the layer is shaped. +follows is a tour of how the surface is shaped. ## The auto-throwing convention @@ -46,14 +46,14 @@ Bindings that don't return a status (`mdb_strerror`, `mdb_version`, ## ccall glue: passing values to `Ptr{MDB_val}` LMDB exchanges keys and values through a `Ptr{MDB_val}` argument: a -two-field struct of `(size, data_ptr)` plus an out-pointer for the +two-field struct of `(size, data_ptr)`, plus an out-pointer for the ccall to fill in. LMDB.jl ships `Base.cconvert` overloads on -`Ptr{MDB_val}` that route any of `String`, `AbstractArray` (with -bitstype element type), `Base.RefValue` over a bitstype, any bitstype -scalar, or a pre-built `Ref{MDB_val}` (used as an out-param) into a -self-rooted argument that `ccall`'s automatic `GC.@preserve` keeps alive -across the call. Callers never need to write `Ref(...)` or -`GC.@preserve` for input arguments. +`Ptr{MDB_val}` for `String`, `AbstractArray` (with bitstype element +type), `Base.RefValue` over a bitstype, any bitstype scalar, and a +pre-built `Ref{MDB_val}` (used as an out-param). Each routes through a +self-rooted argument that `ccall`'s automatic `GC.@preserve` keeps +alive across the call, so callers don't have to write `Ref(...)` or +`GC.@preserve` for input arguments themselves. ```julia import LMDB @@ -93,9 +93,9 @@ The package ships these defaults: | `Vector{E}` for bitstype `E` | one alloc + `unsafe_copyto!`; the buffer is Julia-owned | | any bitstype scalar `T` | one `unsafe_load` of `sizeof(T)` bytes — zero allocations | -Custom representations are added by overloading `Base.read` on the -abstract `IO` (the idiomatic Julia form — keeps the decoder portable -to other byte sources): +Add custom representations by overloading `Base.read` on the abstract +`IO`. This is the idiomatic Julia form and keeps the decoder portable +to other byte sources: ```julia struct AtimedBlob end @@ -115,31 +115,31 @@ Base.read(io::IO, ::Type{T}) = read!(io, Ref{T}())[] ``` This is the analogue of heed's `BytesDecode<'txn>` trait. Every typed -read in the Julia API — `tryget`, `get`, `key`, `value`, `item`, typed -`walk`, `pop!`, `replace!` — funnels through `read(::MDBValueIO, T)`, -so a single method opt-in is enough to make a custom representation -usable across the package. Because `MDBValueIO <: IO`, all the standard -`Base` IO primitives (`position`, `seek`, `skip`, `read(io)`, -`read(io, n::Integer)`, `read!(io, A)`, `bytesavailable`, `eof`) work -out of the box, which makes structured framed-value decoders read -exactly like any other Julia parser. +read in the Julia wrappers (`tryget`, `get`, `key`, `value`, `item`, +typed `walk`, `pop!`, `replace!`) goes through `read(::MDBValueIO, T)`, +so one method definition makes a custom representation usable across +the package. Because `MDBValueIO <: IO`, the standard `Base` IO +primitives (`position`, `seek`, `skip`, `read(io)`, `read(io, +n::Integer)`, `read!(io, A)`, `bytesavailable`, `eof`) work out of the +box, so structured framed-value decoders read exactly like any other +Julia parser. ## Memory ownership rules - The `mv_data` pointer of an `MDB_val` produced by a *read* is into - LMDB's mmap. It is **valid only for the producing transaction's - lifetime** — copy out anything you want to retain past commit. - The default `Vector{E}` and `String` `read(::MDBValueIO, T)` methods + LMDB's mmap. It is valid only for the producing transaction's + lifetime; copy out anything you want to retain past commit. The + default `Vector{E}` and `String` `read(::MDBValueIO, T)` methods always copy; custom decoders are responsible for doing the same. - The `mv_data` pointer of an `MDB_val` produced by a `MDB_RESERVE` - *write* points into the LMDB write buffer and is **valid only inside - the surrounding write transaction**. [`put_reserved!`](@ref) wraps + *write* points into the LMDB write buffer and is valid only inside + the surrounding write transaction. [`put_reserved!`](@ref) wraps this; don't escape its `buf` argument. ## Unwrapped LMDB features A few LMDB features are reachable only through the C API because the -Julia API deliberately doesn't include them: +Julia wrappers deliberately don't include them: - **Custom comparators.** `LMDB.mdb_set_compare` / `LMDB.mdb_set_dupsort` accept a `MDB_cmp_func` callback. Use diff --git a/docs/src/man/transactions.md b/docs/src/man/transactions.md index 901e08b..3c93e5c 100644 --- a/docs/src/man/transactions.md +++ b/docs/src/man/transactions.md @@ -18,8 +18,7 @@ txn = start(env; flags = MDB_RDONLY) # read-only LMDB can hold one writer plus an unlimited number of readers concurrently. Read txns do not block writers and vice versa. -The do-block form is the recommended shape — it commits on normal -return and aborts on throw: +The do-block form commits on normal return and aborts on throw: ```julia result = start(env) do txn @@ -33,8 +32,8 @@ end # commits if no throw ## Commit / abort `commit(txn)` writes the txn's modifications to disk and frees the -handle. `abort(txn)` discards them. Both are idempotent — calling them -twice (or on a never-started txn) is a silent no-op. `Transaction`'s +handle; `abort(txn)` discards them. Both are idempotent: calling them +twice, or on a never-started txn, is a silent no-op. `Transaction`'s finalizer calls `abort`, so an abandoned write txn eventually releases LMDB's exclusive write mutex. @@ -44,9 +43,9 @@ undefined behaviour. ## Read-only transactions -Read-only txns are cheap to start and stop, but for tight loops the +Read-only txns are cheap to start and stop, but in a tight loop the [`reset`](@ref Base.reset(::LMDB.Transaction)) / [`renew`](@ref renew) -pair is even cheaper: +pair is cheaper still: ```julia txn = start(env; flags = MDB_RDONLY) @@ -63,9 +62,9 @@ end abort(txn) ``` -`reset` is only valid on read-only txns; `renew` fetches a new snapshot -of the database. Without `renew`, the parked txn would not see writes -committed in the meantime. +`reset` is only valid on read-only txns. `renew` fetches a fresh +database snapshot; without it, the parked txn won't see writes that +landed in the meantime. ## Sub-transactions @@ -90,8 +89,8 @@ start(env) do parent end ``` -LMDB does not support nested *read-only* txns — pass a write txn as the -parent. +LMDB does not support nested *read-only* txns; the parent must be a +write txn. ## Reader slots @@ -101,9 +100,9 @@ high-concurrency read workloads, or call [`reader_check(env)`](@ref) to reap slots left behind by crashed processes. Aggressive `for … break` over an `LMDBDict` without GC pressure can -pile up read txns; the explicit -[`walk(f, cur)`](@ref API-Cur-walk) form inside an `open(txn) do …` -block is leak-free. +pile up read txns. If that's a concern, use +[`walk(f, cur)`](@ref API-Cur-walk) inside an explicit +`open(txn) do …` block instead. ## Picking flags diff --git a/res/wrap.jl b/res/wrap.jl index c1e35a3..fa2738f 100644 --- a/res/wrap.jl +++ b/res/wrap.jl @@ -9,8 +9,8 @@ using JuliaFormatter using LMDB_jll -# Tier-1 bindings whose return type is `Cint` but is **not** an LMDB status -# code. The `@checked` post-processor leaves these alone so callers don't get +# Bindings whose return type is `Cint` but is **not** an LMDB status code. +# The `@checked` post-processor leaves these alone so callers don't get # spurious throws. const UNCHECKED_CINT = ( "mdb_env_get_maxkeysize", # returns the maximum key size, not a status. diff --git a/src/LMDB.jl b/src/LMDB.jl index 46a0c64..aede100 100644 --- a/src/LMDB.jl +++ b/src/LMDB.jl @@ -23,25 +23,25 @@ export # commonly-needed write flags MDB_NOOVERWRITE, MDB_NODUPDATA, MDB_APPEND, MDB_RESERVE, - # Julia API — environment + # Julia wrappers — environment Environment, create, environment, sync, set!, unset!, info, stat, path, isopen, isflagset, reader_check, reader_list, - # Julia API — transaction + # Julia wrappers — transaction Transaction, start, abort, commit, reset, renew, - # Julia API — database (DBI) + # Julia wrappers — database (DBI) DBI, drop, get, put!, put_reserved!, delete!, tryget, replace!, - # Julia API — cursor + # Julia wrappers — cursor Cursor, count, transaction, database, seek!, seek_last!, seek_range!, next!, prev!, key, value, item, walk, seek_first_dup!, seek_last_dup!, next_dup!, prev_dup!, next_nodup!, prev_nodup!, - # High-level abstractions + # High-level interface LMDBDict # --------------------------------------------------------------------------- @@ -106,9 +106,9 @@ is_map_full # `unchecked_*` companion (returns the raw `Cint` for callers that need to # inspect it, e.g. branching on `MDB_NOTFOUND`). # -# Use as `LMDB.mdb_env_create`, `LMDB.MDB_NOTLS`, `LMDB.MDB_val`. Mostly -# relevant to power users; the Julia API (`Environment`, `Transaction`, …) is -# the recommended surface. +# Use as `LMDB.mdb_env_create`, `LMDB.MDB_NOTLS`, `LMDB.MDB_val`. The Julia +# wrappers (`Environment`, `Transaction`, …) wrap these and are what most +# callers should reach for. # --------------------------------------------------------------------------- include("checked.jl") @@ -123,15 +123,15 @@ is_keyexist(err::LMDBError) = err.code == MDB_KEYEXIST is_map_full(err::LMDBError) = err.code == MDB_MAP_FULL # --------------------------------------------------------------------------- -# ccall glue between the C API and the Julia API: `MDBValue`, `MDBArg`, and -# the `MDBValueIO <: IO` extension point used to plug in custom decoders via +# ccall glue between the C API and the Julia wrappers: `MDBValue`, `MDBArg`, +# and the `MDBValueIO <: IO` wrapper used to plug in custom decoders via # `Base.read(io, T)`. # --------------------------------------------------------------------------- include("common.jl") # --------------------------------------------------------------------------- -# Julia API — Julian wrappers around the raw bindings. +# Julia wrappers around the raw bindings. # --------------------------------------------------------------------------- include("env.jl") @@ -140,7 +140,7 @@ include("dbi.jl") include("cur.jl") # --------------------------------------------------------------------------- -# High-level abstractions — `LMDBDict`. +# High-level interface — `LMDBDict`. # --------------------------------------------------------------------------- include("dicts.jl") diff --git a/src/common.jl b/src/common.jl index d5b84c1..0bb54d0 100644 --- a/src/common.jl +++ b/src/common.jl @@ -73,11 +73,11 @@ passed to `tryget` / `get` / `key` / `value` / `item` / typed `walk` / the primitive numeric types (`Int8`/…/`Int128`, `Float16`/…/`Float64`, `Bool`, `Char`, `Ptr{T}`) plus `String`, all zero-allocation thanks to the `@inline` `unsafe_read` override below. The package adds two more -overloads — `Vector{E}` for any bitstype `E` and `UInt8` — that -consume the remaining buffer in a single copy. +overloads, `Vector{E}` for any bitstype `E` and `UInt8`, that consume +the remaining buffer in a single copy. -To plug in a custom representation — including bitstype structs that -Base's primitive reads don't cover — define a single `Base.read` +To plug in a custom representation (including bitstype structs that +Base's primitive reads don't cover), define a single `Base.read` method on your own type. Defining it on the abstract `IO` is the idiomatic Julia form and keeps the decoder portable to other byte sources: @@ -97,11 +97,10 @@ pattern: Base.read(io::IO, ::Type{T}) = read!(io, Ref{T}())[] This is the analogue of heed's `BytesDecode<'txn>` trait, expressed -through Julia's existing IO extension point rather than a bespoke -function. +through Julia's existing IO interface rather than a bespoke function. -The underlying buffer points into LMDB's mmap and is **only valid for -the producing transaction's lifetime** — copy out anything you want to +The underlying buffer points into LMDB's mmap and is only valid for +the producing transaction's lifetime; copy out anything you want to retain past commit/abort. The default `String` and `Vector{E}` reads both copy. """ diff --git a/src/cur.jl b/src/cur.jl index 9623c74..7809ba5 100644 --- a/src/cur.jl +++ b/src/cur.jl @@ -328,14 +328,14 @@ end walk(f, cur::Cursor, ::Type{K}, ::Type{V}=K; from = nothing) Typed overload of `walk` mirroring the `tryget(txn, dbi, key, T)` / -`key(cur, T)` / `seek!(cur, key, T)` shape used elsewhere in the Julia API. +`key(cur, T)` / `seek!(cur, key, T)` shape used elsewhere in the Julia wrappers. Decodes each key and value through `read(::MDBValueIO, K)` / `read(::MDBValueIO, V)` before passing them to `f(k::K, v::V)`. Same stop contract as the raw form: `f` returning `false` halts iteration. Define a custom `Base.read(io::LMDB.MDBValueIO, ::Type{T})` to control -what gets decoded — e.g. a `(atime, size)` tuple from a framed value, -or a zero-copy view. This is the iteration counterpart to +what gets decoded (for example, a `(atime, size)` tuple from a framed +value, or a zero-copy view). This is the iteration counterpart to `tryget(..., T)`. """ function walk(f, cur::Cursor, ::Type{K}, ::Type{V} = K; @@ -359,8 +359,8 @@ end Delete the entry the cursor is currently positioned at. Throws `LMDBError` if the cursor is not on a live entry (LMDB returns `EINVAL`, not `MDB_NOTFOUND`, so the Bool/idempotent shape used by -`delete!(txn, dbi, key)` doesn't apply here — position the cursor first -with `seek!`/`next!` if you need to recover from a missing entry. +`delete!(txn, dbi, key)` doesn't apply here; position the cursor first +with `seek!`/`next!` if you need to recover from a missing entry). After a successful delete, LMDB advances the cursor to the next entry. """ diff --git a/src/dbi.jl b/src/dbi.jl index 623d47f..52cb390 100644 --- a/src/dbi.jl +++ b/src/dbi.jl @@ -72,10 +72,10 @@ end Allocate `size` bytes of value space at `key` directly in LMDB's mmap'd write buffer, then call `f(buf::Vector{UInt8})` so the caller fills it in place. Equivalent to `put!` with the `MDB_RESERVE` flag, -without the intermediate `Vector{UInt8}` round-trip — useful when the -value's bytes can be produced directly into a destination buffer -(e.g. by `unsafe_store!` of a header followed by `copyto!` of a -payload). Heed's `Database::put_reserved` plays the same role. +without the intermediate `Vector{UInt8}` round-trip. Useful when the +value's bytes can be produced straight into a destination buffer (for +example, an `unsafe_store!` of a header followed by `copyto!` of a +payload). Equivalent to heed's `Database::put_reserved`. `buf` is an `unsafe_wrap` over the LMDB-allocated page; it is *only valid inside `f`* (and only inside the enclosing write txn). The @@ -103,8 +103,8 @@ the database. Returns `true` if an entry was removed, `false` if the key was not present. Other LMDB errors propagate as `LMDBError`. The Bool-return / no-throw-on-miss shape matches `Base.delete!`'s "if -any" contract and the dominant LMDB-binding convention (heed, py-lmdb, -lmdb-js, lmdbxx). +any" contract and the LMDB-binding convention shared by heed, py-lmdb, +lmdb-js, and lmdbxx. """ delete!(txn::Transaction, dbi::DBI, key) = _delete!(txn, dbi, key, MDBValue()) delete!(txn::Transaction, dbi::DBI, key, val) = _delete!(txn, dbi, key, val) diff --git a/src/dicts.jl b/src/dicts.jl index ac21a63..d571f2e 100644 --- a/src/dicts.jl +++ b/src/dicts.jl @@ -1,9 +1,10 @@ """ LMDBDict{K,V}(path; readonly, rdahead, mapsize, readers, dbs) -A persistent `AbstractDict{K,V}` backed by a single LMDB environment + -default DBI. The keys and values are encoded as raw bytes — `String`, -`Vector{T}` (where `T` is bitstype), or any bitstype scalar. +A persistent `AbstractDict{K,V}` backed by a single LMDB environment +plus its default DBI. Keys and values are encoded as raw bytes; +`String`, `Vector{T}` (where `T` is bitstype), or any bitstype scalar +all work. For prefix-scoped scans (e.g. hierarchical "directory" key schemes), see `LMDB.scan` / `LMDB.scan_keys` / `LMDB.scan_values` / `LMDB.list_dirs`. diff --git a/src/txn.jl b/src/txn.jl index 58a4bd2..13d5dd3 100644 --- a/src/txn.jl +++ b/src/txn.jl @@ -52,7 +52,7 @@ end """Abandon all the operations of the transaction instead of saving them. The transaction and its cursors must not be used after, because its handle is freed. -Idempotent — safe to call after a previous `commit`/`abort` or on a never-opened txn. +Idempotent: safe to call after a previous `commit`/`abort` or on a never-opened txn. """ function abort(txn::Transaction) txn.handle == C_NULL && return diff --git a/test/integration.jl b/test/integration.jl index e7c9ac6..d5ed6e3 100644 --- a/test/integration.jl +++ b/test/integration.jl @@ -26,10 +26,10 @@ end @testset "Integration" begin # Power-user pattern: open an env via the Environment kwargs ctor, run -# a write txn through the Julia API, then a read txn through a cursor walk -# using only the Julia API + raw MDB_val refs (the shape cuTile.DiskCache -# follows). Regression guard: ensures no future change breaks the -# `walk(...) do k_ref, v_ref` zero-copy idiom. +# a write txn through the Julia wrappers, then a read txn through a cursor +# walk using only the Julia wrappers + raw MDB_val refs (the shape +# cuTile.DiskCache follows). Regression guard: ensures no future change +# breaks the `walk(...) do k_ref, v_ref` zero-copy idiom. mktempdir() do dir env = Environment(dir; mapsize = 1 << 28, From b59e6873d8b3df079909f2362711297a47f5203c Mon Sep 17 00:00:00 2001 From: Tim Besard Date: Fri, 22 May 2026 10:18:35 +0200 Subject: [PATCH 7/8] Land CompatHelper separately. --- .github/workflows/CompatHelper.yml | 46 ------------------------------ 1 file changed, 46 deletions(-) delete mode 100644 .github/workflows/CompatHelper.yml diff --git a/.github/workflows/CompatHelper.yml b/.github/workflows/CompatHelper.yml deleted file mode 100644 index 866c0bc..0000000 --- a/.github/workflows/CompatHelper.yml +++ /dev/null @@ -1,46 +0,0 @@ -name: CompatHelper -on: - schedule: - - cron: 0 0 * * * - workflow_dispatch: - -permissions: - contents: write - pull-requests: write - -jobs: - CompatHelper: - runs-on: ubuntu-latest - steps: - - name: Check if Julia is already available in the PATH - id: julia_in_path - run: which julia - continue-on-error: true - - name: Install Julia, but only if it is not already available in the PATH - uses: julia-actions/setup-julia@v2 - with: - version: '1' - arch: ${{ runner.arch }} - if: steps.julia_in_path.outcome != 'success' - - name: "Add the General registry via Git" - run: | - import Pkg - ENV["JULIA_PKG_SERVER"] = "" - Pkg.Registry.add("General") - shell: julia --color=yes {0} - - name: "Install CompatHelper" - run: | - import Pkg - name = "CompatHelper" - uuid = "aa819f21-2bde-4658-8897-bab36330d9b7" - version = "3" - Pkg.add(; name, uuid, version) - shell: julia --color=yes {0} - - name: "Run CompatHelper" - run: | - import CompatHelper - CompatHelper.main() - shell: julia --color=yes {0} - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - COMPATHELPER_PRIV: ${{ secrets.DOCUMENTER_KEY }} From 4ba010c6f88a9474d28a08f8654d3df1803ed30d Mon Sep 17 00:00:00 2001 From: Tim Besard Date: Fri, 22 May 2026 10:40:50 +0200 Subject: [PATCH 8/8] Revert test cleanup (0e712a6) to unblock CI. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The test-cleanup commit's finalizer-via-GC.gc() tests in test/env.jl crash on Windows (EXCEPTION_ACCESS_VIOLATION in mdb_txn_end) and hang on Linux. Restoring the pre-cleanup test/ shape — module wrappers and all — keeps this docs/release-prep branch shippable. The cleanup can return as its own PR with the GC-dependent tests hardened or skipped on Windows. --- test/common.jl | 14 +-- test/cur.jl | 218 ++++++++++++++++----------------- test/dbi.jl | 282 ++++++++++++++++++++++--------------------- test/dupsort.jl | 112 +++++++++-------- test/env.jl | 285 ++++++++++++++++---------------------------- test/integration.jl | 182 ++++++++++++++-------------- test/liblmdb.jl | 102 ++++++++-------- 7 files changed, 558 insertions(+), 637 deletions(-) diff --git a/test/common.jl b/test/common.jl index 7f00247..0683c80 100644 --- a/test/common.jl +++ b/test/common.jl @@ -1,13 +1,9 @@ using LMDB using Test -@testset "common" begin + @test LMDB.version()[1] >= v"0.9.15" -@test LMDB.version()[1] >= v"0.9.15" - -# LMDBError -ex = LMDBError(Cint(0)) -@test_throws LMDBError throw(ex) -@test ex.code == 0 - -end # @testset "common" + # LMDBError + ex = LMDBError(Cint(0)) + @test_throws LMDBError throw(ex) + @test ex.code == 0 diff --git a/test/cur.jl b/test/cur.jl index 8a35b65..c4c2a84 100644 --- a/test/cur.jl +++ b/test/cur.jl @@ -1,14 +1,15 @@ -using LMDB -using Test +module LMDB_CUR + using LMDB + using Test -@testset "Cursor" begin + const dbname = "testdb" + key = 10 + val = "key value is " -key = 10 -val = "key value is " + # Create dir + mkdir(dbname) -# Procedural style + block style smoke test, exercising cursor put!/walk -# round-trip and the parent accessors. -mktempdir() do dbname + # Procedural style env = create() try open(env, dbname) @@ -51,121 +52,122 @@ mktempdir() do dbname end end end -end - -# Cursor positioning + walk primitives. -mktempdir() do dir - environment(dir) do env - start(env) do txn - open(txn) do dbi - LMDB.put!(txn, dbi, "a", "1") - LMDB.put!(txn, dbi, "b", "2") - LMDB.put!(txn, dbi, "c", "3") - - LMDB.open(txn, dbi) do cur - @test LMDB.seek!(cur, String) == "a" - @test LMDB.value(cur, String) == "1" - @test LMDB.key(cur, String) == "a" - @test LMDB.item(cur, String, String) == ("a" => "1") - - @test LMDB.next!(cur, String) == "b" - @test LMDB.value(cur, String) == "2" - - @test LMDB.seek_last!(cur, String) == "c" - @test LMDB.prev!(cur, String) == "b" - - @test LMDB.seek!(cur, "a", String) == "a" - @test LMDB.seek!(cur, "missing", String) === nothing - - @test LMDB.seek_range!(cur, "ab", String) == "b" - @test LMDB.seek_range!(cur, "z", String) === nothing - - # walk over everything - ks = String[] - LMDB.walk(cur) do k_ref, _ - push!(ks, read(LMDB.MDBValueIO(k_ref[]), String)) - end - @test ks == ["a", "b", "c"] - - # walk from a starting key - ks2 = String[] - LMDB.walk(cur; from="b") do k_ref, _ - push!(ks2, read(LMDB.MDBValueIO(k_ref[]), String)) - end - @test ks2 == ["b", "c"] - # walk from a key past the last entry — no callbacks. - ks3 = String[] - LMDB.walk(cur; from="z") do k_ref, _ - push!(ks3, read(LMDB.MDBValueIO(k_ref[]), String)) + # Remove db dir + rm(dbname, recursive=true) + + # Cursor positioning + walk primitives. + mktempdir() do dir + environment(dir) do env + start(env) do txn + open(txn) do dbi + LMDB.put!(txn, dbi, "a", "1") + LMDB.put!(txn, dbi, "b", "2") + LMDB.put!(txn, dbi, "c", "3") + + LMDB.open(txn, dbi) do cur + @test LMDB.seek!(cur, String) == "a" + @test LMDB.value(cur, String) == "1" + @test LMDB.key(cur, String) == "a" + @test LMDB.item(cur, String, String) == ("a" => "1") + + @test LMDB.next!(cur, String) == "b" + @test LMDB.value(cur, String) == "2" + + @test LMDB.seek_last!(cur, String) == "c" + @test LMDB.prev!(cur, String) == "b" + + @test LMDB.seek!(cur, "a", String) == "a" + @test LMDB.seek!(cur, "missing", String) === nothing + + @test LMDB.seek_range!(cur, "ab", String) == "b" + @test LMDB.seek_range!(cur, "z", String) === nothing + + # walk over everything + ks = String[] + LMDB.walk(cur) do k_ref, _ + push!(ks, read(LMDB.MDBValueIO(k_ref[]), String)) + end + @test ks == ["a", "b", "c"] + + # walk from a starting key + ks2 = String[] + LMDB.walk(cur; from="b") do k_ref, _ + push!(ks2, read(LMDB.MDBValueIO(k_ref[]), String)) + end + @test ks2 == ["b", "c"] + + # walk from a key past the last entry — no callbacks. + ks3 = String[] + LMDB.walk(cur; from="z") do k_ref, _ + push!(ks3, read(LMDB.MDBValueIO(k_ref[]), String)) + end + @test isempty(ks3) + + # typed walk: each ref decoded via read(MDBValueIO, K) + # / read(MDBValueIO, V). + kv = Pair{String, String}[] + LMDB.walk(cur, String, String) do k, v + push!(kv, k => v) + end + @test kv == ["a" => "1", "b" => "2", "c" => "3"] + + # typed walk respects the false-stops contract. + seen = Pair{String, String}[] + LMDB.walk(cur, String, String) do k, v + push!(seen, k => v) + k == "b" ? false : nothing + end + @test seen == ["a" => "1", "b" => "2"] end - @test isempty(ks3) - - # typed walk: each ref decoded via read(MDBValueIO, K) - # / read(MDBValueIO, V). - kv = Pair{String, String}[] - LMDB.walk(cur, String, String) do k, v - push!(kv, k => v) - end - @test kv == ["a" => "1", "b" => "2", "c" => "3"] - - # typed walk respects the false-stops contract. - seen = Pair{String, String}[] - LMDB.walk(cur, String, String) do k, v - push!(seen, k => v) - k == "b" ? false : nothing - end - @test seen == ["a" => "1", "b" => "2"] end end end end -end -# seek!/next! on an empty database returns nothing. -mktempdir() do dir - environment(dir) do env - start(env) do txn - open(txn) do dbi - LMDB.open(txn, dbi) do cur - @test LMDB.seek!(cur, String) === nothing - @test LMDB.seek_last!(cur, String) === nothing - @test LMDB.seek!(cur, "x", String) === nothing - @test LMDB.seek_range!(cur, "x", String) === nothing - - ks = String[] - LMDB.walk(cur) do k_ref, _ - push!(ks, read(LMDB.MDBValueIO(k_ref[]), String)) + # seek!/next! on an empty database returns nothing. + mktempdir() do dir + environment(dir) do env + start(env) do txn + open(txn) do dbi + LMDB.open(txn, dbi) do cur + @test LMDB.seek!(cur, String) === nothing + @test LMDB.seek_last!(cur, String) === nothing + @test LMDB.seek!(cur, "x", String) === nothing + @test LMDB.seek_range!(cur, "x", String) === nothing + + ks = String[] + LMDB.walk(cur) do k_ref, _ + push!(ks, read(LMDB.MDBValueIO(k_ref[]), String)) + end + @test isempty(ks) end - @test isempty(ks) end end end end -end -# Cursor.delete!: removes the entry the cursor is on; LMDB advances to -# the next entry. Throws on an unpositioned cursor (EINVAL), unlike the -# txn-based `delete!(txn, dbi, key)` which is Bool-returning on -# MDB_NOTFOUND. -mktempdir() do dir - environment(dir) do env - start(env) do txn - open(txn) do dbi - LMDB.put!(txn, dbi, "a", "1") - LMDB.put!(txn, dbi, "b", "2") - - LMDB.open(txn, dbi) do cur - @test LMDB.seek!(cur, "a", String) == "a" - LMDB.delete!(cur) # removes "a", cursor now on "b" - LMDB.delete!(cur) # removes "b" - @test_throws LMDBError LMDB.delete!(cur) # no live entry + # Cursor.delete!: removes the entry the cursor is on; LMDB advances to + # the next entry. Throws on an unpositioned cursor (EINVAL), unlike the + # txn-based `delete!(txn, dbi, key)` which is Bool-returning on + # MDB_NOTFOUND. + mktempdir() do dir + environment(dir) do env + start(env) do txn + open(txn) do dbi + LMDB.put!(txn, dbi, "a", "1") + LMDB.put!(txn, dbi, "b", "2") + + LMDB.open(txn, dbi) do cur + @test LMDB.seek!(cur, "a", String) == "a" + LMDB.delete!(cur) # removes "a", cursor now on "b" + LMDB.delete!(cur) # removes "b" + @test_throws LMDBError LMDB.delete!(cur) # no live entry + end + @test LMDB.tryget(txn, dbi, "a", String) === nothing + @test LMDB.tryget(txn, dbi, "b", String) === nothing end - @test LMDB.tryget(txn, dbi, "a", String) === nothing - @test LMDB.tryget(txn, dbi, "b", String) === nothing end end end end - -end # @testset "Cursor" diff --git a/test/dbi.jl b/test/dbi.jl index 0bdb87c..17c9631 100644 --- a/test/dbi.jl +++ b/test/dbi.jl @@ -1,162 +1,170 @@ -using LMDB -using Test +module LMDB_DBI + using LMDB + using Test -@testset "DBI" begin + const dbname = "testdb" + key = 10 + val = "key value is " -key = 10 -val = "key value is " - -# Procedural style + block style smoke test, exercising String, Int, and -# Vector{Int} round-trips through put!/get/delete!. -mktempdir() do dbname - env = create() + # Create dir + mkdir(dbname) try - open(env, dbname) - txn = start(env) - dbi = open(txn) - put!(txn, dbi, key+1, val*string(key+1)) - put!(txn, dbi, key, val*string(key)) - put!(txn, dbi, key+2, key+2) - put!(txn, dbi, key+3, [key, key+1, key+2]) - @test isopen(txn) - commit(txn) - @test !isopen(txn) - close(env, dbi) - @test !isopen(dbi) - finally - close(env) - end - @test !isopen(env) - - # Block style - create() do env - set!(env, LMDB.MDB_NOSYNC) - open(env, dbname) - start(env) do txn - open(txn, flags = Cuint(LMDB.MDB_REVERSEKEY)) do dbi - k = key - value = get(txn, dbi, k, String) - @test value == val*string(k) - delete!(txn, dbi, k) - k += 1 - value = get(txn, dbi, k, String) - @test value == val*string(k) - delete!(txn, dbi, k, value) - @test_throws LMDBError get(txn, dbi, k, String) - k += 1 - value = get(txn, dbi, k, Int) - @test value == k - k += 1 - value = get(txn, dbi, k, Vector{Int}) - @test value == [key, key+1, key+2] + + # Procedural style + env = create() + try + open(env, dbname) + txn = start(env) + dbi = open(txn) + put!(txn, dbi, key+1, val*string(key+1)) + put!(txn, dbi, key, val*string(key)) + put!(txn, dbi, key+2, key+2) + put!(txn, dbi, key+3, [key, key+1, key+2]) + @test isopen(txn) + commit(txn) + @test !isopen(txn) + close(env, dbi) + @test !isopen(dbi) + finally + close(env) + end + @test !isopen(env) + + # Block style + create() do env + set!(env, LMDB.MDB_NOSYNC) + open(env, dbname) + start(env) do txn + open(txn, flags = Cuint(LMDB.MDB_REVERSEKEY)) do dbi + k = key + value = get(txn, dbi, k, String) + println("Got value for key $(k): $(value)") + @test value == val*string(k) + delete!(txn, dbi, k) + k += 1 + value = get(txn, dbi, k, String) + println("Got value for key $(k): $(value)") + @test value == val*string(k) + delete!(txn, dbi, k, value) + @test_throws LMDBError get(txn, dbi, k, String) + k += 1 + value = get(txn, dbi, k, Int) + println("Got value for key $(k): $(value)") + @test value == k + k += 1 + value = get(txn, dbi, k, Vector{Int}) + println("Got value for key $(k): $(value)") + @test value == [key, key+1, key+2] + end end end + + finally + rm(dbname, recursive=true) end -end -# tryget / get-with-default / stat(txn, dbi) — fresh env so the entry -# count is deterministic. -mktempdir() do dir - environment(dir) do env - start(env) do txn - open(txn) do dbi - LMDB.put!(txn, dbi, "k1", "v1") - LMDB.put!(txn, dbi, "k2", "v2") - - @test LMDB.tryget(txn, dbi, "k1", String) == "v1" - @test LMDB.tryget(txn, dbi, "missing", String) === nothing - @test get(txn, dbi, "k2", String, "fallback") == "v2" - @test get(txn, dbi, "missing", String, "fallback") == "fallback" - - s = LMDB.stat(txn, dbi) - @test s isa NamedTuple - @test s.entries == 2 - @test s.psize > 0 + # tryget / get-with-default / stat(txn, dbi) — fresh env so the entry + # count is deterministic. + mktempdir() do dir + environment(dir) do env + start(env) do txn + open(txn) do dbi + LMDB.put!(txn, dbi, "k1", "v1") + LMDB.put!(txn, dbi, "k2", "v2") + + @test LMDB.tryget(txn, dbi, "k1", String) == "v1" + @test LMDB.tryget(txn, dbi, "missing", String) === nothing + @test get(txn, dbi, "k2", String, "fallback") == "v2" + @test get(txn, dbi, "missing", String, "fallback") == "fallback" + + s = LMDB.stat(txn, dbi) + @test s isa NamedTuple + @test s.entries == 2 + @test s.psize > 0 + end end end end -end -# put_reserved!: callback-style MDB_RESERVE write. -mktempdir() do dir - environment(dir) do env - start(env) do txn - open(txn) do dbi - # Write a 16-byte value where bytes 0..7 are a UInt64 - # header and bytes 8..15 are payload. The buffer hands - # back is the LMDB-allocated mmap page; we fill it - # in place — no intermediate Vector. - LMDB.put_reserved!(txn, dbi, "framed", 16) do buf - @test buf isa Vector{UInt8} - @test length(buf) == 16 - unsafe_store!(Ptr{UInt64}(pointer(buf)), - htol(UInt64(0xdeadbeef))) - for i in 1:8 - buf[8 + i] = UInt8(i) + # put_reserved!: callback-style MDB_RESERVE write. + mktempdir() do dir + environment(dir) do env + start(env) do txn + open(txn) do dbi + # Write a 16-byte value where bytes 0..7 are a UInt64 + # header and bytes 8..15 are payload. The buffer hands + # back is the LMDB-allocated mmap page; we fill it + # in place — no intermediate Vector. + LMDB.put_reserved!(txn, dbi, "framed", 16) do buf + @test buf isa Vector{UInt8} + @test length(buf) == 16 + unsafe_store!(Ptr{UInt64}(pointer(buf)), + htol(UInt64(0xdeadbeef))) + for i in 1:8 + buf[8 + i] = UInt8(i) + end end + raw = LMDB.tryget(txn, dbi, "framed", Vector{UInt8}) + @test length(raw) == 16 + @test ltoh(reinterpret(UInt64, raw[1:8])[1]) == + UInt64(0xdeadbeef) + @test raw[9:16] == UInt8[1, 2, 3, 4, 5, 6, 7, 8] + + # Return value: whatever the callback returns. + rv = LMDB.put_reserved!(txn, dbi, "rv", 4) do buf + fill!(buf, 0xab) + :sentinel + end + @test rv === :sentinel end - raw = LMDB.tryget(txn, dbi, "framed", Vector{UInt8}) - @test length(raw) == 16 - @test ltoh(reinterpret(UInt64, raw[1:8])[1]) == - UInt64(0xdeadbeef) - @test raw[9:16] == UInt8[1, 2, 3, 4, 5, 6, 7, 8] - - # Return value: whatever the callback returns. - rv = LMDB.put_reserved!(txn, dbi, "rv", 4) do buf - fill!(buf, 0xab) - :sentinel - end - @test rv === :sentinel end end end -end -# delete!: Bool-returning, idempotent on MDB_NOTFOUND. -mktempdir() do dir - environment(dir) do env - start(env) do txn - open(txn) do dbi - LMDB.put!(txn, dbi, "k1", "v1") - LMDB.put!(txn, dbi, "k2", "v2") - - # Present key → true, returns and entry is gone. - @test LMDB.delete!(txn, dbi, "k1") === true - @test LMDB.tryget(txn, dbi, "k1", String) === nothing - - # Missing key → false, no exception. - @test LMDB.delete!(txn, dbi, "ghost") === false - @test LMDB.delete!(txn, dbi, "k1") === false # already gone - - # Idempotent: a second delete on the same key is a no-op. - @test LMDB.delete!(txn, dbi, "k2") === true - @test LMDB.delete!(txn, dbi, "k2") === false + # delete!: Bool-returning, idempotent on MDB_NOTFOUND. + mktempdir() do dir + environment(dir) do env + start(env) do txn + open(txn) do dbi + LMDB.put!(txn, dbi, "k1", "v1") + LMDB.put!(txn, dbi, "k2", "v2") + + # Present key → true, returns and entry is gone. + @test LMDB.delete!(txn, dbi, "k1") === true + @test LMDB.tryget(txn, dbi, "k1", String) === nothing + + # Missing key → false, no exception. + @test LMDB.delete!(txn, dbi, "ghost") === false + @test LMDB.delete!(txn, dbi, "k1") === false # already gone + + # Idempotent: a second delete on the same key is a no-op. + @test LMDB.delete!(txn, dbi, "k2") === true + @test LMDB.delete!(txn, dbi, "k2") === false + end end end end -end -# replace! / pop! -mktempdir() do dir - environment(dir) do env - start(env) do txn - open(txn) do dbi - # replace! on a missing key returns nothing and creates the entry. - @test LMDB.replace!(txn, dbi, "k", "v1") === nothing - @test LMDB.tryget(txn, dbi, "k", String) == "v1" - - # replace! on an existing key returns the old value. - @test LMDB.replace!(txn, dbi, "k", "v2") == "v1" - @test LMDB.tryget(txn, dbi, "k", String) == "v2" - - # pop! returns the value and deletes. - @test LMDB.pop!(txn, dbi, "k", String) == "v2" - @test LMDB.tryget(txn, dbi, "k", String) === nothing - # pop! on a missing key returns nothing. - @test LMDB.pop!(txn, dbi, "k", String) === nothing + # replace! / pop! + mktempdir() do dir + environment(dir) do env + start(env) do txn + open(txn) do dbi + # replace! on a missing key returns nothing and creates the entry. + @test LMDB.replace!(txn, dbi, "k", "v1") === nothing + @test LMDB.tryget(txn, dbi, "k", String) == "v1" + + # replace! on an existing key returns the old value. + @test LMDB.replace!(txn, dbi, "k", "v2") == "v1" + @test LMDB.tryget(txn, dbi, "k", String) == "v2" + + # pop! returns the value and deletes. + @test LMDB.pop!(txn, dbi, "k", String) == "v2" + @test LMDB.tryget(txn, dbi, "k", String) === nothing + # pop! on a missing key returns nothing. + @test LMDB.pop!(txn, dbi, "k", String) === nothing + end end end end end - -end # @testset "DBI" diff --git a/test/dupsort.jl b/test/dupsort.jl index 2304a41..2ba2bb1 100644 --- a/test/dupsort.jl +++ b/test/dupsort.jl @@ -1,62 +1,60 @@ -using LMDB -using Test - -@testset "DupSort" begin - -# DupSort: a single key can hold multiple sorted values. Verify the -# dup-aware cursor ops navigate within and across keys correctly, and -# that delete!(txn, dbi, key, val) removes one specific dup. -mktempdir() do dir - environment(dir) do env - start(env) do txn - open(txn, flags = MDB_DUPSORT) do dbi - LMDB.put!(txn, dbi, "k1", "a") - LMDB.put!(txn, dbi, "k1", "b") - LMDB.put!(txn, dbi, "k1", "c") - LMDB.put!(txn, dbi, "k2", "x") - LMDB.put!(txn, dbi, "k2", "y") - - LMDB.open(txn, dbi) do cur - # Position at first entry; walk through k1's dups. - @test LMDB.seek!(cur, String) == "k1" - @test LMDB.value(cur, String) == "a" - @test LMDB.next_dup!(cur, String) == "b" - @test LMDB.next_dup!(cur, String) == "c" - @test LMDB.next_dup!(cur, String) === nothing # no more dups - - # Jump to next key, skipping any remaining dups (none here). - @test LMDB.next_nodup!(cur, String) == "k2" - @test LMDB.value(cur, String) == "x" - @test LMDB.next_dup!(cur, String) == "y" - - # Reset to first dup of current key. - @test LMDB.seek_first_dup!(cur, String) == "x" - @test LMDB.seek_last_dup!(cur, String) == "y" - @test LMDB.prev_dup!(cur, String) == "x" - - # prev_nodup! moves back to the last dup of the previous key. - @test LMDB.prev_nodup!(cur, String) == "k1" - @test LMDB.value(cur, String) == "c" - end - - # Count dups for k1 via cursor count(). - LMDB.open(txn, dbi) do cur - @test LMDB.seek!(cur, "k1", String) == "k1" - @test count(cur) == 3 - end - - # Dup-aware delete: delete!(txn, dbi, key, val) removes only - # that one duplicate. - LMDB.delete!(txn, dbi, "k1", "b") - LMDB.open(txn, dbi) do cur - @test LMDB.seek!(cur, "k1", String) == "k1" - @test LMDB.value(cur, String) == "a" - @test LMDB.next_dup!(cur, String) == "c" # "b" is gone - @test LMDB.next_dup!(cur, String) === nothing +module LMDB_DupSort + using LMDB + using Test + + # DupSort: a single key can hold multiple sorted values. Verify the + # dup-aware cursor ops navigate within and across keys correctly, and + # that delete!(txn, dbi, key, val) removes one specific dup. + mktempdir() do dir + environment(dir) do env + start(env) do txn + open(txn, flags = MDB_DUPSORT) do dbi + LMDB.put!(txn, dbi, "k1", "a") + LMDB.put!(txn, dbi, "k1", "b") + LMDB.put!(txn, dbi, "k1", "c") + LMDB.put!(txn, dbi, "k2", "x") + LMDB.put!(txn, dbi, "k2", "y") + + LMDB.open(txn, dbi) do cur + # Position at first entry; walk through k1's dups. + @test LMDB.seek!(cur, String) == "k1" + @test LMDB.value(cur, String) == "a" + @test LMDB.next_dup!(cur, String) == "b" + @test LMDB.next_dup!(cur, String) == "c" + @test LMDB.next_dup!(cur, String) === nothing # no more dups + + # Jump to next key, skipping any remaining dups (none here). + @test LMDB.next_nodup!(cur, String) == "k2" + @test LMDB.value(cur, String) == "x" + @test LMDB.next_dup!(cur, String) == "y" + + # Reset to first dup of current key. + @test LMDB.seek_first_dup!(cur, String) == "x" + @test LMDB.seek_last_dup!(cur, String) == "y" + @test LMDB.prev_dup!(cur, String) == "x" + + # prev_nodup! moves back to the last dup of the previous key. + @test LMDB.prev_nodup!(cur, String) == "k1" + @test LMDB.value(cur, String) == "c" + end + + # Count dups for k1 via cursor count(). + LMDB.open(txn, dbi) do cur + @test LMDB.seek!(cur, "k1", String) == "k1" + @test count(cur) == 3 + end + + # Dup-aware delete: delete!(txn, dbi, key, val) removes only + # that one duplicate. + LMDB.delete!(txn, dbi, "k1", "b") + LMDB.open(txn, dbi) do cur + @test LMDB.seek!(cur, "k1", String) == "k1" + @test LMDB.value(cur, String) == "a" + @test LMDB.next_dup!(cur, String) == "c" # "b" is gone + @test LMDB.next_dup!(cur, String) === nothing + end end end end end end - -end # @testset "DupSort" diff --git a/test/env.jl b/test/env.jl index 0696586..49a13ae 100644 --- a/test/env.jl +++ b/test/env.jl @@ -1,204 +1,125 @@ -using LMDB -using Test - -@testset "Environment" begin - -# Open environment -env = create() -@test env.handle != C_NULL -@test env[:Readers] == 126 -@test env[:KeySize] == 511 -@test env[:Flags] == 0 - -# Manipulate flags -@test !isflagset(env[:Flags], Cuint(LMDB.MDB_NOSYNC)) -set!(env, LMDB.MDB_NOSYNC) -@test isflagset(env[:Flags], Cuint(LMDB.MDB_NOSYNC)) -unset!(env, LMDB.MDB_NOSYNC) -@test !isflagset(env[:Flags], Cuint(LMDB.MDB_NOSYNC)) - -# Parameters -@test (env[:Readers] = 100) == 100 -@test (env[:MapSize] = 1000^2) == 1000^2 -@test (env[:DBs] = 10) == 10 -@test env[:Readers] == 100 - -# MapSize must accept values that don't fit in Cuint (#38, PR #37, #40). -big = Csize_t(8) * 1024^3 # 8 GiB -@test (env[:MapSize] = big) == big - -# Setting :Flags via setindex! used to fall through to a warning (#24). -@test !isflagset(env[:Flags], Cuint(LMDB.MDB_NOSYNC)) -env[:Flags] = LMDB.MDB_NOSYNC -@test isflagset(env[:Flags], Cuint(LMDB.MDB_NOSYNC)) -unset!(env, LMDB.MDB_NOSYNC) - -# Unknown options error instead of silently warning + returning bogus values. -@test_throws ArgumentError env[:Bogus] = 1 -@test_throws ArgumentError env[:Bogus] - -# Open a DB on the env, then close it. -mktempdir() do dir - ret = open(env, dir) - @test ret[1] == 0 - - # stat(env) returns the main DB's stats; before any puts, there are - # no entries and a positive page size. - s = stat(env) - @test s isa NamedTuple - @test s.psize > 0 - @test s.entries == 0 - - # Close environment - close(env) - @test !isopen(env) - - # do block - create() do env - set!(env, LMDB.MDB_NOSYNC) - open(env, dir) - @test isopen(env) - end -end - -# High-level Environment(path; ...) constructor. -mktempdir() do dir - big = Csize_t(8) * 1024^3 - env = Environment(dir; mapsize = big, maxreaders = 42, maxdbs = 4, - flags = MDB_NOSYNC | MDB_NOTLS) +module LMDB_Env + using LMDB + using Test + + const dbname = "testdb" + + # Open environemnt + env = create() + @test env.handle != C_NULL + @test env[:Readers] == 126 + @test env[:KeySize] == 511 + @test env[:Flags] == 0 + + # Manipulate flags + @test !isflagset(env[:Flags], Cuint(LMDB.MDB_NOSYNC)) + set!(env, LMDB.MDB_NOSYNC) + @test isflagset(env[:Flags], Cuint(LMDB.MDB_NOSYNC)) + unset!(env, LMDB.MDB_NOSYNC) + @test !isflagset(env[:Flags], Cuint(LMDB.MDB_NOSYNC)) + + # Parameters + @test (env[:Readers] = 100) == 100 + @test (env[:MapSize] = 1000^2) == 1000^2 + @test (env[:DBs] = 10) == 10 + @test env[:Readers] == 100 + + # Map sizes >4 GiB must not silently truncate (issue #38). + big_mapsize = Csize_t(5) * 1024 * 1024 * 1024 + @test (env[:MapSize] = big_mapsize) == big_mapsize + + # :Flags setindex! used to fall through to a warn(...) that no longer + # exists, throwing UndefVarError (issue #24). + env[:Flags] = LMDB.MDB_NOSYNC + @test isflagset(env[:Flags], Cuint(LMDB.MDB_NOSYNC)) + unset!(env, LMDB.MDB_NOSYNC) + + # Unknown options used to be silently swallowed. + @test_throws ArgumentError env[:Bogus] = 1 + + # open db + isdir(dbname) || mkdir(dbname) try - @test isopen(env) - @test env[:Readers] == 42 - @test info(env).mapsize == big - @test isflagset(env[:Flags], Cuint(MDB_NOSYNC)) - @test isflagset(env[:Flags], Cuint(MDB_NOTLS)) - finally - close(env) - end + ret = open(env, dbname) + @test ret[1] == 0 - # On failure during open, the Environment ctor closes the partial env. - @test_throws LMDBError Environment(joinpath(dir, "definitely_does_not_exist")) -end + # stat(env) returns the main DB's stats; before any puts, there are + # no entries and a positive page size. + s = stat(env) + @test s isa NamedTuple + @test s.psize > 0 + @test s.entries == 0 -# Finalizers: an abandoned write txn must be aborted by GC so a later -# write txn doesn't block on LMDB's exclusive write mutex. If the -# finalizer doesn't fire, `start(env)` below would deadlock. -mktempdir() do dir - env = Environment(dir) - try - # Open a write txn and let it become unreachable without commit/abort. - let txn = start(env) - @test isopen(txn) - end - GC.gc(); GC.gc() - # If the finalizer aborted the abandoned txn, this succeeds. - txn2 = start(env) - try - dbi = open(txn2) - LMDB.put!(txn2, dbi, "k", "v") - finally - commit(txn2) - end - finally + # Close environment close(env) - end -end + @test !isopen(env) -# Cursor finalizer: an abandoned cursor must be cleaned up so its -# parent txn can commit. (LMDB requires cursors on a write txn to be -# closed before commit; for read txns it's safer too.) -mktempdir() do dir - env = Environment(dir) - try - start(env) do txn - dbi = open(txn) - let cur = LMDB.open(txn, dbi) - @test isopen(cur) - end # cur out of scope - GC.gc(); GC.gc() - # If the finalizer ran, we can still use the txn. - LMDB.put!(txn, dbi, "k", "v") + # do block + create() do env + set!(env, LMDB.MDB_NOSYNC) + open(env, dbname) + @test isopen(env) end finally - close(env) - end -end - -# Cursor finalizer is safe even after its parent txn has been -# explicitly committed: write-txn cursors are freed by the txn's -# commit per `lmdb.h`, so `mdb_cursor_close` afterwards would be UB. -# The defensive check in `close(::Cursor)` skips the LMDB call once -# the parent txn handle is gone. -mktempdir() do dir - env = Environment(dir) - try - txn = start(env) - dbi = open(txn) - cur = LMDB.open(txn, dbi) - LMDB.put!(txn, dbi, "k", "v") - commit(txn) # invalidates write-txn cursors - @test !isopen(txn) - cur = nothing # drop the binding - GC.gc(); GC.gc() # finalizer should be a no-op - finally - close(env) + rm(dbname, recursive=true) end -end -# Parent refs: env(txn) and transaction(cur) return the actual parents. -mktempdir() do dir - env = Environment(dir) - try - start(env) do txn - @test LMDB.env(txn) === env - dbi = open(txn) - LMDB.open(txn, dbi) do cur - @test LMDB.transaction(cur) === txn - end + # High-level Environment(path; ...) constructor. + mktempdir() do dir + big = Csize_t(8) * 1024^3 + env = Environment(dir; mapsize = big, maxreaders = 42, maxdbs = 4, + flags = MDB_NOSYNC | MDB_NOTLS) + try + @test isopen(env) + @test env[:Readers] == 42 + @test info(env).mapsize == big + @test isflagset(env[:Flags], Cuint(MDB_NOSYNC)) + @test isflagset(env[:Flags], Cuint(MDB_NOTLS)) + finally + close(env) end - finally - close(env) + + # On failure during open, the Environment ctor closes the partial env. + @test_throws LMDBError Environment(joinpath(dir, "definitely_does_not_exist")) end -end -# reader_check / reader_list / copy -mktempdir() do dir - environment(dir) do env - # Fresh env: no stale readers. - @test reader_check(env) == 0 - - # reader_list always emits a header line listing slot fields. - txt = reader_list(env) - @test txt isa String - @test !isempty(txt) - - # Round-trip a copy. - start(env) do txn - open(txn) do dbi - LMDB.put!(txn, dbi, "k", "v") + # reader_check / reader_list / copy + mktempdir() do dir + environment(dir) do env + # Fresh env: no stale readers. + @test reader_check(env) == 0 + + # reader_list always emits a header line listing slot fields. + txt = reader_list(env) + @test txt isa String + @test !isempty(txt) + + # Round-trip a copy. + start(env) do txn + open(txn) do dbi + LMDB.put!(txn, dbi, "k", "v") + end end - end - mktempdir() do dst - copy(env, dst) - environment(dst) do env2 - start(env2) do txn - open(txn) do dbi - @test LMDB.tryget(txn, dbi, "k", String) == "v" + mktempdir() do dst + copy(env, dst) + environment(dst) do env2 + start(env2) do txn + open(txn) do dbi + @test LMDB.tryget(txn, dbi, "k", String) == "v" + end end end end - end - mktempdir() do dst - copy(env, dst; compact=true) - environment(dst) do env2 - start(env2) do txn - open(txn) do dbi - @test LMDB.tryget(txn, dbi, "k", String) == "v" + mktempdir() do dst + copy(env, dst; compact=true) + environment(dst) do env2 + start(env2) do txn + open(txn) do dbi + @test LMDB.tryget(txn, dbi, "k", String) == "v" + end end end end end end end - -end # @testset "Environment" diff --git a/test/integration.jl b/test/integration.jl index d5ed6e3..bd8387d 100644 --- a/test/integration.jl +++ b/test/integration.jl @@ -1,108 +1,106 @@ -using LMDB -using Test +module LMDB_Integration + using LMDB + using Test -# cuTile-shaped framed value: 8-byte LE atime prefix, then the payload. -# Defined at file scope to demonstrate the `Base.read(::IO, …)` -# extension contract and to regression-guard it (a downstream package -# — cuTile's DiskCache — relies on being able to plug in its own -# decoder against the abstract `IO`, without depending on -# `LMDB.MDBValueIO` in its own type signatures). -struct AtimedBlob end -const _ATIME_PREFIX = 8 + # cuTile-shaped framed value: 8-byte LE atime prefix, then the payload. + # Defined here at module scope to demonstrate the `Base.read(::IO, …)` + # extension contract and to regression-guard it (a downstream package + # — cuTile's DiskCache — relies on being able to plug in its own + # decoder against the abstract `IO`, without depending on + # `LMDB.MDBValueIO` in its own type signatures). + struct AtimedBlob end + const _ATIME_PREFIX = 8 -function Base.read(io::IO, ::Type{AtimedBlob}) - bytesavailable(io) < _ATIME_PREFIX && return UInt8[] - skip(io, _ATIME_PREFIX) - return read(io, Vector{UInt8}) -end - -pack_atimed(atime::UInt64, payload::Vector{UInt8}) = begin - out = Vector{UInt8}(undef, _ATIME_PREFIX + length(payload)) - GC.@preserve out unsafe_store!(Ptr{UInt64}(pointer(out)), htol(atime)) - copyto!(out, _ATIME_PREFIX + 1, payload, 1, length(payload)) - out -end + function Base.read(io::IO, ::Type{AtimedBlob}) + bytesavailable(io) < _ATIME_PREFIX && return UInt8[] + skip(io, _ATIME_PREFIX) + return read(io, Vector{UInt8}) + end -@testset "Integration" begin + pack_atimed(atime::UInt64, payload::Vector{UInt8}) = begin + out = Vector{UInt8}(undef, _ATIME_PREFIX + length(payload)) + GC.@preserve out unsafe_store!(Ptr{UInt64}(pointer(out)), htol(atime)) + copyto!(out, _ATIME_PREFIX + 1, payload, 1, length(payload)) + out + end -# Power-user pattern: open an env via the Environment kwargs ctor, run -# a write txn through the Julia wrappers, then a read txn through a cursor -# walk using only the Julia wrappers + raw MDB_val refs (the shape -# cuTile.DiskCache follows). Regression guard: ensures no future change -# breaks the `walk(...) do k_ref, v_ref` zero-copy idiom. -mktempdir() do dir - env = Environment(dir; - mapsize = 1 << 28, - maxreaders = 64, - flags = MDB_NOTLS | MDB_NORDAHEAD) - try - dbi, psize = start(env) do txn - d = open(txn) - (d, LMDB.stat(txn, d).psize) - end - @test psize > 0 + # Power-user pattern: open an env via the Environment kwargs ctor, run + # a write txn through tier-2, then a read txn through a cursor walk + # using only tier-2 + raw MDB_val refs (the shape cuTile.DiskCache + # follows). Regression guard: ensures no future change breaks the + # `walk(...) do k_ref, v_ref` zero-copy idiom. + mktempdir() do dir + env = Environment(dir; + mapsize = 1 << 28, + maxreaders = 64, + flags = MDB_NOTLS | MDB_NORDAHEAD) + try + dbi, psize = start(env) do txn + d = open(txn) + (d, LMDB.stat(txn, d).psize) + end + @test psize > 0 - # Populate. - start(env) do txn - for i in 1:5 - LMDB.put!(txn, dbi, "key$(i)", "value$(i)") + # Populate. + start(env) do txn + for i in 1:5 + LMDB.put!(txn, dbi, "key$(i)", "value$(i)") + end end - end - # Julia-API read txn + cursor walk over the LMDB-owned mmap, like - # cuTile's eviction scan: zero allocations beyond the per-entry - # tuple. - entries = Tuple{String, Int}[] - start(env; flags = MDB_RDONLY) do txn - LMDB.open(txn, dbi) do cur - LMDB.walk(cur) do k_ref, v_ref - kv = k_ref[]; vv = v_ref[] - k = unsafe_string(Ptr{UInt8}(kv.mv_data), kv.mv_size) - push!(entries, (k, Int(vv.mv_size))) + # Tier-2 read txn + cursor walk over the LMDB-owned mmap, like + # cuTile's eviction scan: zero allocations beyond the per-entry + # tuple. + entries = Tuple{String, Int}[] + start(env; flags = MDB_RDONLY) do txn + LMDB.open(txn, dbi) do cur + LMDB.walk(cur) do k_ref, v_ref + kv = k_ref[]; vv = v_ref[] + k = unsafe_string(Ptr{UInt8}(kv.mv_data), kv.mv_size) + push!(entries, (k, Int(vv.mv_size))) + end end end - end - @test length(entries) == 5 - @test first.(entries) == ["key$i" for i in 1:5] - @test all(e -> e[2] == sizeof("value1"), entries) + @test length(entries) == 5 + @test first.(entries) == ["key$i" for i in 1:5] + @test all(e -> e[2] == sizeof("value1"), entries) - # tryget vs is_notfound — common cuTile-shaped read path. - start(env; flags = MDB_RDONLY) do txn - @test LMDB.tryget(txn, dbi, "key3", String) == "value3" - @test LMDB.tryget(txn, dbi, "ghost", String) === nothing - end + # tryget vs is_notfound — common cuTile-shaped read path. + start(env; flags = MDB_RDONLY) do txn + @test LMDB.tryget(txn, dbi, "key3", String) == "value3" + @test LMDB.tryget(txn, dbi, "ghost", String) === nothing + end - # Batch delete: present keys return true, missing return false, - # no exception on either path. cuTile's `_delete_batch!` uses - # the Bool to count actual evictions. - deleted = 0 - start(env) do txn - for k in ["key1", "ghost", "key3"] - LMDB.delete!(txn, dbi, k) && (deleted += 1) + # Batch delete: present keys return true, missing return false, + # no exception on either path. cuTile's `_delete_batch!` uses + # the Bool to count actual evictions. + deleted = 0 + start(env) do txn + for k in ["key1", "ghost", "key3"] + LMDB.delete!(txn, dbi, k) && (deleted += 1) + end + end + @test deleted == 2 + start(env; flags = MDB_RDONLY) do txn + @test LMDB.tryget(txn, dbi, "key1", String) === nothing + @test LMDB.tryget(txn, dbi, "key2", String) == "value2" + @test LMDB.tryget(txn, dbi, "key3", String) === nothing end - end - @test deleted == 2 - start(env; flags = MDB_RDONLY) do txn - @test LMDB.tryget(txn, dbi, "key1", String) === nothing - @test LMDB.tryget(txn, dbi, "key2", String) == "value2" - @test LMDB.tryget(txn, dbi, "key3", String) === nothing - end - # MDBValueIO extension: write an 8-byte-prefixed framed value - # and read it back via `tryget(..., AtimedBlob)`, getting the - # payload tail with one alloc + skip + copy, no slicing. - payload = Vector{UInt8}("cubin-bytes-here") - atime = UInt64(0xdeadbeefcafebabe) - start(env) do txn - LMDB.put!(txn, dbi, "framed", pack_atimed(atime, payload)) - end - start(env; flags = MDB_RDONLY) do txn - @test LMDB.tryget(txn, dbi, "framed", AtimedBlob) == payload - @test LMDB.tryget(txn, dbi, "ghost", AtimedBlob) === nothing + # MDBValueIO extension: write an 8-byte-prefixed framed value + # and read it back via `tryget(..., AtimedBlob)`, getting the + # payload tail with one alloc + skip + copy, no slicing. + payload = Vector{UInt8}("cubin-bytes-here") + atime = UInt64(0xdeadbeefcafebabe) + start(env) do txn + LMDB.put!(txn, dbi, "framed", pack_atimed(atime, payload)) + end + start(env; flags = MDB_RDONLY) do txn + @test LMDB.tryget(txn, dbi, "framed", AtimedBlob) == payload + @test LMDB.tryget(txn, dbi, "ghost", AtimedBlob) === nothing + end + finally + close(env) end - finally - close(env) end end - -end # @testset "Integration" diff --git a/test/liblmdb.jl b/test/liblmdb.jl index 586a317..931c40b 100644 --- a/test/liblmdb.jl +++ b/test/liblmdb.jl @@ -1,56 +1,54 @@ -using LMDB -using Test - -@testset "liblmdb" begin - -# @checked: status-returning bindings auto-throw an LMDBError on a -# non-zero return. -mktempdir() do dir - env_ref = Ref{Ptr{LMDB.MDB_env}}(C_NULL) - LMDB.mdb_env_create(env_ref) - env = env_ref[] - try - # Opening a *file* (not a directory) when MDB_NOSUBDIR isn't - # set should fail with a non-zero status that @checked turns - # into an LMDBError. - f = touch(joinpath(dir, "not_a_dir")) - @test_throws LMDBError LMDB.mdb_env_open(env, - f, Cuint(0), LMDB.mode_t(0o644)) - finally - LMDB.mdb_env_close(env) +module LMDB_LibLMDB + using LMDB + using Test + + # @checked: status-returning bindings auto-throw an LMDBError on a + # non-zero return. + mktempdir() do dir + env_ref = Ref{Ptr{LMDB.MDB_env}}(C_NULL) + LMDB.mdb_env_create(env_ref) + env = env_ref[] + try + # Opening a *file* (not a directory) when MDB_NOSUBDIR isn't + # set should fail with a non-zero status that @checked turns + # into an LMDBError. + f = touch(joinpath(dir, "not_a_dir")) + @test_throws LMDBError LMDB.mdb_env_open(env, + f, Cuint(0), LMDB.mode_t(0o644)) + finally + LMDB.mdb_env_close(env) + end end -end - -# unchecked_*: returns the raw Cint without throwing — caller decides. -mktempdir() do dir - env_ref = Ref{Ptr{LMDB.MDB_env}}(C_NULL) - LMDB.mdb_env_create(env_ref) - env = env_ref[] - try - LMDB.mdb_env_open(env, dir, Cuint(0), LMDB.mode_t(0o755)) - - txn_ref = Ref{Ptr{LMDB.MDB_txn}}(C_NULL) - LMDB.mdb_txn_begin(env, C_NULL, Cuint(0), txn_ref) - txn = txn_ref[] - dbi_ref = Ref{LMDB.MDB_dbi}(0) - LMDB.mdb_dbi_open(txn, C_NULL, Cuint(0), dbi_ref) - dbi = dbi_ref[] - - # Look up a key that doesn't exist — unchecked returns - # MDB_NOTFOUND, no exception. - key = "missing" - val_ref = Ref(LMDB.MDBValue()) - ret = LMDB.unchecked_mdb_get(txn, dbi, key, val_ref) - @test ret == LMDB.MDB_NOTFOUND - - # The checked counterpart throws. - @test_throws LMDBError LMDB.mdb_get(txn, dbi, key, val_ref) - - LMDB.mdb_txn_abort(txn) - finally - LMDB.mdb_env_close(env) + # unchecked_*: returns the raw Cint without throwing — caller decides. + mktempdir() do dir + env_ref = Ref{Ptr{LMDB.MDB_env}}(C_NULL) + LMDB.mdb_env_create(env_ref) + env = env_ref[] + try + LMDB.mdb_env_open(env, dir, Cuint(0), LMDB.mode_t(0o755)) + + txn_ref = Ref{Ptr{LMDB.MDB_txn}}(C_NULL) + LMDB.mdb_txn_begin(env, C_NULL, Cuint(0), txn_ref) + txn = txn_ref[] + + dbi_ref = Ref{LMDB.MDB_dbi}(0) + LMDB.mdb_dbi_open(txn, C_NULL, Cuint(0), dbi_ref) + dbi = dbi_ref[] + + # Look up a key that doesn't exist — unchecked returns + # MDB_NOTFOUND, no exception. + key = "missing" + val_ref = Ref(LMDB.MDBValue()) + ret = LMDB.unchecked_mdb_get(txn, dbi, key, val_ref) + @test ret == LMDB.MDB_NOTFOUND + + # The checked counterpart throws. + @test_throws LMDBError LMDB.mdb_get(txn, dbi, key, val_ref) + + LMDB.mdb_txn_abort(txn) + finally + LMDB.mdb_env_close(env) + end end end - -end # @testset "liblmdb"