From 281af7b1ab898bc4b1a59701921a0caec2f31e9c Mon Sep 17 00:00:00 2001 From: Tim Besard Date: Sat, 2 May 2026 09:46:34 +0200 Subject: [PATCH 1/4] Refactor LMDBDict to use tier-2 primitives only. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit dicts.jl no longer reaches into liblmdb directly: keys/values/collect/ list_dirs/valuesize switch from LMDBIterator to a private _walk_prefix helper built on `walk` + `seek_range!`/`next!`, and haskey/get/get! use `tryget` and `get(...,default)` instead of `unchecked_mdb_get`. `get!`-with-default now writes inside the same write txn that observed the missing key, instead of opening a separate write txn after a read txn — a small atomicity improvement; the previous form raced against concurrent writers. Also: `walk` now stops if its callback returns `false` (any other return — including `nothing` — continues), which is what makes the prefix-scan loop in `_walk_prefix` work without dropping back to tier-1 cursor ops. `grep '\bmdb_' src/dicts.jl` is empty after this commit. Co-Authored-By: Claude Opus 4.7 (1M context) --- src/cur.jl | 7 ++- src/dicts.jl | 169 ++++++++++++++++++++++++++++++--------------------- 2 files changed, 103 insertions(+), 73 deletions(-) diff --git a/src/cur.jl b/src/cur.jl index 0f4c411..b0b35b0 100644 --- a/src/cur.jl +++ b/src/cur.jl @@ -429,9 +429,10 @@ Walk every entry the cursor visits, calling starts at the first key (`MDB_FIRST`) when `from === nothing`, otherwise at the smallest key `>= from` (`MDB_SET_RANGE`). -Inside `f`, `key_ref[]` and `val_ref[]` point into LMDB-owned memory and are -valid only for the duration of the surrounding transaction; copy out anything -you want to retain. +Iteration stops when `f` returns `false` (any other return value, including +`nothing`, continues). Inside `f`, `key_ref[]` and `val_ref[]` point into +LMDB-owned memory and are valid only for the duration of the surrounding +transaction; copy out anything you want to retain. """ function walk(f, cur::Cursor; from = nothing) key_ref = Ref(MDBValue()) diff --git a/src/dicts.jl b/src/dicts.jl index 7eecc0a..71d4357 100644 --- a/src/dicts.jl +++ b/src/dicts.jl @@ -12,15 +12,19 @@ mutable struct LMDBDict{K,V} end function LMDBDict{K,V}(path::String; readonly = false, rdahead = false, mapsize::Union{Integer,Nothing} = nothing, - maxreaders::Union{Integer,Nothing} = nothing, - maxdbs::Union{Integer,Nothing} = nothing) where {K,V} - txnflags = readonly ? Cuint(MDB_RDONLY) : zero(Cuint) + readers::Union{Integer,Nothing} = nothing, + dbs::Union{Integer,Nothing} = nothing) where {K,V} + flags = readonly ? MDB_RDONLY : zero(Cuint) if !rdahead - txnflags = txnflags | Cuint(MDB_NORDAHEAD) - end - env = LMDB.Environment(path; mapsize, maxreaders, maxdbs) - # A transaction just for getting a DBI handle. - dbi = LMDB.start(env, flags = txnflags) do txn + flags = flags | MDB_NORDAHEAD + end + env = LMDB.create() + mapsize === nothing || (env[:MapSize] = mapsize) + readers === nothing || (env[:Readers] = readers) + dbs === nothing || (env[:DBs] = dbs) + open(env, path) + #A transaction just for getting a DBI handle + dbi = LMDB.start(env,flags=flags) do txn LMDB.open(txn) end LMDBDict{K,V}(env, dbi) @@ -48,115 +52,140 @@ function txn_dbi_do(f, d; readonly = false) end end +# Does `kv`'s data start with `prefix`? +@inline function _has_prefix(kv::LMDB.MDB_val, prefix::Vector{UInt8}) + kv.mv_size < length(prefix) && return false + p = Ptr{UInt8}(kv.mv_data) + @inbounds for i in 1:length(prefix) + unsafe_load(p, i) == prefix[i] || return false + end + return true +end + +# Walk every entry under `prefix` (or every entry if `prefix` is empty), +# stopping at the first key that no longer matches. +function _walk_prefix(f, cur, prefix::Vector{UInt8}) + if isempty(prefix) + LMDB.walk(f, cur) + else + LMDB.walk(cur; from = prefix) do k_ref, v_ref + _has_prefix(k_ref[], prefix) || return false + f(k_ref, v_ref) + return nothing + end + end +end + function list_dirs(d::LMDBDict{String}; prefix = "", sep = '/') + bprefix = Vector{UInt8}(prefix) + sepb = UInt8(sep) + out = String[] cursor_do(d, readonly = true) do cur - bprefix = Vector{UInt8}(prefix) - iter = LMDB.LMDBIterator(cur,LMDB.DirectoryLister(;sep = sep, lprefix = length(bprefix)),bprefix) - collect(iter) + k = isempty(bprefix) ? LMDB.seek!(cur, Vector{UInt8}) : + LMDB.seek_range!(cur, bprefix, Vector{UInt8}) + while k !== nothing + (length(k) >= length(bprefix) && + view(k, 1:length(bprefix)) == bprefix) || break + sepidx = findnext(==(sepb), k, length(bprefix) + 1) + if sepidx === nothing + push!(out, String(copy(k))) + k = LMDB.next!(cur, Vector{UInt8}) + else + push!(out, String(@view k[1:sepidx])) + next_marker = copy(k[1:sepidx]) + next_marker[end] = next_marker[end] + 0x01 + k = LMDB.seek_range!(cur, next_marker, Vector{UInt8}) + end + end end + return out end function Base.keys(d::LMDBDict{K}; prefix=UInt8[]) where K + bprefix = Vector{UInt8}(prefix) + out = K[] cursor_do(d, readonly = true) do cur - collect(keys(cur,K,prefix=prefix)) + _walk_prefix(cur, bprefix) do k_ref, _ + push!(out, LMDB.mdb_unpack(K, k_ref)) + end end + return out end function Base.values(d::LMDBDict{K,V}; prefix=UInt8[]) where {K,V} + bprefix = Vector{UInt8}(prefix) + out = V[] cursor_do(d, readonly = true) do cur - collect(values(cur,V,prefix=prefix)) + _walk_prefix(cur, bprefix) do _, v_ref + push!(out, LMDB.mdb_unpack(V, v_ref)) + end end + return out end function Base.collect(d::LMDBDict{K,V}; prefix=UInt8[]) where {K,V} + bprefix = Vector{UInt8}(prefix) + out = Pair{K,V}[] cursor_do(d, readonly = true) do cur - collect((LMDB.LMDBIterator(cur,LMDB.ReturnBoth{K,V}(),Vector{UInt8}(prefix)))) + _walk_prefix(cur, bprefix) do k_ref, v_ref + push!(out, LMDB.mdb_unpack(K, k_ref) => LMDB.mdb_unpack(V, v_ref)) + end end + return out end function valuesize(d::LMDBDict; prefix = UInt8[]) + bprefix = Vector{UInt8}(prefix) + total = 0 cursor_do(d, readonly = true) do cur - iter = LMDB.LMDBIterator(cur,LMDB.ReturnValueSize(),Vector{UInt8}(prefix)) - sum(iter) + _walk_prefix(cur, bprefix) do _, v_ref + total += Int(v_ref[].mv_size) + end end + return total end -function Base.getindex(d::LMDBDict{K,V},k) where {K,V} - cursor_do(d, readonly = true) do cur - LMDB.get(cur, convert(K,k), V, LMDB.MDB_SET_KEY) +function Base.getindex(d::LMDBDict{K,V}, k) where {K,V} + txn_dbi_do(d, readonly = true) do txn, dbi + LMDB.get(txn, dbi, convert(K, k), V) end end -function Base.haskey(d::LMDBDict{K}, key) where K +function Base.haskey(d::LMDBDict{K,V}, key) where {K,V} txn_dbi_do(d, readonly = true) do txn, dbi - mdb_val_ref = Ref(MDBValue()) - ret = unchecked_mdb_get(txn, dbi, convert(K, key), mdb_val_ref) - if ret == MDB_NOTFOUND - return false - elseif ret == Cint(0) - return true - else - throw(LMDB.LMDBError(ret)) - end + LMDB.tryget(txn, dbi, convert(K, key), V) !== nothing end end function Base.get(d::LMDBDict{K,V}, key, default) where {K,V} txn_dbi_do(d, readonly = true) do txn, dbi - mdb_val_ref = Ref(MDBValue()) - ret = unchecked_mdb_get(txn, dbi, convert(K, key), mdb_val_ref) - if ret == MDB_NOTFOUND - return default - elseif ret == Cint(0) - return mdb_unpack(V, mdb_val_ref) - else - throw(LMDB.LMDBError(ret)) - end + LMDB.get(txn, dbi, convert(K, key), V, default) end end function Base.get!(d::LMDBDict{K,V}, key, default) where {K,V} - txn_dbi_do(d, readonly = true) do txn, dbi - mdb_val_ref = Ref(MDBValue()) - ret = unchecked_mdb_get(txn, dbi, convert(K, key), mdb_val_ref) - if ret == MDB_NOTFOUND - d[key] = default - return default - elseif ret == Cint(0) - return mdb_unpack(V, mdb_val_ref) - else - throw(LMDB.LMDBError(ret)) - end + txn_dbi_do(d) do txn, dbi + v = LMDB.tryget(txn, dbi, convert(K, key), V) + v !== nothing && return v + LMDB.put!(txn, dbi, convert(K, key), convert(V, default)) + return default end end function Base.get(f::F, d::LMDBDict{K,V}, key) where {K,V,F<:Union{Function, Type}} txn_dbi_do(d, readonly = true) do txn, dbi - mdb_val_ref = Ref(MDBValue()) - ret = unchecked_mdb_get(txn, dbi, convert(K, key), mdb_val_ref) - if ret == MDB_NOTFOUND - return f() - elseif ret == Cint(0) - return mdb_unpack(V, mdb_val_ref) - else - throw(LMDB.LMDBError(ret)) - end + v = LMDB.tryget(txn, dbi, convert(K, key), V) + v === nothing ? f() : v end end function Base.get!(f::F, d::LMDBDict{K,V}, key) where {K,V,F<:Union{Function, Type}} - txn_dbi_do(d, readonly = true) do txn, dbi - mdb_val_ref = Ref(MDBValue()) - ret = unchecked_mdb_get(txn, dbi, convert(K, key), mdb_val_ref) - if ret == MDB_NOTFOUND - default = f() - d[key] = default - return default - elseif ret == Cint(0) - return mdb_unpack(V, mdb_val_ref) - else - throw(LMDB.LMDBError(ret)) - end + txn_dbi_do(d) do txn, dbi + v = LMDB.tryget(txn, dbi, convert(K, key), V) + v !== nothing && return v + default = f() + LMDB.put!(txn, dbi, convert(K, key), convert(V, default)) + return default end end From ef02e58a01ad9258fbb92a3e7db29b22d9cf3477 Mon Sep 17 00:00:00 2001 From: Tim Besard Date: Sat, 2 May 2026 11:05:19 +0200 Subject: [PATCH 2/4] Make LMDBDict <: AbstractDict; LLVM-style iterate. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit LMDBDict now subtypes AbstractDict{K,V}, so merge!/filter!/pairs/==/ hash/lazy keys/lazy values come from the standard library. The hand-rolled keys/values/collect overloads with a `prefix` kwarg are gone; `keys(d)` and `values(d)` are the lazy `KeySet`/`ValueIterator` the AbstractDict contract promises. Iteration follows LLVM.jl's `BasicBlockInstructionSet` pattern: the state is `(txn, cur)`, opened on the first `iterate` call (analogous to LLVM's `LLVMGetFirstInstruction` seed) and advanced with `MDB_NEXT` each step. Normal completion commits the txn and closes the cursor; early `break`/throw is reclaimed by the Step-9 finalizers. Other AbstractDict-shaped changes: - `getindex(d, k)` throws `KeyError(k)` on miss (was `LMDBError`). - `pop!(d, k)` throws `KeyError`; `pop!(d, k, default)` returns the default; both are atomic in one write txn. - `length(d)` via `mdb_stat`; `isempty`, `empty!` follow. - `delete!(d, k)` no longer throws on missing keys (Base contract). Prefix-scoped scans move to LMDB-namespaced helpers — `LMDB.scan(d)`, `LMDB.scan_keys(d)`, `LMDB.scan_values(d)` — alongside `list_dirs` and `valuesize`. Defaults: - `MDB_NOTLS` is now set on the LMDBDict env, matching py-lmdb. Without it, a `length(d)` call mid-iteration tripped MDB_BAD_RSLOT. Co-Authored-By: Claude Opus 4.7 (1M context) --- README.md | 16 ++- src/LMDB.jl | 61 +++++++--- src/dicts.jl | 313 ++++++++++++++++++++++++++++++++++----------------- test/dict.jl | 286 +++++++++++++++++++++++++++++----------------- 4 files changed, 452 insertions(+), 224 deletions(-) diff --git a/README.md b/README.md index 1b82a89..6a25e87 100644 --- a/README.md +++ b/README.md @@ -27,7 +27,9 @@ inspect the raw status code. ### Tier 3 — `LMDBDict` -A persistent `Dict`-like object backed by one LMDB environment + DBI. +`LMDBDict{K,V} <: AbstractDict{K,V}`, so the standard library does most +of the work — `merge!`, `filter!`, `pairs`, `==`, `hash`, `keys`, +`values`, lazy iteration — all come for free: ```julia using LMDB @@ -37,13 +39,19 @@ d["beta/x"] = Float32[10, 11] d["beta/y"] = Float32[12, 13] @show d["alpha"] -@show haskey(d, "alpha"), haskey(d, "missing") -@show keys(d, prefix = "beta/") # ["beta/x", "beta/y"] -@show LMDB.list_dirs(d) # ["alpha", "beta/"] +@show haskey(d, "alpha"), haskey(d, "missing") # missing throws KeyError +@show length(d) # 3 +for (k, v) in d + @show k, v +end +@show LMDB.scan_keys(d, prefix = "beta/") # ["beta/x", "beta/y"] +@show LMDB.list_dirs(d) # ["alpha", "beta/"] close(d) ``` Constructor kwargs: `mapsize`, `readers`, `dbs`, `readonly`, `rdahead`. +The env is opened with `MDB_NOTLS` so multiple read txns can coexist on +one thread — needed for interleaved reads or task-parallel access. ### Tier 2 — explicit env / txn / cursor diff --git a/src/LMDB.jl b/src/LMDB.jl index d5fab26..0fa17ab 100644 --- a/src/LMDB.jl +++ b/src/LMDB.jl @@ -1,23 +1,48 @@ module LMDB -import Base: open, close, getindex, setindex!, put!, pop!, replace!, reset, copy, - isopen, count, delete!, keys, get, show, stat - -export Environment, create, open, close, sync, set!, unset!, getindex, setindex!, path, info, stat, show, - reader_check, reader_list, copy, - Transaction, start, abort, commit, reset, renew, environment, - DBI, drop, delete!, keys, get, put!, put_reserved!, tryget, replace!, pop!, - Cursor, count, transaction, database, - seek_first_dup!, seek_last_dup!, - next_dup!, prev_dup!, next_nodup!, prev_nodup!, - isflagset, isopen, - # commonly-needed env / write flags - MDB_RDONLY, MDB_NOTLS, MDB_NORDAHEAD, MDB_NOSUBDIR, - MDB_NOSYNC, MDB_NOMETASYNC, MDB_WRITEMAP, MDB_NOMEMINIT, - MDB_CREATE, MDB_DUPSORT, MDB_INTEGERKEY, MDB_REVERSEKEY, - MDB_DUPFIXED, MDB_INTEGERDUP, MDB_REVERSEDUP, - MDB_NOOVERWRITE, MDB_NODUPDATA, MDB_APPEND, MDB_RESERVE, - LMDBError, LMDBDict +import Base: open, close, getindex, setindex!, put!, pop!, replace!, reset, + isopen, count, delete!, keys, get, show, show, stat, copy, + empty!, length, isempty, iterate, haskey +import Base.Iterators: drop + +export + # error type + matchers + LMDBError, is_notfound, is_keyexist, is_map_full, + + # commonly-needed status codes + MDB_NOTFOUND, MDB_KEYEXIST, MDB_MAP_FULL, + + # commonly-needed env flags + MDB_RDONLY, MDB_NOTLS, MDB_NORDAHEAD, MDB_NOSUBDIR, + MDB_NOSYNC, MDB_NOMETASYNC, MDB_WRITEMAP, MDB_NOMEMINIT, + + # commonly-needed db flags + MDB_CREATE, MDB_DUPSORT, MDB_INTEGERKEY, MDB_REVERSEKEY, + MDB_DUPFIXED, MDB_INTEGERDUP, MDB_REVERSEDUP, + + # commonly-needed write flags + MDB_NOOVERWRITE, MDB_NODUPDATA, MDB_APPEND, MDB_RESERVE, + + # tier 2 — environment + Environment, create, environment, + sync, set!, unset!, info, stat, path, isopen, isflagset, + reader_check, reader_list, + + # tier 2 — transaction + Transaction, start, abort, commit, reset, renew, + + # tier 2 — database (DBI) + DBI, drop, get, put!, delete!, tryget, replace!, + + # tier 2 — cursor + Cursor, count, transaction, database, + seek!, seek_last!, seek_range!, next!, prev!, + key, value, item, walk, + seek_first_dup!, seek_last_dup!, + next_dup!, prev_dup!, next_nodup!, prev_nodup!, + + # tier 3 + LMDBDict # --------------------------------------------------------------------------- # Error type. Defined here so the `@checked` macro can reference it; the diff --git a/src/dicts.jl b/src/dicts.jl index 71d4357..cb62481 100644 --- a/src/dicts.jl +++ b/src/dicts.jl @@ -1,10 +1,20 @@ -mutable struct LMDBDict{K,V} +""" + LMDBDict{K,V}(path; readonly, rdahead, mapsize, readers, dbs) + +A persistent `AbstractDict{K,V}` backed by a single LMDB environment + +default DBI. The keys and values are encoded as raw bytes — `String`, +`Vector{T}` (where `T` is bitstype), or any bitstype scalar. + +For prefix-scoped scans (e.g. hierarchical "directory" key schemes), +see `LMDB.scan` / `LMDB.scan_keys` / `LMDB.scan_values` / `LMDB.list_dirs`. +""" +mutable struct LMDBDict{K,V} <: AbstractDict{K,V} env::LMDB.Environment dbi::LMDB.DBI function LMDBDict{K,V}(env::LMDB.Environment, dbi::LMDB.DBI) where {K,V} x = new{K,V}(env, dbi) finalizer(x) do d - LMDB.close(d.env,d.dbi) + LMDB.close(d.env, d.dbi) LMDB.close(d.env) end x @@ -14,32 +24,33 @@ function LMDBDict{K,V}(path::String; readonly = false, rdahead = false, mapsize::Union{Integer,Nothing} = nothing, readers::Union{Integer,Nothing} = nothing, dbs::Union{Integer,Nothing} = nothing) where {K,V} - flags = readonly ? MDB_RDONLY : zero(Cuint) - if !rdahead - flags = flags | MDB_NORDAHEAD - end - env = LMDB.create() - mapsize === nothing || (env[:MapSize] = mapsize) - readers === nothing || (env[:Readers] = readers) - dbs === nothing || (env[:DBs] = dbs) - open(env, path) - #A transaction just for getting a DBI handle - dbi = LMDB.start(env,flags=flags) do txn + # MDB_NOTLS: drop LMDB's default thread-local reader slots, so a single + # thread can hold multiple concurrent read txns. Required for any + # interleaved read (e.g. `length(d)` mid-iteration) and for read txns + # in a multi-task setting. Same default as py-lmdb. + envflags = Cuint(MDB_NOTLS) + rdahead || (envflags |= Cuint(MDB_NORDAHEAD)) + readonly && (envflags |= Cuint(MDB_RDONLY)) + env = LMDB.Environment(path; mapsize, maxreaders = readers, maxdbs = dbs, + flags = envflags) + dbi = LMDB.start(env) do txn LMDB.open(txn) end LMDBDict{K,V}(env, dbi) end LMDBDict(path::String; kwargs...) = LMDBDict{String, Vector{UInt8}}(path; kwargs...) -Base.keytype(::LMDBDict{K}) where K = K -Base.eltype(::LMDBDict{<:Any,V}) where V = V + function Base.close(d::LMDBDict) - LMDB.close(d.env,d.dbi) + LMDB.close(d.env, d.dbi) LMDB.close(d.env) end + +# --- internal helpers --- + function cursor_do(f, d; readonly = false) txnflags = readonly ? Cuint(LMDB.MDB_RDONLY) : Cuint(0) LMDB.start(d.env, flags = txnflags) do txn - LMDB.open(txn,d.dbi) do cur + LMDB.open(txn, d.dbi) do cur f(cur) end end @@ -52,7 +63,6 @@ function txn_dbi_do(f, d; readonly = false) end end -# Does `kv`'s data start with `prefix`? @inline function _has_prefix(kv::LMDB.MDB_val, prefix::Vector{UInt8}) kv.mv_size < length(prefix) && return false p = Ptr{UInt8}(kv.mv_data) @@ -62,8 +72,6 @@ end return true end -# Walk every entry under `prefix` (or every entry if `prefix` is empty), -# stopping at the first key that no longer matches. function _walk_prefix(f, cur, prefix::Vector{UInt8}) if isempty(prefix) LMDB.walk(f, cur) @@ -76,129 +84,232 @@ function _walk_prefix(f, cur, prefix::Vector{UInt8}) end end -function list_dirs(d::LMDBDict{String}; prefix = "", sep = '/') - bprefix = Vector{UInt8}(prefix) - sepb = UInt8(sep) - out = String[] - cursor_do(d, readonly = true) do cur - k = isempty(bprefix) ? LMDB.seek!(cur, Vector{UInt8}) : - LMDB.seek_range!(cur, bprefix, Vector{UInt8}) - while k !== nothing - (length(k) >= length(bprefix) && - view(k, 1:length(bprefix)) == bprefix) || break - sepidx = findnext(==(sepb), k, length(bprefix) + 1) - if sepidx === nothing - push!(out, String(copy(k))) - k = LMDB.next!(cur, Vector{UInt8}) - else - push!(out, String(@view k[1:sepidx])) - next_marker = copy(k[1:sepidx]) - next_marker[end] = next_marker[end] + 0x01 - k = LMDB.seek_range!(cur, next_marker, Vector{UInt8}) - end - end - end - return out +# --- AbstractDict interface --- + +# Iteration: state is `(txn, cur)` opened on the first iterate (LLVM-style; +# the cursor's internal position is the moral equivalent of LLVM's +# `LLVMGetNextInstruction` next-pointer). On normal completion the txn is +# committed and the cursor closed; on early break/throw, Cursor's and +# Transaction's finalizers reclaim them. +function Base.iterate(d::LMDBDict) + txn = LMDB.start(d.env; flags = Cuint(MDB_RDONLY)) + cur = LMDB.open(txn, d.dbi) + return _iter_step(d, txn, cur, MDB_FIRST) end +Base.iterate(d::LMDBDict, (txn, cur)::Tuple{Transaction,Cursor}) = + _iter_step(d, txn, cur, MDB_NEXT) -function Base.keys(d::LMDBDict{K}; prefix=UInt8[]) where K - bprefix = Vector{UInt8}(prefix) - out = K[] - cursor_do(d, readonly = true) do cur - _walk_prefix(cur, bprefix) do k_ref, _ - push!(out, LMDB.mdb_unpack(K, k_ref)) - end +function _iter_step(::LMDBDict{K,V}, txn::Transaction, cur::Cursor, + op::MDB_cursor_op) where {K,V} + k_ref = Ref(MDBValue()) + v_ref = Ref(MDBValue()) + ret = LMDB.unchecked_mdb_cursor_get(cur, k_ref, v_ref, op) + if ret == MDB_NOTFOUND + LMDB.close(cur) + LMDB.commit(txn) + return nothing + elseif !iszero(ret) + LMDB.close(cur) + LMDB.abort(txn) + throw(LMDBError(ret)) end - return out + return (LMDB.mdb_unpack(K, k_ref) => LMDB.mdb_unpack(V, v_ref), (txn, cur)) end -function Base.values(d::LMDBDict{K,V}; prefix=UInt8[]) where {K,V} - bprefix = Vector{UInt8}(prefix) - out = V[] - cursor_do(d, readonly = true) do cur - _walk_prefix(cur, bprefix) do _, v_ref - push!(out, LMDB.mdb_unpack(V, v_ref)) - end +Base.IteratorSize(::Type{<:LMDBDict}) = Base.HasLength() + +function Base.length(d::LMDBDict) + txn_dbi_do(d, readonly = true) do txn, dbi + Int(LMDB.stat(txn, dbi).entries) end - return out end -function Base.collect(d::LMDBDict{K,V}; prefix=UInt8[]) where {K,V} - bprefix = Vector{UInt8}(prefix) - out = Pair{K,V}[] - cursor_do(d, readonly = true) do cur - _walk_prefix(cur, bprefix) do k_ref, v_ref - push!(out, LMDB.mdb_unpack(K, k_ref) => LMDB.mdb_unpack(V, v_ref)) - end +Base.isempty(d::LMDBDict) = iszero(length(d)) + +function Base.getindex(d::LMDBDict{K,V}, k) where {K,V} + txn_dbi_do(d, readonly = true) do txn, dbi + v = LMDB.tryget(txn, dbi, convert(K, k), V) + v === nothing ? throw(KeyError(k)) : v end - return out end -function valuesize(d::LMDBDict; prefix = UInt8[]) - bprefix = Vector{UInt8}(prefix) - total = 0 - cursor_do(d, readonly = true) do cur - _walk_prefix(cur, bprefix) do _, v_ref - total += Int(v_ref[].mv_size) - end +function Base.haskey(d::LMDBDict{K,V}, k) where {K,V} + txn_dbi_do(d, readonly = true) do txn, dbi + LMDB.tryget(txn, dbi, convert(K, k), V) !== nothing end - return total end -function Base.getindex(d::LMDBDict{K,V}, k) where {K,V} +function Base.get(d::LMDBDict{K,V}, k, default) where {K,V} txn_dbi_do(d, readonly = true) do txn, dbi - LMDB.get(txn, dbi, convert(K, k), V) + LMDB.get(txn, dbi, convert(K, k), V, default) end end -function Base.haskey(d::LMDBDict{K,V}, key) where {K,V} +function Base.get(f::Base.Callable, d::LMDBDict{K,V}, k) where {K,V} txn_dbi_do(d, readonly = true) do txn, dbi - LMDB.tryget(txn, dbi, convert(K, key), V) !== nothing + v = LMDB.tryget(txn, dbi, convert(K, k), V) + v === nothing ? f() : v end end -function Base.get(d::LMDBDict{K,V}, key, default) where {K,V} - txn_dbi_do(d, readonly = true) do txn, dbi - LMDB.get(txn, dbi, convert(K, key), V, default) +function Base.get!(d::LMDBDict{K,V}, k, default) where {K,V} + txn_dbi_do(d) do txn, dbi + v = LMDB.tryget(txn, dbi, convert(K, k), V) + v !== nothing && return v + LMDB.put!(txn, dbi, convert(K, k), convert(V, default)) + return default end end -function Base.get!(d::LMDBDict{K,V}, key, default) where {K,V} +function Base.get!(f::Base.Callable, d::LMDBDict{K,V}, k) where {K,V} txn_dbi_do(d) do txn, dbi - v = LMDB.tryget(txn, dbi, convert(K, key), V) + v = LMDB.tryget(txn, dbi, convert(K, k), V) v !== nothing && return v - LMDB.put!(txn, dbi, convert(K, key), convert(V, default)) + default = f() + LMDB.put!(txn, dbi, convert(K, k), convert(V, default)) return default end end -function Base.get(f::F, d::LMDBDict{K,V}, key) where {K,V,F<:Union{Function, Type}} - txn_dbi_do(d, readonly = true) do txn, dbi - v = LMDB.tryget(txn, dbi, convert(K, key), V) - v === nothing ? f() : v +function Base.setindex!(d::LMDBDict{K,V}, v, k) where {K,V} + txn_dbi_do(d) do txn, dbi + LMDB.put!(txn, dbi, convert(K, k), convert(V, v)) end + return d end -function Base.get!(f::F, d::LMDBDict{K,V}, key) where {K,V,F<:Union{Function, Type}} +function Base.delete!(d::LMDBDict{K}, k) where K txn_dbi_do(d) do txn, dbi - v = LMDB.tryget(txn, dbi, convert(K, key), V) - v !== nothing && return v - default = f() - LMDB.put!(txn, dbi, convert(K, key), convert(V, default)) - return default + try + LMDB.delete!(txn, dbi, convert(K, k)) + catch e + e isa LMDBError && LMDB.is_notfound(e) || rethrow() + end end + return d end -function Base.setindex!(d::LMDBDict{K,V},v,k) where {K,V} +function Base.pop!(d::LMDBDict{K,V}, k) where {K,V} txn_dbi_do(d) do txn, dbi - LMDB.put!(txn,dbi,convert(K,k),convert(V,v)) + v = LMDB.pop!(txn, dbi, convert(K, k), V) + v === nothing ? throw(KeyError(k)) : v end - v end -function Base.delete!(d::LMDBDict{K},k) where K +function Base.pop!(d::LMDBDict{K,V}, k, default) where {K,V} txn_dbi_do(d) do txn, dbi - LMDB.delete!(txn, dbi, convert(K,k)) + v = LMDB.pop!(txn, dbi, convert(K, k), V) + v === nothing ? default : v end - d +end + +function Base.empty!(d::LMDBDict) + txn_dbi_do(d) do txn, dbi + LMDB.drop(txn, dbi; delete = false) + end + return d +end + +# AbstractDict's default implementations of `keys`, `values`, `pairs`, +# `merge!`, `filter!`, `==`, `hash`, `in(::Pair, d)` etc. all kick in for +# free now that `iterate` and `length` are defined. + +# --- prefix-scan helpers (LMDB-namespaced; not Base extensions) --- + +""" + scan(d::LMDBDict; prefix=UInt8[]) -> Vector{Pair{K,V}} + +Eagerly collect every `key => value` pair whose key starts with `prefix` +(byte-prefix; pass a `String` or `Vector{UInt8}`). Pass an empty prefix +to scan the whole dict. +""" +function scan(d::LMDBDict{K,V}; prefix = UInt8[]) where {K,V} + bprefix = Vector{UInt8}(prefix) + out = Pair{K,V}[] + cursor_do(d, readonly = true) do cur + _walk_prefix(cur, bprefix) do k_ref, v_ref + push!(out, mdb_unpack(K, k_ref) => mdb_unpack(V, v_ref)) + end + end + return out +end + +""" + scan_keys(d::LMDBDict; prefix=UInt8[]) -> Vector{K} + +Eagerly collect every key whose key starts with `prefix`. +""" +function scan_keys(d::LMDBDict{K}; prefix = UInt8[]) where K + bprefix = Vector{UInt8}(prefix) + out = K[] + cursor_do(d, readonly = true) do cur + _walk_prefix(cur, bprefix) do k_ref, _ + push!(out, mdb_unpack(K, k_ref)) + end + end + return out +end + +""" + scan_values(d::LMDBDict; prefix=UInt8[]) -> Vector{V} + +Eagerly collect every value whose key starts with `prefix`. +""" +function scan_values(d::LMDBDict{K,V}; prefix = UInt8[]) where {K,V} + bprefix = Vector{UInt8}(prefix) + out = V[] + cursor_do(d, readonly = true) do cur + _walk_prefix(cur, bprefix) do _, v_ref + push!(out, mdb_unpack(V, v_ref)) + end + end + return out +end + +""" + list_dirs(d::LMDBDict{String}; prefix="", sep='/') -> Vector{String} + +For dicts that use a hierarchical String key scheme (e.g. `"a/b/c"`), +return the immediate children of `prefix`. A child is either a leaf +key (no `sep` after `prefix`) or a directory marker (`prefix*name*sep`). +""" +function list_dirs(d::LMDBDict{String}; prefix = "", sep = '/') + bprefix = Vector{UInt8}(prefix) + sepb = UInt8(sep) + out = String[] + cursor_do(d, readonly = true) do cur + k = isempty(bprefix) ? LMDB.seek!(cur, Vector{UInt8}) : + LMDB.seek_range!(cur, bprefix, Vector{UInt8}) + while k !== nothing + (length(k) >= length(bprefix) && + view(k, 1:length(bprefix)) == bprefix) || break + sepidx = findnext(==(sepb), k, length(bprefix) + 1) + if sepidx === nothing + push!(out, String(copy(k))) + k = LMDB.next!(cur, Vector{UInt8}) + else + push!(out, String(@view k[1:sepidx])) + next_marker = copy(k[1:sepidx]) + next_marker[end] = next_marker[end] + 0x01 + k = LMDB.seek_range!(cur, next_marker, Vector{UInt8}) + end + end + end + return out +end + +""" + valuesize(d::LMDBDict; prefix=UInt8[]) -> Int + +Sum of the byte sizes of all values whose key starts with `prefix`. +""" +function valuesize(d::LMDBDict; prefix = UInt8[]) + bprefix = Vector{UInt8}(prefix) + total = 0 + cursor_do(d, readonly = true) do cur + _walk_prefix(cur, bprefix) do _, v_ref + total += Int(v_ref[].mv_size) + end + end + return total end diff --git a/test/dict.jl b/test/dict.jl index 3fd9cf8..6867254 100644 --- a/test/dict.jl +++ b/test/dict.jl @@ -1,118 +1,202 @@ using Test, LMDB + @testset "Dictionary-like interface" begin -p1 = tempname() -#Test dict with string keys, float64 values -mkpath(p1) -d = LMDBDict{String, Float64}(p1) -d["x"] = 5.0 -d["y"] = 12.0 -d["z"] = 3 -@test d["x"] === 5.0 -@test d["y"] === 12.0 -@test d["z"] === 3.0 -@test haskey(d,"x") -@test !haskey(d,"a") -@test keys(d) == ["x", "y", "z"] -@test values(d) == [5.0, 12.0, 3.0] -@test collect(d) == ["x"=>5.0, "y"=>12.0, "z"=>3.0] -@test LMDB.valuesize(d) == sizeof(Float64)*3 -delete!(d,"z") -@test !haskey(d,"z") -@test_throws LMDB.LMDBError d["z"] - -#Test int key and values -p2 = tempname() -mkpath(p2) -d = LMDBDict{Int64, Int16}(p2) -for i in 1:10 - d[i] = i+1 -end -@test keys(d) == 1:10 -@test values(d) == 2:11 -@test d[2] === Int16(3) -@test d[3.0] == 4 -@test eltype(d) == Int16 -@test keytype(d) == Int64 - -#Some extra tests for string dicts -p3 = tempname() -mkpath(p3) -d = LMDBDict{String, Vector{Float32}}(p3) -d["aa/a"] = Float32[1,2,3,4] -d["aa/b"] = Float32.(2:12) -d["aa/c"] = [10,11,12] -d["b"] = [0,0,0] -@test d["aa/a"] == 1:4 -@test d["aa/b"] == 2:12 -@test d["aa/c"] == 10:12 -@test d["b"] == [0,0,0] -@test LMDB.list_dirs(d) == ["aa/", "b"] -@test LMDB.list_dirs(d,prefix="aa/") == ["aa/a", "aa/b", "aa/c"] -@test LMDB.valuesize(d,prefix="aa/") == sizeof(Float32)*18 - -@testset "Tests for get and get!" begin - mktempdir() do dir - d = LMDBDict{String, String}(dir) - @test !haskey(d, "foo") - @test get(d, "foo", "bar") == "bar" - @test !haskey(d, "foo") - @test get!(d, "foo", "bar") == "bar" - @test haskey(d, "foo") - @test d["foo"] == "bar" - @test get(d, "foo", "hello") == "bar" - @test d["foo"] == "bar" - @test get!(d, "foo", "hello") == "bar" - @test d["foo"] == "bar" - end + + # Basic round-trip with String keys, Float64 values. mktempdir() do dir - d = LMDBDict{String, String}(dir) - @test !haskey(d, "foo") - @test get(() -> "bar", d, "foo") == "bar" - @test !haskey(d, "foo") - @test get!(() -> "bar", d, "foo") == "bar" - @test haskey(d, "foo") - @test d["foo"] == "bar" - @test get(() -> "hello", d, "foo") == "bar" - @test d["foo"] == "bar" - @test get!(() -> "hello", d, "foo") == "bar" - @test d["foo"] == "bar" + d = LMDBDict{String, Float64}(dir) + d["x"] = 5.0 + d["y"] = 12.0 + d["z"] = 3 + @test d["x"] === 5.0 + @test d["y"] === 12.0 + @test d["z"] === 3.0 + @test haskey(d, "x") + @test !haskey(d, "a") + + # AbstractDict iteration: keys/values/pairs are lazy iterators. + @test collect(keys(d)) == ["x", "y", "z"] + @test collect(values(d)) == [5.0, 12.0, 3.0] + @test collect(d) == ["x"=>5.0, "y"=>12.0, "z"=>3.0] + @test collect(pairs(d)) == ["x"=>5.0, "y"=>12.0, "z"=>3.0] + @test length(d) == 3 + @test !isempty(d) + @test eltype(d) == Pair{String, Float64} + @test keytype(d) == String + @test valtype(d) == Float64 + + # in(::Pair) — comes free from AbstractDict. + @test ("x" => 5.0) in d + @test !(("x" => 99.0) in d) + + # `for` loop yields Pair{K,V}. + seen = Pair{String,Float64}[] + for kv in d + push!(seen, kv) + end + @test seen == ["x"=>5.0, "y"=>12.0, "z"=>3.0] + + # delete! / pop! / KeyError on missing. + delete!(d, "z") + @test !haskey(d, "z") + @test_throws KeyError d["z"] + @test_throws KeyError pop!(d, "z") + @test pop!(d, "z", :missing) === :missing + @test pop!(d, "y") === 12.0 + @test !haskey(d, "y") + + @test LMDB.valuesize(d) == sizeof(Float64)*1 # only "x" left + close(d) end -end -@testset "String -> Int64 round-trip (#46)" begin + # Int → Int with a numeric key range. mktempdir() do dir - d = LMDBDict{String, Int64}(dir) - d["aa"] = 2 - d["ab"] = 3 - d["ac"] = 2 - @test d["aa"] === Int64(2) - @test d["ab"] === Int64(3) - @test d["ac"] === Int64(2) - # Force a GC pass between the writes and the reads. Before the - # GC.@preserve fix this would alias unrelated memory. - GC.gc(); GC.gc() - @test d["aa"] === Int64(2) - @test d["ab"] === Int64(3) + d = LMDBDict{Int64, Int16}(dir) + for i in 1:10 + d[i] = i+1 + end + @test collect(keys(d)) == 1:10 + @test collect(values(d)) == 2:11 + @test length(d) == 10 + @test d[2] === Int16(3) + @test d[3.0] == 4 + @test eltype(d) == Pair{Int64, Int16} + @test valtype(d) == Int16 + @test keytype(d) == Int64 + + # empty! drops every entry. + empty!(d) + @test length(d) == 0 + @test isempty(d) + @test_throws KeyError d[1] + close(d) end -end -@testset "Vector value owns its memory (#41)" begin + # Hierarchical keys: prefix-scan helpers + list_dirs. mktempdir() do dir d = LMDBDict{String, Vector{Float32}}(dir) - d["k"] = Float32[1,2,3,4] - v = d["k"] + d["aa/a"] = Float32[1,2,3,4] + d["aa/b"] = Float32.(2:12) + d["aa/c"] = [10,11,12] + d["b"] = [0,0,0] + @test d["aa/a"] == 1:4 + @test d["aa/b"] == 2:12 + @test d["aa/c"] == 10:12 + @test d["b"] == [0,0,0] + + @test LMDB.list_dirs(d) == ["aa/", "b"] + @test LMDB.list_dirs(d, prefix = "aa/") == ["aa/a", "aa/b", "aa/c"] + @test LMDB.scan_keys(d, prefix = "aa/") == ["aa/a", "aa/b", "aa/c"] + @test LMDB.scan_values(d, prefix = "aa/") == + [Float32[1,2,3,4], Float32.(2:12), Float32[10,11,12]] + @test LMDB.scan(d, prefix = "aa/") == + ["aa/a"=>Float32[1,2,3,4], "aa/b"=>Float32.(2:12), + "aa/c"=>Float32[10,11,12]] + @test LMDB.valuesize(d, prefix = "aa/") == sizeof(Float32)*18 close(d) - GC.gc(); GC.gc() - @test v == Float32[1,2,3,4] end -end -@testset "LMDBDict mapsize kwarg (#45)" begin + # Iteration leaves no leftover state — a second pass returns the same + # entries, and an interleaved `length` (which opens a separate read txn) + # works mid-iteration. mktempdir() do dir - d = LMDBDict{String, String}(dir; mapsize = 64 * 1024 * 1024) - d["k"] = "v" - @test d["k"] == "v" + d = LMDBDict{String, Int}(dir) + d["a"] = 1; d["b"] = 2; d["c"] = 3 + @test collect(d) == ["a"=>1, "b"=>2, "c"=>3] + @test collect(d) == ["a"=>1, "b"=>2, "c"=>3] + n = 0 + for _ in d + n += 1 + @test length(d) == 3 + end + @test n == 3 + close(d) + end + + @testset "env kwargs in LMDBDict ctor (#45)" begin + mktempdir() do dir + big = Csize_t(8) * 1024^3 # 8 GiB + d = LMDBDict{String, Int64}(dir; mapsize=big, readers=42, dbs=4) + @test d.env[:Readers] == 42 + @test LMDB.info(d.env).mapsize == big + d["x"] = 1 + @test d["x"] === Int64(1) + close(d) + end + end + + @testset "double close is a no-op (#42)" begin + mktempdir() do dir + d = LMDBDict{String,Int}(dir) + d["a"] = 1 + close(d) + @test (close(d); true) + end end -end + @testset "Vector value owns its memory (#41)" begin + mktempdir() do dir + d = LMDBDict{String, Vector{Float32}}(dir) + d["k"] = Float32[1,2,3,4] + v = d["k"] + close(d) + GC.gc(); GC.gc() + @test v == Float32[1,2,3,4] + end + end + + @testset "String -> Int64 round-trip (#46)" begin + mktempdir() do dir + d = LMDBDict{String, Int64}(dir) + d["aa"] = 2 + d["ab"] = 3 + d["ac"] = 2 + @test d["aa"] === Int64(2) + @test d["ab"] === Int64(3) + @test d["ac"] === Int64(2) + @test collect(d) == ["aa"=>2, "ab"=>3, "ac"=>2] + end + end + + @testset "Tests for get and get!" begin + mktempdir() do dir + d = LMDBDict{String, String}(dir) + @test !haskey(d, "foo") + @test get(d, "foo", "bar") == "bar" + @test !haskey(d, "foo") + @test get!(d, "foo", "bar") == "bar" + @test haskey(d, "foo") + @test d["foo"] == "bar" + @test get(d, "foo", "hello") == "bar" + @test d["foo"] == "bar" + @test get!(d, "foo", "hello") == "bar" + @test d["foo"] == "bar" + end + mktempdir() do dir + d = LMDBDict{String, String}(dir) + @test !haskey(d, "foo") + @test get(() -> "bar", d, "foo") == "bar" + @test !haskey(d, "foo") + @test get!(() -> "bar", d, "foo") == "bar" + @test haskey(d, "foo") + @test d["foo"] == "bar" + @test get(() -> "hello", d, "foo") == "bar" + @test d["foo"] == "bar" + @test get!(() -> "hello", d, "foo") == "bar" + @test d["foo"] == "bar" + end + end + + @testset "Generic AbstractDict machinery: merge!/filter!" begin + mktempdir() do dir + d = LMDBDict{String, Int}(dir) + merge!(d, Dict("a" => 1, "b" => 2, "c" => 3)) + @test sort(collect(keys(d))) == ["a", "b", "c"] + @test d["b"] == 2 + + filter!(p -> isodd(p.second), d) + @test sort(collect(keys(d))) == ["a", "c"] + close(d) + end + end end From 1c579e7d760540f9e3e7d51e4e6b6110ddb8678d Mon Sep 17 00:00:00 2001 From: Tim Besard Date: Sat, 2 May 2026 11:14:19 +0200 Subject: [PATCH 3/4] Drop dead iterators, fix mbd_unpack typo, add pop!(d), pin DBI in Cursor. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit cur.jl loses ~100 lines of legacy iterator scaffolding that nothing in the new API exercises: - LMDBIterator{R} + ReturnKeys/ReturnValues/ReturnBoth/ReturnValueSize - DirectoryLister + process_returns + init_values + arcopy - keys(cur, T; prefix) / Base.values(cur, T; prefix) / a broken iterate(cur, K, V) that referenced an undefined `prefix` variable - get(cur, key, T, op) — superseded by seek!+value at the right op `mbd_unpack` was always a typo for `mdb_unpack`; renamed in src and tests. Cursor now stores its parent DBI alongside its parent Transaction; database(cur) returns it directly instead of round-tripping through `mdb_cursor_dbi` and synthesizing a fresh wrapper with an empty name. Adds the AbstractDict gap that Base.Dict supplies but Base.AbstractDict does not — `pop!(d::LMDBDict)` (no key form) pops the lexicographically-first entry, throws ArgumentError if empty. Co-Authored-By: Claude Opus 4.7 (1M context) --- src/common.jl | 10 ---- src/cur.jl | 148 +++----------------------------------------------- src/dicts.jl | 13 +++++ test/cur.jl | 26 +++++---- test/dict.jl | 6 +- 5 files changed, 41 insertions(+), 162 deletions(-) diff --git a/src/common.jl b/src/common.jl index 7894506..f4c9715 100644 --- a/src/common.jl +++ b/src/common.jl @@ -59,16 +59,6 @@ function Base.cconvert(::Type{Ptr{MDB_val}}, x::T) where {T} MDBArg(Ref(x)) end -# Private: build an `MDB_val` whose `mv_data` aliases `buf`. The pointer -# is taken via `unsafe_convert`; the caller MUST keep `buf` alive across -# any ccall that reads the resulting `MDB_val` (in practice, by wrapping -# the call site in `GC.@preserve buf ...`). Used by the cursor iterator, -# which already threads its key buffer through the iteration state for -# exactly this reason. -@inline _mdb_val_for(buf::Vector{T}) where {T} = - MDB_val(Csize_t(sizeof(T) * length(buf)), - Ptr{Cvoid}(Base.unsafe_convert(Ptr{T}, buf))) - """ mdb_unpack(::Type{T}, ref::Ref{MDB_val}) -> T diff --git a/src/cur.jl b/src/cur.jl index b0b35b0..190d054 100644 --- a/src/cur.jl +++ b/src/cur.jl @@ -1,15 +1,16 @@ """ A handle to a cursor structure for navigating through a database. -A `Cursor` keeps a reference to its parent `Transaction` to expose it via -`transaction(cur)` and to keep the txn alive under GC. The cursor's -finalizer closes any still-open handle. +A `Cursor` keeps references to its parent `Transaction` and `DBI`, both +to expose them via `transaction(cur)` / `database(cur)` and to keep the +txn alive under GC. The cursor's finalizer closes any still-open handle. """ mutable struct Cursor handle::Ptr{MDB_cursor} txn::Transaction - function Cursor(txn::Transaction, h::Ptr{MDB_cursor}) - c = new(h, txn) + dbi::DBI + function Cursor(txn::Transaction, dbi::DBI, h::Ptr{MDB_cursor}) + c = new(h, txn, dbi) finalizer(close, c) return c end @@ -24,7 +25,7 @@ isopen(cur::Cursor) = cur.handle != C_NULL function open(txn::Transaction, dbi::DBI) cur_ptr_ref = Ref{Ptr{MDB_cursor}}(C_NULL) mdb_cursor_open(txn, dbi, cur_ptr_ref) - return Cursor(txn, cur_ptr_ref[]) + return Cursor(txn, dbi, cur_ptr_ref[]) end "Wrapper of Cursor `open` for `do` construct" @@ -59,143 +60,12 @@ end "Return the cursor's transaction." transaction(cur::Cursor) = cur.txn -"Return the cursor's database" -function database(cur::Cursor) - dbi = mdb_cursor_dbi(cur) - (dbi == 0) && return nothing - return DBI(dbi, "") -end +"Return the cursor's database." +database(cur::Cursor) = cur.dbi Base.show(io::IO, cur::Cursor) = print(io, "Cursor(", isopen(cur) ? "open" : "closed", ")") -"Type to implement the Iterator interface" -struct LMDBIterator{R} - cur::Cursor - r::R - prefix::Vector{UInt8} -end -struct ReturnKeys{K} end -struct ReturnValues{V} end -struct ReturnBoth{K,V} end -struct ReturnValueSize end - -arcopy(x::Array) = copy(x) -arcopy(x) = x -# process_returns returns (retval, next_op, key_buf). key_buf is the Julia -# buffer the next iteration will fill `mdb_key_ref` from; it must outlive -# the next mdb_cursor_get call. Variants that don't seed a SET_RANGE -# return `nothing` for key_buf. -process_returns(::ReturnKeys{K}, mdb_key_ref, _) where K = arcopy(mdb_unpack(K, mdb_key_ref)), MDB_NEXT, nothing -process_returns(::ReturnValues{V}, _, mdb_val_ref) where V = arcopy(mdb_unpack(V, mdb_val_ref)), MDB_NEXT, nothing -process_returns(::ReturnBoth{K,V}, mdb_key_ref, mdb_val_ref) where {K,V} = arcopy((mdb_unpack(K, mdb_key_ref)) => arcopy(mdb_unpack(V, mdb_val_ref))), MDB_NEXT, nothing -process_returns(::ReturnValueSize, _, mdb_val_ref) = mdb_val_ref[].mv_size, MDB_NEXT, nothing -function init_values(d::LMDBIterator) - if !isempty(d.prefix) - # The Ref is initialised lazily inside `iterate`, under - # `GC.@preserve key_buf`, so the pointer into `d.prefix` is - # only taken when the buffer is provably alive. - k = Ref(MDBValue()) - return k, Ref(MDBValue()), MDB_SET_RANGE, d.prefix - else - return Ref(MDBValue()), Ref(MDBValue()), MDB_FIRST, nothing - end -end - -Base.iterate(iter::LMDBIterator) = Base.iterate(iter, init_values(iter)) - -"Iterate over database" -function Base.iterate(iter::LMDBIterator, refs) - mdb_key_ref, mdb_val_ref, cursor_op, key_buf = refs - - # If we have a key buffer (SET_RANGE seed or DirectoryLister rewrite), - # fill `mdb_key_ref` with its pointer under GC.@preserve — the pointer - # is only valid while `key_buf` is rooted, and ccall extends that - # rooting through the call. unchecked_* because the iterator branches - # on MDB_NOTFOUND itself. - ret = GC.@preserve key_buf begin - key_buf === nothing || (mdb_key_ref[] = _mdb_val_for(key_buf)) - unchecked_mdb_cursor_get(iter.cur, mdb_key_ref, mdb_val_ref, cursor_op) - end - - if ret == 0 - #Check if we are still in key prefix - if !isempty(iter.prefix) - k = mdb_unpack(Vector{UInt8}, mdb_key_ref) - if any(i->!=(i...),zip(iter.prefix, k)) - return nothing - end - end - pr = process_returns(iter.r, mdb_key_ref, mdb_val_ref) - pr === nothing && return nothing - retval, nextop, next_key_buf = pr - return (retval, (mdb_key_ref, mdb_val_ref, nextop, next_key_buf)) - elseif ret == MDB_NOTFOUND - return nothing - else - throw(LMDBError(ret)) - end -end - -struct DirectoryLister{K} - sep::UInt8 - istart::Int -end -function DirectoryLister(; sep = '/', lprefix=0) - DirectoryLister{String}(UInt8(sep),lprefix+1) -end - -function process_returns(l::DirectoryLister{K}, mdb_key_ref, _) where K - k = mdb_unpack(Vector{UInt8}, mdb_key_ref) - nextsep = findnext(==(l.sep),k,l.istart) - if nextsep === nothing - return arcopy(mdb_unpack(K, mdb_key_ref)), MDB_NEXT, nothing - else - k = copy(k) - resize!(k,nextsep) - # Decode `k` under GC.@preserve — `_mdb_val_for(k)` takes a raw - # pointer into `k` that must outlive the `mdb_unpack` read. - local kout - GC.@preserve k begin - kref = Ref(_mdb_val_for(k)) - kout = arcopy(mdb_unpack(K, kref)) - end - k[end] = k[end]+1 - # Return `k` as the next iteration's key_buf; `iterate` will - # fill `mdb_key_ref` from it under its own GC.@preserve. - return kout, MDB_SET_RANGE, k - end -end - - -Base.IteratorSize(::LMDBIterator) = Base.SizeUnknown() -Base.eltype(::Type{<:LMDBIterator{<:ReturnKeys{K}}}) where K = K -Base.eltype(::Type{<:LMDBIterator{<:ReturnValues{V}}}) where V = V -Base.eltype(::Type{<:LMDBIterator{<:ReturnBoth{K,V}}}) where {K,V} = Pair{K,V} -Base.eltype(::Type{<:LMDBIterator{<:ReturnValueSize}}) = Csize_t - -"Return iterator over keys of uniform, specified type" -function keys(cur::Cursor, ::Type{T}; prefix = UInt8[]) where T - return LMDBIterator(cur, ReturnKeys{T}(), Vector{UInt8}(prefix)) -end - -function Base.values(cur::Cursor, ::Type{T}; prefix = UInt8[]) where T - return LMDBIterator(cur,ReturnValues{T}(),Vector{UInt8}(prefix)) -end - -function Base.iterate(cur::Cursor, ::Type{K}, ::Type{V}) where {K,V} - return Base.iterate(LMDBIterator(cur, ReturnBoth{K,V}()),Vector{UInt8}(prefix)) -end - -"""Retrieve by cursor. - -This function retrieves key/data pairs from the database. -""" -function get(cur::Cursor, key, ::Type{T}, op::MDB_cursor_op=MDB_SET_KEY) where T - val_ref = Ref(MDBValue()) - mdb_cursor_get(cur, key, val_ref, op) - return mdb_unpack(T, val_ref) -end # Populate `key_ref` with `searchkey`'s data. Returns the heap-rooted argument # that the caller must keep alive across the surrounding ccall (use diff --git a/src/dicts.jl b/src/dicts.jl index cb62481..73019d4 100644 --- a/src/dicts.jl +++ b/src/dicts.jl @@ -203,6 +203,19 @@ function Base.pop!(d::LMDBDict{K,V}, k, default) where {K,V} end end +# `pop!(d)` without a key — pops the first entry, mirroring `Base.pop!(::Dict)`. +function Base.pop!(d::LMDBDict{K,V}) where {K,V} + txn_dbi_do(d) do txn, dbi + LMDB.open(txn, dbi) do cur + LMDB.seek!(cur, K) === nothing && + throw(ArgumentError("LMDBDict must be non-empty")) + pair = LMDB.item(cur, K, V) + LMDB.delete!(cur) + return pair + end + end +end + function Base.empty!(d::LMDBDict) txn_dbi_do(d) do txn, dbi LMDB.drop(txn, dbi; delete = false) diff --git a/test/cur.jl b/test/cur.jl index 9d9975c..8993413 100644 --- a/test/cur.jl +++ b/test/cur.jl @@ -22,7 +22,11 @@ module LMDB_CUR try @test 0 == put!(cur, key+1, val*string(key+1)) @test 0 == put!(cur, key, val*string(key)) - @test issetequal(collect(keys(cur, typeof(key))), [11, 10]) + ks = typeof(key)[] + LMDB.walk(cur) do k_ref, _ + push!(ks, LMDB.mdb_unpack(typeof(key), k_ref)) + end + @test issetequal(ks, [11, 10]) finally close(cur) commit(txn) @@ -34,17 +38,15 @@ module LMDB_CUR end @test !isopen(env) - # Block style - environment(dbname) do env # open environment - start(env) do txn # start transaction - open(txn) do dbi # open database - open(txn, dbi) do cur # open cursor - curtxn = transaction(cur) - @test curtxn.handle == txn.handle - curdbi = database(cur) - @test curdbi.handle == dbi.handle - v = get(cur, key, String) - println("Got value for key $(key): $(v)") + # Block style: parent accessors return the actual handles, not synthetic ones. + environment(dbname) do env + start(env) do txn + open(txn) do dbi + open(txn, dbi) do cur + @test transaction(cur) === txn + @test database(cur) === dbi + @test LMDB.seek!(cur, key, typeof(key)) == key + v = LMDB.value(cur, String) @test val*string(key) == v end end diff --git a/test/dict.jl b/test/dict.jl index 6867254..815b5b6 100644 --- a/test/dict.jl +++ b/test/dict.jl @@ -44,8 +44,12 @@ using Test, LMDB @test pop!(d, "z", :missing) === :missing @test pop!(d, "y") === 12.0 @test !haskey(d, "y") - @test LMDB.valuesize(d) == sizeof(Float64)*1 # only "x" left + + # `pop!(d)` (no key) pops the lexicographically-first entry. + @test pop!(d) == ("x" => 5.0) + @test isempty(d) + @test_throws ArgumentError pop!(d) close(d) end From fe773a46081628123edb56757c8747d2b45dbe7f Mon Sep 17 00:00:00 2001 From: Tim Besard Date: Sun, 3 May 2026 20:43:10 +0200 Subject: [PATCH 4/4] Batch LMDBDict bulk updates into a single write transaction. AbstractDict's default merge! / mergewith! / filter! call d[k]=v or delete!(d, k) in a loop, and each one of those opens its own LMDB write txn (with its own commit/fsync). For a 1k-entry merge! that's 1k transactions, which dominates wall time on durable storage. Override the three to land in one txn instead. --- src/dicts.jl | 45 +++++++++++++++++++++++++++++++++++++++++++-- test/dict.jl | 7 +++++++ 2 files changed, 50 insertions(+), 2 deletions(-) diff --git a/src/dicts.jl b/src/dicts.jl index 73019d4..64ad7de 100644 --- a/src/dicts.jl +++ b/src/dicts.jl @@ -224,8 +224,49 @@ function Base.empty!(d::LMDBDict) end # AbstractDict's default implementations of `keys`, `values`, `pairs`, -# `merge!`, `filter!`, `==`, `hash`, `in(::Pair, d)` etc. all kick in for -# free now that `iterate` and `length` are defined. +# `==`, `hash`, `in(::Pair, d)` etc. all kick in for free now that +# `iterate` and `length` are defined. + +# Override the bulk-update fallbacks so they land in a single LMDB write +# txn. AbstractDict's default `merge!` / `mergewith!` / `filter!` call +# `d[k]=v` / `delete!(d,k)` in a loop, and each of those opens its own +# write txn (and commit/fsync) per pair. +function Base.merge!(d::LMDBDict{K,V}, others::AbstractDict...) where {K,V} + txn_dbi_do(d) do txn, dbi + for other in others, (k, v) in other + LMDB.put!(txn, dbi, convert(K, k), convert(V, v)) + end + end + return d +end + +function Base.mergewith!(combine, d::LMDBDict{K,V}, others::AbstractDict...) where {K,V} + txn_dbi_do(d) do txn, dbi + for other in others, (k, v) in other + kk = convert(K, k) + existing = LMDB.tryget(txn, dbi, kk, V) + new = existing === nothing ? convert(V, v) : + convert(V, combine(existing, v)) + LMDB.put!(txn, dbi, kk, new) + end + end + return d +end + +function Base.filter!(f, d::LMDBDict{K,V}) where {K,V} + txn_dbi_do(d) do txn, dbi + to_delete = K[] + LMDB.open(txn, dbi) do cur + LMDB.walk(cur, K, V) do k, v + f(k => v) || push!(to_delete, k) + end + end + for k in to_delete + LMDB.delete!(txn, dbi, k) + end + end + return d +end # --- prefix-scan helpers (LMDB-namespaced; not Base extensions) --- diff --git a/test/dict.jl b/test/dict.jl index 815b5b6..f313af9 100644 --- a/test/dict.jl +++ b/test/dict.jl @@ -198,6 +198,13 @@ using Test, LMDB @test sort(collect(keys(d))) == ["a", "b", "c"] @test d["b"] == 2 + # mergewith! combines existing values via `combine`, falls back + # to the new value when the key is absent. + mergewith!(+, d, Dict("a" => 10, "d" => 4)) + @test d["a"] == 11 + @test d["d"] == 4 + @test d["b"] == 2 + filter!(p -> isodd(p.second), d) @test sort(collect(keys(d))) == ["a", "c"] close(d)