Improve LMDBDict

README.md

@@ -27,7 +27,9 @@ inspect the raw status code.
 
 ### Tier 3 — `LMDBDict`
 
-A persistent `Dict`-like object backed by one LMDB environment + DBI.
+`LMDBDict{K,V} <: AbstractDict{K,V}`, so the standard library does most
+of the work — `merge!`, `filter!`, `pairs`, `==`, `hash`, `keys`,
+`values`, lazy iteration — all come for free:
 
 ```julia
 using LMDB
@@ -37,13 +39,19 @@ d["beta/x"] = Float32[10, 11]
 d["beta/y"] = Float32[12, 13]
 
 @show d["alpha"]
-@show haskey(d, "alpha"), haskey(d, "missing")
-@show keys(d, prefix = "beta/")           # ["beta/x", "beta/y"]
-@show LMDB.list_dirs(d)                   # ["alpha", "beta/"]
+@show haskey(d, "alpha"), haskey(d, "missing")  # missing throws KeyError
+@show length(d)                                  # 3
+for (k, v) in d
+    @show k, v
+end
+@show LMDB.scan_keys(d, prefix = "beta/")       # ["beta/x", "beta/y"]
+@show LMDB.list_dirs(d)                         # ["alpha", "beta/"]
 close(d)
 ```
 
 Constructor kwargs: `mapsize`, `readers`, `dbs`, `readonly`, `rdahead`.
+The env is opened with `MDB_NOTLS` so multiple read txns can coexist on
+one thread — needed for interleaved reads or task-parallel access.
 
 ### Tier 2 — explicit env / txn / cursor
 

src/LMDB.jl

@@ -1,23 +1,48 @@
 module LMDB
 
-import Base: open, close, getindex, setindex!, put!, pop!, replace!, reset, copy,
-             isopen, count, delete!, keys, get, show, stat
-
-export Environment, create, open, close, sync, set!, unset!, getindex, setindex!, path, info, stat, show,
-       reader_check, reader_list, copy,
-       Transaction, start, abort, commit, reset, renew, environment,
-       DBI, drop, delete!, keys, get, put!, put_reserved!, tryget, replace!, pop!,
-       Cursor, count, transaction, database,
-       seek_first_dup!, seek_last_dup!,
-       next_dup!, prev_dup!, next_nodup!, prev_nodup!,
-       isflagset, isopen,
-       # commonly-needed env / write flags
-       MDB_RDONLY, MDB_NOTLS, MDB_NORDAHEAD, MDB_NOSUBDIR,
-       MDB_NOSYNC, MDB_NOMETASYNC, MDB_WRITEMAP, MDB_NOMEMINIT,
-       MDB_CREATE, MDB_DUPSORT, MDB_INTEGERKEY, MDB_REVERSEKEY,
-       MDB_DUPFIXED, MDB_INTEGERDUP, MDB_REVERSEDUP,
-       MDB_NOOVERWRITE, MDB_NODUPDATA, MDB_APPEND, MDB_RESERVE,
-       LMDBError, LMDBDict
+import Base: open, close, getindex, setindex!, put!, pop!, replace!, reset,
+             isopen, count, delete!, keys, get, show, show, stat, copy,
+             empty!, length, isempty, iterate, haskey
+import Base.Iterators: drop
+
+export
+    # error type + matchers
+    LMDBError, is_notfound, is_keyexist, is_map_full,
+
+    # commonly-needed status codes
+    MDB_NOTFOUND, MDB_KEYEXIST, MDB_MAP_FULL,
+
+    # commonly-needed env flags
+    MDB_RDONLY, MDB_NOTLS, MDB_NORDAHEAD, MDB_NOSUBDIR,
+    MDB_NOSYNC, MDB_NOMETASYNC, MDB_WRITEMAP, MDB_NOMEMINIT,
+
+    # commonly-needed db flags
+    MDB_CREATE, MDB_DUPSORT, MDB_INTEGERKEY, MDB_REVERSEKEY,
+    MDB_DUPFIXED, MDB_INTEGERDUP, MDB_REVERSEDUP,
+
+    # commonly-needed write flags
+    MDB_NOOVERWRITE, MDB_NODUPDATA, MDB_APPEND, MDB_RESERVE,
+
+    # tier 2 — environment
+    Environment, create, environment,
+    sync, set!, unset!, info, stat, path, isopen, isflagset,
+    reader_check, reader_list,
+
+    # tier 2 — transaction
+    Transaction, start, abort, commit, reset, renew,
+
+    # tier 2 — database (DBI)
+    DBI, drop, get, put!, delete!, tryget, replace!,
+
+    # tier 2 — cursor
+    Cursor, count, transaction, database,
+    seek!, seek_last!, seek_range!, next!, prev!,
+    key, value, item, walk,
+    seek_first_dup!, seek_last_dup!,
+    next_dup!, prev_dup!, next_nodup!, prev_nodup!,
+
+    # tier 3
+    LMDBDict
 
 # ---------------------------------------------------------------------------
 # Error type. Defined here so the `@checked` macro can reference it; the

src/common.jl

@@ -59,16 +59,6 @@ function Base.cconvert(::Type{Ptr{MDB_val}}, x::T) where {T}
     MDBArg(Ref(x))
 end
 
-# Private: build an `MDB_val` whose `mv_data` aliases `buf`. The pointer
-# is taken via `unsafe_convert`; the caller MUST keep `buf` alive across
-# any ccall that reads the resulting `MDB_val` (in practice, by wrapping
-# the call site in `GC.@preserve buf ...`). Used by the cursor iterator,
-# which already threads its key buffer through the iteration state for
-# exactly this reason.
-@inline _mdb_val_for(buf::Vector{T}) where {T} =
-    MDB_val(Csize_t(sizeof(T) * length(buf)),
-            Ptr{Cvoid}(Base.unsafe_convert(Ptr{T}, buf)))
-
 """
     mdb_unpack(::Type{T}, ref::Ref{MDB_val}) -> T
 

src/cur.jl

@@ -1,15 +1,16 @@
 """
 A handle to a cursor structure for navigating through a database.
 
-A `Cursor` keeps a reference to its parent `Transaction` to expose it via
-`transaction(cur)` and to keep the txn alive under GC. The cursor's
-finalizer closes any still-open handle.
+A `Cursor` keeps references to its parent `Transaction` and `DBI`, both
+to expose them via `transaction(cur)` / `database(cur)` and to keep the
+txn alive under GC. The cursor's finalizer closes any still-open handle.
 """
 mutable struct Cursor
     handle::Ptr{MDB_cursor}
     txn::Transaction
-    function Cursor(txn::Transaction, h::Ptr{MDB_cursor})
-        c = new(h, txn)
+    dbi::DBI
+    function Cursor(txn::Transaction, dbi::DBI, h::Ptr{MDB_cursor})
+        c = new(h, txn, dbi)
         finalizer(close, c)
         return c
     end
@@ -24,7 +25,7 @@ isopen(cur::Cursor) = cur.handle != C_NULL
 function open(txn::Transaction, dbi::DBI)
     cur_ptr_ref = Ref{Ptr{MDB_cursor}}(C_NULL)
     mdb_cursor_open(txn, dbi, cur_ptr_ref)
-    return Cursor(txn, cur_ptr_ref[])
+    return Cursor(txn, dbi, cur_ptr_ref[])
 end
 
 "Wrapper of Cursor `open` for `do` construct"
@@ -59,143 +60,12 @@ end
 "Return the cursor's transaction."
 transaction(cur::Cursor) = cur.txn
 
-"Return the cursor's database"
-function database(cur::Cursor)
-    dbi = mdb_cursor_dbi(cur)
-    (dbi == 0) && return nothing
-    return DBI(dbi, "")
-end
+"Return the cursor's database."
+database(cur::Cursor) = cur.dbi
 
 Base.show(io::IO, cur::Cursor) =
     print(io, "Cursor(", isopen(cur) ? "open" : "closed", ")")
 
-"Type to implement the Iterator interface"
-struct LMDBIterator{R}
-   cur::Cursor
-   r::R
-   prefix::Vector{UInt8}
-end
-struct ReturnKeys{K} end
-struct ReturnValues{V} end
-struct ReturnBoth{K,V} end
-struct ReturnValueSize end
-
-arcopy(x::Array) = copy(x)
-arcopy(x) = x
-# process_returns returns (retval, next_op, key_buf). key_buf is the Julia
-# buffer the next iteration will fill `mdb_key_ref` from; it must outlive
-# the next mdb_cursor_get call. Variants that don't seed a SET_RANGE
-# return `nothing` for key_buf.
-process_returns(::ReturnKeys{K}, mdb_key_ref, _) where K = arcopy(mdb_unpack(K, mdb_key_ref)), MDB_NEXT, nothing
-process_returns(::ReturnValues{V}, _, mdb_val_ref) where V = arcopy(mdb_unpack(V, mdb_val_ref)), MDB_NEXT, nothing
-process_returns(::ReturnBoth{K,V}, mdb_key_ref, mdb_val_ref) where {K,V} = arcopy((mdb_unpack(K, mdb_key_ref)) => arcopy(mdb_unpack(V, mdb_val_ref))), MDB_NEXT, nothing
-process_returns(::ReturnValueSize, _, mdb_val_ref) = mdb_val_ref[].mv_size, MDB_NEXT, nothing
-function init_values(d::LMDBIterator)
-    if !isempty(d.prefix)
-        # The Ref is initialised lazily inside `iterate`, under
-        # `GC.@preserve key_buf`, so the pointer into `d.prefix` is
-        # only taken when the buffer is provably alive.
-        k = Ref(MDBValue())
-        return k, Ref(MDBValue()), MDB_SET_RANGE, d.prefix
-    else
-        return Ref(MDBValue()), Ref(MDBValue()), MDB_FIRST, nothing
-    end
-end
-
-Base.iterate(iter::LMDBIterator) = Base.iterate(iter, init_values(iter))
-
-"Iterate over database"
-function Base.iterate(iter::LMDBIterator, refs)
-    mdb_key_ref, mdb_val_ref, cursor_op, key_buf = refs
-
-    # If we have a key buffer (SET_RANGE seed or DirectoryLister rewrite),
-    # fill `mdb_key_ref` with its pointer under GC.@preserve — the pointer
-    # is only valid while `key_buf` is rooted, and ccall extends that
-    # rooting through the call. unchecked_* because the iterator branches
-    # on MDB_NOTFOUND itself.
-    ret = GC.@preserve key_buf begin
-        key_buf === nothing || (mdb_key_ref[] = _mdb_val_for(key_buf))
-        unchecked_mdb_cursor_get(iter.cur, mdb_key_ref, mdb_val_ref, cursor_op)
-    end
-
-    if ret == 0
-        #Check if we are still in key prefix
-        if !isempty(iter.prefix)
-            k = mdb_unpack(Vector{UInt8}, mdb_key_ref)
-            if any(i->!=(i...),zip(iter.prefix, k))
-                return nothing
-            end
-        end
-        pr = process_returns(iter.r, mdb_key_ref, mdb_val_ref)
-        pr === nothing && return nothing
-        retval, nextop, next_key_buf = pr
-        return (retval, (mdb_key_ref, mdb_val_ref, nextop, next_key_buf))
-    elseif ret == MDB_NOTFOUND
-        return nothing
-    else
-        throw(LMDBError(ret))
-    end
-end
-
-struct DirectoryLister{K}
-    sep::UInt8
-    istart::Int
-end
-function DirectoryLister(; sep = '/', lprefix=0)
-    DirectoryLister{String}(UInt8(sep),lprefix+1)
-end
-
-function process_returns(l::DirectoryLister{K}, mdb_key_ref, _) where K
-    k = mdb_unpack(Vector{UInt8}, mdb_key_ref)
-    nextsep = findnext(==(l.sep),k,l.istart)
-    if nextsep === nothing
-        return arcopy(mdb_unpack(K, mdb_key_ref)), MDB_NEXT, nothing
-    else
-        k = copy(k)
-        resize!(k,nextsep)
-        # Decode `k` under GC.@preserve — `_mdb_val_for(k)` takes a raw
-        # pointer into `k` that must outlive the `mdb_unpack` read.
-        local kout
-        GC.@preserve k begin
-            kref = Ref(_mdb_val_for(k))
-            kout = arcopy(mdb_unpack(K, kref))
-        end
-        k[end] = k[end]+1
-        # Return `k` as the next iteration's key_buf; `iterate` will
-        # fill `mdb_key_ref` from it under its own GC.@preserve.
-        return kout, MDB_SET_RANGE, k
-    end
-end
-
-
-Base.IteratorSize(::LMDBIterator) = Base.SizeUnknown()
-Base.eltype(::Type{<:LMDBIterator{<:ReturnKeys{K}}}) where K = K
-Base.eltype(::Type{<:LMDBIterator{<:ReturnValues{V}}}) where V = V
-Base.eltype(::Type{<:LMDBIterator{<:ReturnBoth{K,V}}}) where {K,V} = Pair{K,V}
-Base.eltype(::Type{<:LMDBIterator{<:ReturnValueSize}}) = Csize_t
-
-"Return iterator over keys of uniform, specified type"
-function keys(cur::Cursor, ::Type{T}; prefix = UInt8[]) where T
-    return LMDBIterator(cur, ReturnKeys{T}(), Vector{UInt8}(prefix))
-end
-
-function Base.values(cur::Cursor, ::Type{T}; prefix = UInt8[]) where T
-    return LMDBIterator(cur,ReturnValues{T}(),Vector{UInt8}(prefix))
-end
-
-function Base.iterate(cur::Cursor, ::Type{K}, ::Type{V}) where {K,V}
-    return Base.iterate(LMDBIterator(cur, ReturnBoth{K,V}()),Vector{UInt8}(prefix))
-end
-
-"""Retrieve by cursor.
-
-This function retrieves key/data pairs from the database.
-"""
-function get(cur::Cursor, key, ::Type{T}, op::MDB_cursor_op=MDB_SET_KEY) where T
-    val_ref = Ref(MDBValue())
-    mdb_cursor_get(cur, key, val_ref, op)
-    return mdb_unpack(T, val_ref)
-end
 
 # Populate `key_ref` with `searchkey`'s data. Returns the heap-rooted argument
 # that the caller must keep alive across the surrounding ccall (use
@@ -429,9 +299,10 @@ Walk every entry the cursor visits, calling
 starts at the first key (`MDB_FIRST`) when `from === nothing`, otherwise at
 the smallest key `>= from` (`MDB_SET_RANGE`).
 
-Inside `f`, `key_ref[]` and `val_ref[]` point into LMDB-owned memory and are
-valid only for the duration of the surrounding transaction; copy out anything
-you want to retain.
+Iteration stops when `f` returns `false` (any other return value, including
+`nothing`, continues). Inside `f`, `key_ref[]` and `val_ref[]` point into
+LMDB-owned memory and are valid only for the duration of the surrounding
+transaction; copy out anything you want to retain.
 """
 function walk(f, cur::Cursor; from = nothing)
     key_ref = Ref(MDBValue())