From f5735d30607ad7f727f3c898ffcb01a662460cc6 Mon Sep 17 00:00:00 2001 From: Tim Besard Date: Sun, 3 May 2026 18:17:55 +0200 Subject: [PATCH] Switch typed-read extension point to Base.read(::MDBValueIO, T). MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Replace `LMDB.mdb_unpack(::Type{T}, ::Ref{MDB_val})` — the package's typed-read customization point — with a plain `IO` view over the LMDB-owned `MDB_val`. Users now plug in custom decoders by defining the idiomatic `Base.read(io::IO, ::Type{MyType})`, which works against any byte stream and is the convention every other Julia decoder uses. struct PrefixedBlob end function Base.read(io::IO, ::Type{PrefixedBlob}) skip(io, 8) return read(io, String) end LMDB.tryget(txn, dbi, key, PrefixedBlob) `MDBValueIO <: IO` exposes `position` / `seek` / `seekstart` / `seekend` / `skip` / `bytesavailable` / `eof` and overrides `unsafe_read` so Base's primitive `read(io, T)` for `Int8…Float64`/`Bool`/`Char`/`Ptr` inherits zero-allocation. The default `read(io, String)` and `read(io, Vector{E})` consume the rest of the buffer and copy out. Internal call sites in `cur.jl` / `dbi.jl` / `dicts.jl` switch from `mdb_unpack(T, ref)` to `read(MDBValueIO(ref[]), T)`. The package exposes `LMDB.MDBValueIO` for users who override `Base.read`. `mdb_unpack` is dropped — breaking change for the (rare) caller who defined their own `mdb_unpack` method. Preserves PR #52's GC-safety fix: `common.jl` keeps the deferred-pointer-extraction pattern from #52 (the upstream commit this is based on pre-dates that fix). `MDBValue(::String)` and the eager-extraction variants stay deleted; `cconvert(::Ptr{MDB_val}, x)` returns an `MDBArg` carrier with an uninitialized box, and `unsafe_convert(::Ptr{MDB_val}, ::MDBArg{D})` does the extraction inside ccall's preserve window. Tested locally on Julia 1.10, 1.11, 1.12, 1.13. Co-Authored-By: Claude Opus 4.7 --- src/LMDB.jl | 36 ++++++++++ src/common.jl | 159 +++++++++++++++++++++++++++++++++++--------- src/cur.jl | 46 ++++++------- src/dbi.jl | 16 +++-- src/dicts.jl | 10 +-- test/Project.toml | 2 + test/cur.jl | 14 ++-- test/integration.jl | 26 +++----- 8 files changed, 222 insertions(+), 87 deletions(-) diff --git a/src/LMDB.jl b/src/LMDB.jl index 0fa17ab..601cdde 100644 --- a/src/LMDB.jl +++ b/src/LMDB.jl @@ -63,6 +63,42 @@ show(io::IO, err::LMDBError) = print(io, "Code[$(err.code)]: $(err.msg)") "Throw an `LMDBError` if `code` is non-zero. Returns `code` otherwise." @inline check(code) = iszero(code) ? code : throw(LMDBError(code)) +""" + is_notfound(err::LMDBError) -> Bool + +`true` if `err.code == MDB_NOTFOUND` (LMDB's "key not present" status). + +Use this to recover from a missing key when the lookup is not point-typed +(otherwise `tryget` / `get(..., default)` are simpler): + +```julia +try + LMDB.get(txn, dbi, "key", String) +catch e + e isa LMDBError && is_notfound(e) || rethrow() + # treat as missing +end +``` +""" +is_notfound + +""" + is_keyexist(err::LMDBError) -> Bool + +`true` if `err.code == MDB_KEYEXIST`. Raised by `put!` with `MDB_NOOVERWRITE` +or `MDB_NODUPDATA` when the key (or duplicate) is already present. +""" +is_keyexist + +""" + is_map_full(err::LMDBError) -> Bool + +`true` if `err.code == MDB_MAP_FULL`. Raised when a write txn would exceed +the environment's `MapSize`. The remedy is to grow `mapsize` (which can be +done after `close(env)` without rewriting the database). +""" +is_map_full + # --------------------------------------------------------------------------- # Tier 1 — raw bindings, types, constants. Public-but-unexported. # diff --git a/src/common.jl b/src/common.jl index f4c9715..d5b84c1 100644 --- a/src/common.jl +++ b/src/common.jl @@ -60,48 +60,145 @@ function Base.cconvert(::Type{Ptr{MDB_val}}, x::T) where {T} end """ - mdb_unpack(::Type{T}, ref::Ref{MDB_val}) -> T + MDBValueIO(v::MDB_val) <: IO + MDBValueIO(ref::Ref{MDB_val}) <: IO -Decode an `MDB_val` (size + raw `mv_data` pointer) into a Julia value of -type `T`. Called by `tryget` / `get` / cursor accessors after a -successful read. Default methods cover `String`, `Vector{E}` for any -bitstype `E`, and any bitstype scalar; all of them copy out so the -returned value is safe to keep past the producing transaction. +A read-only `IO` view over an LMDB-owned `MDB_val`. Wraps the +`(mv_data, mv_size)` pair as a positionable byte stream so the +package's typed-read path is the standard `Base.read(io, T)`. -This is the package's customization point for typed reads — analogous -to heed's `BytesDecode<'txn>` trait. To plug in a custom value -representation (e.g. skip a framing prefix, parse a tagged buffer, -build a non-bitstype struct), define a method on a marker type: +Any `T` for which `Base.read(io::IO, ::Type{T})` is defined can be +passed to `tryget` / `get` / `key` / `value` / `item` / typed `walk` / +`pop!` / `replace!`. Out of the box this covers everything Base ships: +the primitive numeric types (`Int8`/…/`Int128`, `Float16`/…/`Float64`, +`Bool`, `Char`, `Ptr{T}`) plus `String`, all zero-allocation thanks to +the `@inline` `unsafe_read` override below. The package adds two more +overloads — `Vector{E}` for any bitstype `E` and `UInt8` — that +consume the remaining buffer in a single copy. + +To plug in a custom representation — including bitstype structs that +Base's primitive reads don't cover — define a single `Base.read` +method on your own type. Defining it on the abstract `IO` is the +idiomatic Julia form and keeps the decoder portable to other byte +sources: struct PrefixedBlob end - function LMDB.mdb_unpack(::Type{PrefixedBlob}, ref::Ref{LMDB.MDB_val}) - v = ref[]; sz = Int(v.mv_size) - sz < 8 && return UInt8[] - out = Vector{UInt8}(undef, sz - 8) - unsafe_copyto!(pointer(out), - Ptr{UInt8}(v.mv_data) + 8, sz - 8) - out + function Base.read(io::IO, ::Type{PrefixedBlob}) + bytesavailable(io) < 8 && return UInt8[] + skip(io, 8) + return read(io, Vector{UInt8}) end LMDB.tryget(txn, dbi, key, PrefixedBlob) # → Union{Vector{UInt8}, Nothing} -The `mv_data` pointer is into LMDB's mmap and is only valid for the -producing transaction's lifetime. Custom unpack methods must copy what -they want to keep, exactly as the default `Vector{E}` method does. +For an `isbitstype` struct `T`, the one-liner is the standard Base +pattern: + + Base.read(io::IO, ::Type{T}) = read!(io, Ref{T}())[] + +This is the analogue of heed's `BytesDecode<'txn>` trait, expressed +through Julia's existing IO extension point rather than a bespoke +function. + +The underlying buffer points into LMDB's mmap and is **only valid for +the producing transaction's lifetime** — copy out anything you want to +retain past commit/abort. The default `String` and `Vector{E}` reads +both copy. """ -mdb_unpack(::Type{T}, mdb_val_ref::Ref{MDB_val}) where {T} = _mdb_unpack(T, mdb_val_ref[]) -function _mdb_unpack(::Type{T}, mdb_val::MDB_val) where {T <: String} - unsafe_string(convert(Ptr{UInt8}, mdb_val.mv_data), mdb_val.mv_size) +mutable struct MDBValueIO <: IO + ptr::Ptr{UInt8} + size::Int + pos::Int +end +@inline MDBValueIO(v::MDB_val) = + MDBValueIO(Ptr{UInt8}(v.mv_data), Int(v.mv_size), 0) +@inline MDBValueIO(ref::Ref{MDB_val}) = MDBValueIO(ref[]) + +# IO interface primitives. Defining `read(::MDBValueIO, ::Type{UInt8})` +# and `unsafe_read(::MDBValueIO, ::Ptr{UInt8}, ::UInt)` is enough to +# inherit Base's generic numeric and array reads; the rest are +# convenience getters. +@inline Base.isreadable(::MDBValueIO) = true +@inline Base.iswritable(::MDBValueIO) = false +@inline Base.eof(io::MDBValueIO) = io.pos >= io.size +@inline Base.position(io::MDBValueIO) = io.pos +@inline Base.bytesavailable(io::MDBValueIO) = io.size - io.pos +@inline function Base.seek(io::MDBValueIO, n::Integer) + io.pos = clamp(Int(n), 0, io.size) + return io +end +@inline Base.seekstart(io::MDBValueIO) = (io.pos = 0; io) +@inline Base.seekend(io::MDBValueIO) = (io.pos = io.size; io) +@inline function Base.skip(io::MDBValueIO, n::Integer) + io.pos = clamp(io.pos + Int(n), 0, io.size) + return io +end + +@inline function Base.unsafe_read(io::MDBValueIO, dst::Ptr{UInt8}, n::UInt) + p = io.pos + p + n <= io.size || throw(EOFError()) + GC.@preserve io unsafe_copyto!(dst, io.ptr + p, n) + io.pos = p + Int(n) + return nothing +end + +# Override Base's `@noinline unsafe_read(::IO, ::Ref{T}, ::Integer)` +# (base/io.jl, "mark noinline to ensure ref is gc-rooted somewhere by the +# caller"). The barrier is correct in general but blocks SROA from +# eliminating the `Ref{T}(0)` that Base's `read(::IO, T::Union{Int16,…})` +# allocates. Our copy is bytewise into Julia memory and needs no GC root, +# so we inline through the Ref→Ptr conversion and let escape analysis +# elide the box. This is what makes plain `read(io, T)` for primitive +# numeric `T` allocation-free without needing per-type fast paths. +@inline Base.unsafe_read(io::MDBValueIO, p::Ref{T}, n::Integer) where {T} = + unsafe_read(io, Base.unsafe_convert(Ref{T}, p)::Ptr, n) + +@inline Base.unsafe_read(io::MDBValueIO, p::Ptr, n::Integer) = + unsafe_read(io, convert(Ptr{UInt8}, p), convert(UInt, n)) + +@inline function Base.read(io::MDBValueIO, ::Type{UInt8}) + p = io.pos + p < io.size || throw(EOFError()) + b = unsafe_load(io.ptr + p) + io.pos = p + 1 + return b end -function _mdb_unpack(::Type{V}, mdb_val::MDB_val) where {T, V <: Vector{T}} - # The MDB_val data points into the LMDB-owned mmap and is only valid for - # the lifetime of the transaction. Copy out so the returned Vector owns - # its memory and is safe to retain past commit/abort (issue #41). - src = unsafe_wrap(Array, convert(Ptr{UInt8}, mdb_val.mv_data), mdb_val.mv_size) - copy(reinterpret(T, src)) + +# Note: we deliberately don't define a `Base.read(io::MDBValueIO, ::Type{T}) +# where T` catch-all for `isbitstype(T)`. Such a generic conflicts with +# users defining the idiomatic `Base.read(io::IO, ::Type{MyT})` — Julia +# treats `(MDBValueIO, Type{T} where T)` and `(IO, Type{MyT})` as +# unordered (one is more specific in arg1, the other in arg2), so the +# call ambiguates. Instead, we rely on Base's existing +# `read(::IO, T::Union{Int8,…,Float64,Bool,Char,Ptr})` specialisations +# for the well-known primitives; our `@inline unsafe_read` above lets +# the optimiser elide the `Ref{T}` Base allocates internally, so they +# stay zero-allocation. User-defined types — including `isbitstype` +# structs — just need a one-line `Base.read(io::IO, ::Type{MyT})` method +# defined wherever the user owns the type. + +# Whole-blob defaults: read everything from the current position to the end. +# These mirror what users intuitively expect when calling `read(io, T)` +# against an LMDB-backed value (`String` and `Vector{E}` consume the rest +# of the buffer). +@inline function Base.read(io::MDBValueIO, ::Type{String}) + p = io.pos + n = io.size - p + s = GC.@preserve io unsafe_string(io.ptr + p, n) + io.pos = io.size + return s end -function _mdb_unpack(::Type{T}, mdb_val::MDB_val) where {T} - unsafe_load(convert(Ptr{T}, mdb_val.mv_data)) + +@inline function Base.read(io::MDBValueIO, ::Type{Vector{T}}) where {T} + p = io.pos + nbytes = io.size - p + n, r = divrem(nbytes, sizeof(T)) + iszero(r) || throw(ArgumentError( + "MDB value byte size $(nbytes) is not a multiple of sizeof($T)=$(sizeof(T))")) + out = Vector{T}(undef, n) + GC.@preserve io out unsafe_copyto!(Ptr{UInt8}(pointer(out)), io.ptr + p, nbytes) + io.pos = io.size + return out end diff --git a/src/cur.jl b/src/cur.jl index 190d054..0d8477d 100644 --- a/src/cur.jl +++ b/src/cur.jl @@ -115,7 +115,7 @@ if the database is empty. Wraps `MDB_FIRST`. function seek!(cur::Cursor, ::Type{T}=Vector{UInt8}) where T key_ref = Ref(MDBValue()); val_ref = Ref(MDBValue()) _cursor_seek!(cur, key_ref, val_ref, MDB_FIRST, nothing) || return nothing - return mdb_unpack(T, key_ref) + return Base.read(MDBValueIO(key_ref[]), T) end """ @@ -127,7 +127,7 @@ matched key as `T`, or `nothing` if no such entry exists. Wraps `MDB_SET_KEY`. function seek!(cur::Cursor, searchkey, ::Type{T}=Vector{UInt8}) where T key_ref = Ref(MDBValue()); val_ref = Ref(MDBValue()) _cursor_seek!(cur, key_ref, val_ref, MDB_SET_KEY, searchkey) || return nothing - return mdb_unpack(T, key_ref) + return Base.read(MDBValueIO(key_ref[]), T) end """ @@ -139,7 +139,7 @@ if the database is empty. Wraps `MDB_LAST`. function seek_last!(cur::Cursor, ::Type{T}=Vector{UInt8}) where T key_ref = Ref(MDBValue()); val_ref = Ref(MDBValue()) _cursor_seek!(cur, key_ref, val_ref, MDB_LAST, nothing) || return nothing - return mdb_unpack(T, key_ref) + return Base.read(MDBValueIO(key_ref[]), T) end """ @@ -151,7 +151,7 @@ Position the cursor at the smallest key `>= key`. Returns the matched key as function seek_range!(cur::Cursor, searchkey, ::Type{T}=Vector{UInt8}) where T key_ref = Ref(MDBValue()); val_ref = Ref(MDBValue()) _cursor_seek!(cur, key_ref, val_ref, MDB_SET_RANGE, searchkey) || return nothing - return mdb_unpack(T, key_ref) + return Base.read(MDBValueIO(key_ref[]), T) end """ @@ -163,7 +163,7 @@ the cursor moved past the last entry. Wraps `MDB_NEXT`. function next!(cur::Cursor, ::Type{T}=Vector{UInt8}) where T key_ref = Ref(MDBValue()); val_ref = Ref(MDBValue()) _cursor_seek!(cur, key_ref, val_ref, MDB_NEXT, nothing) || return nothing - return mdb_unpack(T, key_ref) + return Base.read(MDBValueIO(key_ref[]), T) end """ @@ -175,7 +175,7 @@ if the cursor moved past the first entry. Wraps `MDB_PREV`. function prev!(cur::Cursor, ::Type{T}=Vector{UInt8}) where T key_ref = Ref(MDBValue()); val_ref = Ref(MDBValue()) _cursor_seek!(cur, key_ref, val_ref, MDB_PREV, nothing) || return nothing - return mdb_unpack(T, key_ref) + return Base.read(MDBValueIO(key_ref[]), T) end """ @@ -187,7 +187,7 @@ Return the key at the cursor's current position, decoded as `K`. Wraps function key(cur::Cursor, ::Type{K}=Vector{UInt8}) where K key_ref = Ref(MDBValue()); val_ref = Ref(MDBValue()) mdb_cursor_get(cur, key_ref, val_ref, MDB_GET_CURRENT) - return mdb_unpack(K, key_ref) + return Base.read(MDBValueIO(key_ref[]), K) end """ @@ -199,7 +199,7 @@ Return the value at the cursor's current position, decoded as `V`. Wraps function value(cur::Cursor, ::Type{V}=Vector{UInt8}) where V key_ref = Ref(MDBValue()); val_ref = Ref(MDBValue()) mdb_cursor_get(cur, key_ref, val_ref, MDB_GET_CURRENT) - return mdb_unpack(V, val_ref) + return Base.read(MDBValueIO(val_ref[]), V) end """ @@ -211,7 +211,7 @@ Return the (key => value) pair at the cursor's current position. Wraps function item(cur::Cursor, ::Type{K}=Vector{UInt8}, ::Type{V}=Vector{UInt8}) where {K,V} key_ref = Ref(MDBValue()); val_ref = Ref(MDBValue()) mdb_cursor_get(cur, key_ref, val_ref, MDB_GET_CURRENT) - return mdb_unpack(K, key_ref) => mdb_unpack(V, val_ref) + return Base.read(MDBValueIO(key_ref[]), K) => Base.read(MDBValueIO(val_ref[]), V) end """ @@ -224,7 +224,7 @@ value as `V`, or `nothing` if the current entry has no duplicates. Wraps function seek_first_dup!(cur::Cursor, ::Type{V}=Vector{UInt8}) where V key_ref = Ref(MDBValue()); val_ref = Ref(MDBValue()) _cursor_seek!(cur, key_ref, val_ref, MDB_FIRST_DUP, nothing) || return nothing - return mdb_unpack(V, val_ref) + return Base.read(MDBValueIO(val_ref[]), V) end """ @@ -237,7 +237,7 @@ value as `V`, or `nothing` if the current entry has no duplicates. Wraps function seek_last_dup!(cur::Cursor, ::Type{V}=Vector{UInt8}) where V key_ref = Ref(MDBValue()); val_ref = Ref(MDBValue()) _cursor_seek!(cur, key_ref, val_ref, MDB_LAST_DUP, nothing) || return nothing - return mdb_unpack(V, val_ref) + return Base.read(MDBValueIO(val_ref[]), V) end """ @@ -250,7 +250,7 @@ as `V`, or `nothing` if there are no more duplicates of this key. Wraps function next_dup!(cur::Cursor, ::Type{V}=Vector{UInt8}) where V key_ref = Ref(MDBValue()); val_ref = Ref(MDBValue()) _cursor_seek!(cur, key_ref, val_ref, MDB_NEXT_DUP, nothing) || return nothing - return mdb_unpack(V, val_ref) + return Base.read(MDBValueIO(val_ref[]), V) end """ @@ -263,7 +263,7 @@ as `V`, or `nothing` if there are no earlier duplicates. Wraps function prev_dup!(cur::Cursor, ::Type{V}=Vector{UInt8}) where V key_ref = Ref(MDBValue()); val_ref = Ref(MDBValue()) _cursor_seek!(cur, key_ref, val_ref, MDB_PREV_DUP, nothing) || return nothing - return mdb_unpack(V, val_ref) + return Base.read(MDBValueIO(val_ref[]), V) end """ @@ -276,7 +276,7 @@ key. Wraps `MDB_NEXT_NODUP`. function next_nodup!(cur::Cursor, ::Type{K}=Vector{UInt8}) where K key_ref = Ref(MDBValue()); val_ref = Ref(MDBValue()) _cursor_seek!(cur, key_ref, val_ref, MDB_NEXT_NODUP, nothing) || return nothing - return mdb_unpack(K, key_ref) + return Base.read(MDBValueIO(key_ref[]), K) end """ @@ -288,7 +288,7 @@ Move to the last entry of the previous key. Returns the new key as `K`, or function prev_nodup!(cur::Cursor, ::Type{K}=Vector{UInt8}) where K key_ref = Ref(MDBValue()); val_ref = Ref(MDBValue()) _cursor_seek!(cur, key_ref, val_ref, MDB_PREV_NODUP, nothing) || return nothing - return mdb_unpack(K, key_ref) + return Base.read(MDBValueIO(key_ref[]), K) end """ @@ -329,19 +329,19 @@ end Typed overload of `walk` mirroring the `tryget(txn, dbi, key, T)` / `key(cur, T)` / `seek!(cur, key, T)` shape used elsewhere in tier-2. -Decodes each key and value through `mdb_unpack(K, …)` / -`mdb_unpack(V, …)` before passing them to `f(k::K, v::V)`. Same stop -contract as the raw form: `f` returning `false` halts iteration. +Decodes each key and value through `read(::MDBValueIO, K)` / +`read(::MDBValueIO, V)` before passing them to `f(k::K, v::V)`. Same +stop contract as the raw form: `f` returning `false` halts iteration. -Define a custom `mdb_unpack(::Type{T}, ::Ref{MDB_val})` method on a -marker type to control what gets decoded — e.g. a `(atime, size)` -tuple from a framed value, or a zero-copy view. This is the iteration -counterpart to `tryget(..., T)`. +Define a custom `Base.read(io::LMDB.MDBValueIO, ::Type{T})` to control +what gets decoded — e.g. a `(atime, size)` tuple from a framed value, +or a zero-copy view. This is the iteration counterpart to +`tryget(..., T)`. """ function walk(f, cur::Cursor, ::Type{K}, ::Type{V} = K; from = nothing) where {K, V} walk(cur; from) do k_ref, v_ref - f(mdb_unpack(K, k_ref), mdb_unpack(V, v_ref)) + f(Base.read(MDBValueIO(k_ref[]), K), Base.read(MDBValueIO(v_ref[]), V)) end end diff --git a/src/dbi.jl b/src/dbi.jl index 00430a8..623d47f 100644 --- a/src/dbi.jl +++ b/src/dbi.jl @@ -104,9 +104,11 @@ key was not present. Other LMDB errors propagate as `LMDBError`. The Bool-return / no-throw-on-miss shape matches `Base.delete!`'s "if any" contract and the dominant LMDB-binding convention (heed, py-lmdb, -lmdb-js, lmdbxx).""" -function delete!(txn::Transaction, dbi::DBI, key, val=C_NULL) - val_arg = val === C_NULL ? MDBValue() : val +lmdb-js, lmdbxx). +""" +delete!(txn::Transaction, dbi::DBI, key) = _delete!(txn, dbi, key, MDBValue()) +delete!(txn::Transaction, dbi::DBI, key, val) = _delete!(txn, dbi, key, val) +function _delete!(txn::Transaction, dbi::DBI, key, val_arg) ret = unchecked_mdb_del(txn, dbi, key, val_arg) ret == MDB_NOTFOUND && return false iszero(ret) || throw(LMDBError(ret)) @@ -136,20 +138,20 @@ function stat(txn::Transaction, dbi::DBI) end """Get an item from a database. Throws `LMDBError` if `key` is not present.""" -function get(txn::Transaction, dbi::DBI, key, ::Type{T}) where T +@inline function get(txn::Transaction, dbi::DBI, key, ::Type{T}) where T val_ref = Ref(MDBValue()) mdb_get(txn, dbi, key, val_ref) - return mdb_unpack(T, val_ref) + return Base.read(MDBValueIO(val_ref[]), T) end """Get an item from a database, returning `nothing` if `key` is not present. Use this in preference to `get` + try/catch when a missing key is expected.""" -function tryget(txn::Transaction, dbi::DBI, key, ::Type{T}) where T +@inline function tryget(txn::Transaction, dbi::DBI, key, ::Type{T}) where T val_ref = Ref(MDBValue()) ret = unchecked_mdb_get(txn, dbi, key, val_ref) ret == MDB_NOTFOUND && return nothing iszero(ret) || throw(LMDBError(ret)) - return mdb_unpack(T, val_ref) + return Base.read(MDBValueIO(val_ref[]), T) end """Get an item from a database, returning `default` if `key` is not present. diff --git a/src/dicts.jl b/src/dicts.jl index 64ad7de..ac21a63 100644 --- a/src/dicts.jl +++ b/src/dicts.jl @@ -113,7 +113,8 @@ function _iter_step(::LMDBDict{K,V}, txn::Transaction, cur::Cursor, LMDB.abort(txn) throw(LMDBError(ret)) end - return (LMDB.mdb_unpack(K, k_ref) => LMDB.mdb_unpack(V, v_ref), (txn, cur)) + return (Base.read(LMDB.MDBValueIO(k_ref[]), K) => + Base.read(LMDB.MDBValueIO(v_ref[]), V), (txn, cur)) end Base.IteratorSize(::Type{<:LMDBDict}) = Base.HasLength() @@ -282,7 +283,8 @@ function scan(d::LMDBDict{K,V}; prefix = UInt8[]) where {K,V} out = Pair{K,V}[] cursor_do(d, readonly = true) do cur _walk_prefix(cur, bprefix) do k_ref, v_ref - push!(out, mdb_unpack(K, k_ref) => mdb_unpack(V, v_ref)) + push!(out, Base.read(MDBValueIO(k_ref[]), K) => + Base.read(MDBValueIO(v_ref[]), V)) end end return out @@ -298,7 +300,7 @@ function scan_keys(d::LMDBDict{K}; prefix = UInt8[]) where K out = K[] cursor_do(d, readonly = true) do cur _walk_prefix(cur, bprefix) do k_ref, _ - push!(out, mdb_unpack(K, k_ref)) + push!(out, Base.read(MDBValueIO(k_ref[]), K)) end end return out @@ -314,7 +316,7 @@ function scan_values(d::LMDBDict{K,V}; prefix = UInt8[]) where {K,V} out = V[] cursor_do(d, readonly = true) do cur _walk_prefix(cur, bprefix) do _, v_ref - push!(out, mdb_unpack(V, v_ref)) + push!(out, Base.read(MDBValueIO(v_ref[]), V)) end end return out diff --git a/test/Project.toml b/test/Project.toml index 0c36332..2f9f71c 100644 --- a/test/Project.toml +++ b/test/Project.toml @@ -1,2 +1,4 @@ [deps] +BenchmarkTools = "6e4b80f9-dd63-53aa-95a3-0cdb28fa8baf" +LMDB = "11f193de-5e89-5f17-923a-7d207d56daf9" Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40" diff --git a/test/cur.jl b/test/cur.jl index 8993413..c4c2a84 100644 --- a/test/cur.jl +++ b/test/cur.jl @@ -24,7 +24,7 @@ module LMDB_CUR @test 0 == put!(cur, key, val*string(key)) ks = typeof(key)[] LMDB.walk(cur) do k_ref, _ - push!(ks, LMDB.mdb_unpack(typeof(key), k_ref)) + push!(ks, read(LMDB.MDBValueIO(k_ref[]), typeof(key))) end @test issetequal(ks, [11, 10]) finally @@ -86,26 +86,26 @@ module LMDB_CUR # walk over everything ks = String[] LMDB.walk(cur) do k_ref, _ - push!(ks, LMDB.mdb_unpack(String, k_ref)) + push!(ks, read(LMDB.MDBValueIO(k_ref[]), String)) end @test ks == ["a", "b", "c"] # walk from a starting key ks2 = String[] LMDB.walk(cur; from="b") do k_ref, _ - push!(ks2, LMDB.mdb_unpack(String, k_ref)) + push!(ks2, read(LMDB.MDBValueIO(k_ref[]), String)) end @test ks2 == ["b", "c"] # walk from a key past the last entry — no callbacks. ks3 = String[] LMDB.walk(cur; from="z") do k_ref, _ - push!(ks3, LMDB.mdb_unpack(String, k_ref)) + push!(ks3, read(LMDB.MDBValueIO(k_ref[]), String)) end @test isempty(ks3) - # typed walk: each ref decoded via mdb_unpack(K, ...) - # / mdb_unpack(V, ...). + # typed walk: each ref decoded via read(MDBValueIO, K) + # / read(MDBValueIO, V). kv = Pair{String, String}[] LMDB.walk(cur, String, String) do k, v push!(kv, k => v) @@ -138,7 +138,7 @@ module LMDB_CUR ks = String[] LMDB.walk(cur) do k_ref, _ - push!(ks, LMDB.mdb_unpack(String, k_ref)) + push!(ks, read(LMDB.MDBValueIO(k_ref[]), String)) end @test isempty(ks) end diff --git a/test/integration.jl b/test/integration.jl index a8f18d9..bd8387d 100644 --- a/test/integration.jl +++ b/test/integration.jl @@ -3,22 +3,18 @@ module LMDB_Integration using Test # cuTile-shaped framed value: 8-byte LE atime prefix, then the payload. - # Defined here at module scope to demonstrate the `mdb_unpack` extension - # contract and to regression-guard it (a downstream package — cuTile's - # DiskCache — relies on being able to plug in its own unpack method - # without touching LMDB.jl internals). + # Defined here at module scope to demonstrate the `Base.read(::IO, …)` + # extension contract and to regression-guard it (a downstream package + # — cuTile's DiskCache — relies on being able to plug in its own + # decoder against the abstract `IO`, without depending on + # `LMDB.MDBValueIO` in its own type signatures). struct AtimedBlob end const _ATIME_PREFIX = 8 - function LMDB.mdb_unpack(::Type{AtimedBlob}, ref::Ref{LMDB.MDB_val}) - v = ref[] - sz = Int(v.mv_size) - sz < _ATIME_PREFIX && return UInt8[] - out = Vector{UInt8}(undef, sz - _ATIME_PREFIX) - unsafe_copyto!(pointer(out), - Ptr{UInt8}(v.mv_data) + _ATIME_PREFIX, - sz - _ATIME_PREFIX) - return out + function Base.read(io::IO, ::Type{AtimedBlob}) + bytesavailable(io) < _ATIME_PREFIX && return UInt8[] + skip(io, _ATIME_PREFIX) + return read(io, Vector{UInt8}) end pack_atimed(atime::UInt64, payload::Vector{UInt8}) = begin @@ -91,9 +87,9 @@ module LMDB_Integration @test LMDB.tryget(txn, dbi, "key3", String) === nothing end - # mdb_unpack extension: write an 8-byte-prefixed framed value + # MDBValueIO extension: write an 8-byte-prefixed framed value # and read it back via `tryget(..., AtimedBlob)`, getting the - # payload tail in a single copy with no slicing. + # payload tail with one alloc + skip + copy, no slicing. payload = Vector{UInt8}("cubin-bytes-here") atime = UInt64(0xdeadbeefcafebabe) start(env) do txn