Upgrade to LMDB 1.0 (#71)

Author: maleadt

Project.toml

@@ -12,5 +12,5 @@ LMDB_jll = "6206cf0b-f360-5984-af49-5437264c140e"
 
 [compat]
 CEnum = "0.4, 0.5"
-LMDB_jll = "0.9.35"
+LMDB_jll = "1.0"
 julia = "1.10"

README.md

@@ -10,6 +10,10 @@ Memory-Mapped Database. LMDB is an embedded, memory-mapped, ACID key-value
 store developed by Symas for OpenLDAP. It persists to disk while reading
 at near in-memory speeds.
 
+LMDB 1.0 changed the on-disk file format. Databases created by LMDB 0.9
+must be exported with a 0.9 `mdb_dump` and imported with a 1.0 `mdb_load`;
+LMDB 1.0 does not open 0.9 database files directly.
+
 ```julia
 using Pkg; Pkg.add("LMDB")
 ```
@@ -125,3 +129,4 @@ ret == 0 || throw(LMDB.LMDBError(ret))
 
 - LMDB upstream: <https://github.com/LMDB/lmdb>
 - LMDB API docs: <http://www.lmdb.tech/doc/>
+- LMDB 1.0 upgrade notes: <http://www.lmdb.tech/doc/upgrading.html>

docs/src/index.md

@@ -7,6 +7,10 @@ LMDB is an embedded, memory-mapped, ACID key-value store developed by
 Symas for OpenLDAP. It persists to disk while reading at near in-memory
 speeds, limited only by the size of the virtual address space.
 
+LMDB 1.0 changed the on-disk file format. Databases created by LMDB 0.9
+must be exported with a 0.9 `mdb_dump` and imported with a 1.0
+`mdb_load`; LMDB 1.0 does not open 0.9 database files directly.
+
 ```julia
 using Pkg; Pkg.add("LMDB")
 ```

docs/src/lib/environments.md

@@ -27,7 +27,8 @@ path
 ## Configuration
 
 `Environment` exposes its tunables through `getindex` / `setindex!` with
-symbol keys (`:Flags`, `:Readers`, `:MapSize`, `:DBs`, `:KeySize`):
+symbol keys (`:Flags`, `:Readers`, `:MapSize`, `:PageSize`, `:DBs`,
+`:KeySize`):
 
 ```@docs
 Base.setindex!(::Environment, ::Integer, ::Symbol)

docs/src/lib/errors.md

@@ -30,7 +30,11 @@ The rest (`MDB_PAGE_NOTFOUND`, `MDB_CORRUPTED`, `MDB_PANIC`,
 `MDB_VERSION_MISMATCH`, `MDB_INVALID`, `MDB_DBS_FULL`, `MDB_READERS_FULL`,
 `MDB_TLS_FULL`, `MDB_TXN_FULL`, `MDB_CURSOR_FULL`, `MDB_PAGE_FULL`,
 `MDB_MAP_RESIZED`, `MDB_INCOMPATIBLE`, `MDB_BAD_RSLOT`, `MDB_BAD_TXN`,
-`MDB_BAD_VALSIZE`, `MDB_BAD_DBI`) live under the `LMDB.` prefix.
+`MDB_BAD_VALSIZE`, `MDB_BAD_DBI`, `MDB_PROBLEM`, `MDB_BAD_CHECKSUM`,
+`MDB_CRYPTO_FAIL`, `MDB_ENV_ENCRYPTION`, `MDB_TXN_PENDING`,
+`MDB_CANT_ROLLBACK`, `MDB_DBIS_BUSY`, `MDB_SHORT_WRITE`,
+`MDB_ENV_BUSY`, `MDB_IS_READONLY`, `MDB_ADDR_BUSY`) live under the
+`LMDB.` prefix.
 
 ## Where errors come from at each surface
 

docs/src/lib/lowlevel.md

@@ -32,10 +32,12 @@ ret == 0 || throw(LMDB.LMDBError(ret))
 
 Bindings that return non-status data (`mdb_strerror`, `mdb_version`,
 `mdb_txn_id`, `mdb_cmp`, `mdb_dcmp`, `mdb_env_get_maxkeysize`,
-`mdb_env_get_userctx`, `mdb_cursor_txn`, `mdb_cursor_dbi`) and
+`mdb_env_get_userctx`, `mdb_cursor_txn`, `mdb_cursor_dbi`,
+`mdb_cursor_is_db`, `mdb_modload`) and
 `Cvoid`-returning ones (`mdb_env_close`, `mdb_dbi_close`,
-`mdb_txn_abort`, `mdb_txn_reset`, `mdb_cursor_close`) are left bare,
-since there is nothing to check.
+`mdb_txn_abort`, `mdb_txn_reset`, `mdb_cursor_close`,
+`mdb_modunload`, `mdb_modsetup`) are left bare, since there is
+nothing to check.
 
 ## Customisation point: `MDBValueIO`
 
@@ -69,10 +71,12 @@ LMDB.MDB_env       # opaque
 LMDB.MDB_txn       # opaque
 LMDB.MDB_cursor    # opaque
 LMDB.MDB_dbi       # = Cuint
+LMDB.mdb_size_t    # mapsize/page/transaction id integer type
 LMDB.MDB_val       # struct { mv_size::Csize_t; mv_data::Ptr{Cvoid} }
 LMDB.MDB_stat      # struct (page sizes, depth, leaf/branch/overflow page counts, entries)
 LMDB.MDB_envinfo   # struct (mapaddr, mapsize, last_pgno, last_txnid, maxreaders, numreaders)
 LMDB.MDB_cursor_op # @cenum: MDB_FIRST … MDB_PREV_MULTIPLE (19 variants)
+LMDB.MDB_crypto_funcs
 ```
 
 ### Environment
@@ -83,16 +87,19 @@ mdb_env_open
 mdb_env_close
 mdb_env_copy        mdb_env_copy2
 mdb_env_copyfd      mdb_env_copyfd2
+mdb_env_incr_dump   mdb_env_incr_dumpfd   mdb_env_incr_loadfd
 mdb_env_stat        mdb_env_info
 mdb_env_sync
 mdb_env_set_flags   mdb_env_get_flags
 mdb_env_get_path    mdb_env_get_fd
-mdb_env_set_mapsize
+mdb_env_set_mapsize mdb_env_set_pagesize
 mdb_env_set_maxreaders   mdb_env_get_maxreaders
 mdb_env_set_maxdbs
 mdb_env_get_maxkeysize
 mdb_env_set_userctx      mdb_env_get_userctx
 mdb_env_set_assert
+mdb_env_set_encrypt      mdb_env_set_checksum
+mdb_env_rollback
 ```
 
 ### Transaction
@@ -101,7 +108,9 @@ mdb_env_set_assert
 mdb_txn_begin
 mdb_txn_env
 mdb_txn_id
+mdb_txn_flags
 mdb_txn_commit
+mdb_txn_prepare
 mdb_txn_abort
 mdb_txn_reset
 mdb_txn_renew
@@ -135,18 +144,20 @@ mdb_cursor_close
 mdb_cursor_renew
 mdb_cursor_txn
 mdb_cursor_dbi
+mdb_cursor_is_db
 mdb_cursor_get
 mdb_cursor_put
 mdb_cursor_del
 mdb_cursor_count
 ```
 
-### Comparators / readers / version
+### Comparators / readers / crypto modules / version
 
 ```julia
 mdb_cmp     mdb_dcmp
 mdb_reader_list
 mdb_reader_check
+mdb_modload mdb_modunload mdb_modsetup
 mdb_version
 mdb_strerror
 ```
@@ -155,8 +166,8 @@ mdb_strerror
 
 | group | constants |
 |------|-----------|
-| Env flags | `MDB_FIXEDMAP`, `MDB_NOSUBDIR`, `MDB_NOSYNC`, `MDB_RDONLY`, `MDB_NOMETASYNC`, `MDB_WRITEMAP`, `MDB_MAPASYNC`, `MDB_NOTLS`, `MDB_NOLOCK`, `MDB_NORDAHEAD`, `MDB_NOMEMINIT` |
+| Env flags | `MDB_FIXEDMAP`, `MDB_ENCRYPT`, `MDB_NOSUBDIR`, `MDB_NOSYNC`, `MDB_RDONLY`, `MDB_NOMETASYNC`, `MDB_WRITEMAP`, `MDB_MAPASYNC`, `MDB_NOTLS`, `MDB_NOLOCK`, `MDB_NORDAHEAD`, `MDB_NOMEMINIT`, `MDB_PREVSNAPSHOT`, `MDB_REMAP_CHUNKS` |
 | DB flags | `MDB_REVERSEKEY`, `MDB_DUPSORT`, `MDB_INTEGERKEY`, `MDB_DUPFIXED`, `MDB_INTEGERDUP`, `MDB_REVERSEDUP`, `MDB_CREATE` |
 | Write flags | `MDB_NOOVERWRITE`, `MDB_NODUPDATA`, `MDB_CURRENT`, `MDB_RESERVE`, `MDB_APPEND`, `MDB_APPENDDUP`, `MDB_MULTIPLE` |
 | Copy flag | `MDB_CP_COMPACT` |
-| Status codes | `MDB_SUCCESS=0`, `MDB_KEYEXIST`, `MDB_NOTFOUND`, `MDB_PAGE_NOTFOUND`, `MDB_CORRUPTED`, `MDB_PANIC`, `MDB_VERSION_MISMATCH`, `MDB_INVALID`, `MDB_MAP_FULL`, `MDB_DBS_FULL`, `MDB_READERS_FULL`, `MDB_TLS_FULL`, `MDB_TXN_FULL`, `MDB_CURSOR_FULL`, `MDB_PAGE_FULL`, `MDB_MAP_RESIZED`, `MDB_INCOMPATIBLE`, `MDB_BAD_RSLOT`, `MDB_BAD_TXN`, `MDB_BAD_VALSIZE`, `MDB_BAD_DBI` |
+| Status codes | `MDB_SUCCESS=0`, `MDB_KEYEXIST`, `MDB_NOTFOUND`, `MDB_PAGE_NOTFOUND`, `MDB_CORRUPTED`, `MDB_PANIC`, `MDB_VERSION_MISMATCH`, `MDB_INVALID`, `MDB_MAP_FULL`, `MDB_DBS_FULL`, `MDB_READERS_FULL`, `MDB_TLS_FULL`, `MDB_TXN_FULL`, `MDB_CURSOR_FULL`, `MDB_PAGE_FULL`, `MDB_MAP_RESIZED`, `MDB_INCOMPATIBLE`, `MDB_BAD_RSLOT`, `MDB_BAD_TXN`, `MDB_BAD_VALSIZE`, `MDB_BAD_DBI`, `MDB_PROBLEM`, `MDB_BAD_CHECKSUM`, `MDB_CRYPTO_FAIL`, `MDB_ENV_ENCRYPTION`, `MDB_TXN_PENDING`, `MDB_CANT_ROLLBACK`, `MDB_DBIS_BUSY`, `MDB_SHORT_WRITE`, `MDB_ENV_BUSY`, `MDB_IS_READONLY`, `MDB_ADDR_BUSY` |

docs/src/man/environments.md

@@ -15,6 +15,7 @@ the handle, set mapsize/maxreaders/maxdbs/flags, open the directory:
 
 ```julia
 env = Environment("/tmp/mydb"; mapsize    = 1 << 30,   # 1 GiB virtual map
+                               pagesize   = 8192,
                                maxreaders = 510,
                                maxdbs     = 8,
                                flags      = LMDB.MDB_NOTLS)
@@ -23,14 +24,15 @@ env = Environment("/tmp/mydb"; mapsize    = 1 << 30,   # 1 GiB virtual map
 If anything fails on the way through, the half-open env is closed
 before the exception propagates.
 
-After the env is open, `[:Flags]` / `[:Readers]` / `[:MapSize]` /
-`[:DBs]` setindex! keys map to `mdb_env_set_flags` /
-`mdb_env_set_maxreaders` / `mdb_env_set_mapsize` / `mdb_env_set_maxdbs`,
-and `set!` / `unset!` flip individual flag bits.
+Before the env is open, `pagesize` maps to `mdb_env_set_pagesize`.
+The `[:Flags]` / `[:Readers]` / `[:MapSize]` / `[:PageSize]` / `[:DBs]`
+setindex! keys map to `mdb_env_set_flags` / `mdb_env_set_maxreaders` /
+`mdb_env_set_mapsize` / `mdb_env_set_pagesize` / `mdb_env_set_maxdbs`,
+and `set!` / `unset!` flip individual flag bits after open.
 
 `getindex` exposes a few read-only views: `env[:Flags]`,
-`env[:Readers]`, and `env[:KeySize]` (the maximum key length, fixed at
-compile time of the bundled `LMDB_jll`).
+`env[:Readers]`, and `env[:KeySize]` (the maximum key length for the
+environment's configured page size).
 
 The do-block form `Environment(f, path; kwargs...)` opens the env,
 calls `f(env)`, and closes on the way out:
@@ -66,6 +68,12 @@ LMDB.MDB_RDONLY)` on an open env will return `EINVAL`.
 on-disk size. Pick a generous power of two (say, 1 GiB or 8 GiB) up
 front. The on-disk file grows incrementally as data is written.
 
+LMDB 1.0 also lets you choose the DB page size with `pagesize`. The
+default is the OS page size; larger values can allow larger keys and
+`MDB_DUPSORT` data items. After opening, `stat(env).psize` reports the
+actual page size and `env[:KeySize]` reports the corresponding maximum
+key size.
+
 If a write txn would exceed `mapsize`, LMDB returns `MDB_MAP_FULL`. To
 recover, close the env, raise `mapsize`, and reopen. The database
 itself does not need rewriting.

docs/src/man/essentials.md

@@ -10,9 +10,13 @@ After importing LMDB.jl, you can immediately query the bundled library:
 julia> using LMDB
 
 julia> LMDB.version()
-(v"0.9.33", "LMDB 0.9.33: (May 21, 2024)")
+v"1.0.0"
 ```
 
+LMDB 1.0 changed the on-disk file format. Databases created by LMDB
+0.9 must be exported with a 0.9 `mdb_dump` and imported with a 1.0
+`mdb_load`; LMDB 1.0 does not open 0.9 database files directly.
+
 ## A complete example
 
 The easiest entry point is the [`LMDBDict`](@ref), a persistent

docs/src/man/lowlevel.md

@@ -39,9 +39,11 @@ This is exactly the pattern `get(txn, dbi, key, T, default)` uses internally.
 
 Bindings that don't return a status (`mdb_strerror`, `mdb_version`,
 `mdb_txn_id`, `mdb_cmp`, `mdb_dcmp`, `mdb_env_get_maxkeysize`,
-`mdb_cursor_txn`, `mdb_cursor_dbi`) and `Cvoid`-returning ones
+`mdb_cursor_txn`, `mdb_cursor_dbi`, `mdb_cursor_is_db`, `mdb_modload`)
+and `Cvoid`-returning ones
 (`mdb_env_close`, `mdb_dbi_close`, `mdb_txn_abort`, `mdb_txn_reset`,
-`mdb_cursor_close`) are left bare; there is nothing to check.
+`mdb_cursor_close`, `mdb_modunload`, `mdb_modsetup`) are left bare;
+there is nothing to check.
 
 ## ccall glue: passing values to `Ptr{MDB_val}`
 
@@ -148,3 +150,8 @@ Julia wrappers deliberately don't include them:
 - `MDB_GET_MULTIPLE` / `MDB_NEXT_MULTIPLE` cursor ops: reachable by
   passing the constant directly to `LMDB.mdb_cursor_get`. Useful with
   `MDB_DUPFIXED` databases for batched reads.
+- LMDB 1.0 incremental backup, encryption/checksum hooks, and
+  two-phase-commit helpers are exposed as raw bindings:
+  `mdb_env_incr_dump`, `mdb_env_incr_dumpfd`, `mdb_env_incr_loadfd`,
+  `mdb_env_set_encrypt`, `mdb_env_set_checksum`, `mdb_txn_prepare`,
+  and `mdb_env_rollback`.

res/wrap.jl

@@ -16,6 +16,7 @@ const UNCHECKED_CINT = (
     "mdb_env_get_maxkeysize",  # returns the maximum key size, not a status.
     "mdb_cmp",                  # returns the comparison result (-1/0/1).
     "mdb_dcmp",                 # idem for dup-data.
+    "mdb_cursor_is_db",         # returns a boolean int.
 )
 
 # Replace `function $name(args)\n    @ccall ...::Cint\nend` with
@@ -42,6 +43,8 @@ function postprocess!(path::AbstractString)
     end
     write(out, SubString(src, last, lastindex(src)))
     src = String(take!(out))
+    src = replace(src, r"^const MDB_SIZE_MAX = SIZE_MAX$"m =>
+                       "const MDB_SIZE_MAX = typemax(mdb_size_t)")
     # Clang.Generators always emits `export $jll_pkg_name` next to its
     # `using` line; strip it so LMDB_jll stays an implementation detail.
     src = replace(src, r"^export LMDB_jll\n"m => "")