From e45e42ebb76840fa67e49057ed7ed0c17face1d6 Mon Sep 17 00:00:00 2001 From: "jianjian.xie" Date: Fri, 24 Apr 2026 16:03:31 -0700 Subject: [PATCH 1/4] chore(deps): bump lance to 5.0.0-beta.5 (git rev d630106d) Switches lance / lance-core / lance-index / lance-io / lance-linalg from crates.io 3.0.1 to a git+rev pin at lance commit d630106d (release tag v5.0.0-beta.5). Adds lance-datagen / lance-file / lance-table dev deps from the same rev. Beta-5 introduces the segment-model APIs (Scanner::with_index_segments, commit_existing_index_segments) that subsequent commits expose through the C ABI for distributed vector search. The DatasetIndexExt trait moved from lance_index to lance::index; src/index.rs adjusts the import. arrow stays on 57.0.0 (matches beta-5). Adds uuid 1.x for the upcoming UUID-based segment API. --- Cargo.lock | 614 +++++++++++++++++++++++++----------------------- Cargo.toml | 19 +- include/lance.h | 47 ++++ src/index.rs | 3 +- src/scanner.rs | 55 +++++ 5 files changed, 431 insertions(+), 307 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 82acdf4..b057d63 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -16,7 +16,7 @@ checksum = "b169f7a6d4742236a0a00c541b845991d0ac43e546831af1249753ab4c3aa3a0" dependencies = [ "cfg-if", "cipher", - "cpufeatures", + "cpufeatures 0.2.17", ] [[package]] @@ -89,9 +89,9 @@ dependencies = [ [[package]] name = "arc-swap" -version = "1.9.0" +version = "1.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a07d1f37ff60921c83bdfc7407723bdefe89b44b98a9b772f225c8f9d67141a6" +checksum = "6a3a1fd6f75306b68087b831f025c712524bcb19aad54e557b1129cfa0a2b207" dependencies = [ "rustversion", ] @@ -346,9 +346,9 @@ dependencies = [ [[package]] name = "async-compression" -version = "0.4.41" +version = "0.4.42" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d0f9ee0f6e02ffd7ad5816e9464499fba7b3effd01123b515c41d1697c43dad1" +checksum = "e79b3f8a79cccc2898f31920fc69f304859b3bd567490f75ebf51ae1c792a9ac" dependencies = [ "compression-codecs", "compression-core", @@ -463,9 +463,9 @@ dependencies = [ [[package]] name = "aws-lc-rs" -version = "1.16.2" +version = "1.16.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a054912289d18629dc78375ba2c3726a3afe3ff71b4edba9dedfca0e3446d1fc" +checksum = "0ec6fb3fe69024a75fa7e1bfb48aa6cf59706a101658ea01bfd33b2b248a038f" dependencies = [ "aws-lc-sys", "zeroize", @@ -473,9 +473,9 @@ dependencies = [ [[package]] name = "aws-lc-sys" -version = "0.39.0" +version = "0.40.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1fa7e52a4c5c547c741610a2c6f123f3881e409b714cd27e6798ef020c514f0a" +checksum = "f50037ee5e1e41e7b8f9d161680a725bd1626cb6f8c7e901f91f942850852fe7" dependencies = [ "cc", "cmake", @@ -823,9 +823,9 @@ dependencies = [ [[package]] name = "bitflags" -version = "2.11.0" +version = "2.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "843867be96c8daad0d758b57df9392b6d8d271134fce549de6ce169ff98a92af" +checksum = "c4512299f36f043ab09a583e57bceb5a5aab7a73db1805848e8fef3c9e8c78b3" [[package]] name = "bitpacking" @@ -859,16 +859,16 @@ dependencies = [ [[package]] name = "blake3" -version = "1.8.3" +version = "1.8.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2468ef7d57b3fb7e16b576e8377cdbde2320c60e1491e961d11da40fc4f02a2d" +checksum = "4d2d5991425dfd0785aed03aedcf0b321d61975c9b5b3689c774a2610ae0b51e" dependencies = [ "arrayref", "arrayvec", "cc", "cfg-if", "constant_time_eq", - "cpufeatures", + "cpufeatures 0.3.0", ] [[package]] @@ -980,9 +980,9 @@ dependencies = [ [[package]] name = "cc" -version = "1.2.57" +version = "1.2.61" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7a0dd1ca384932ff3641c8718a02769f1698e7563dc6974ffd03346116310423" +checksum = "d16d90359e986641506914ba71350897565610e87ce0ad9e6f28569db3dd5c6d" dependencies = [ "find-msvc-tools", "jobserver", @@ -1044,9 +1044,9 @@ dependencies = [ [[package]] name = "cmake" -version = "0.1.57" +version = "0.1.58" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "75443c44cd6b379beb8c5b45d85d0773baf31cce901fe7bb252f4eff3008ef7d" +checksum = "c0f78a02292a74a88ac736019ab962ece0bc380e3f977bf72e376c5d78ff0678" dependencies = [ "cc", ] @@ -1063,9 +1063,9 @@ dependencies = [ [[package]] name = "compression-codecs" -version = "0.4.37" +version = "0.4.38" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eb7b51a7d9c967fc26773061ba86150f19c50c0d65c887cb1fbe295fd16619b7" +checksum = "ce2548391e9c1929c21bf6aa2680af86fe4c1b33e6cea9ac1cfeec0bd11218cf" dependencies = [ "compression-core", "flate2", @@ -1074,9 +1074,9 @@ dependencies = [ [[package]] name = "compression-core" -version = "0.4.31" +version = "0.4.32" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "75984efb6ed102a0d42db99afb6c1948f0380d1d91808d5529916e6c08b49d8d" +checksum = "cc14f565cf027a105f7a44ccf9e5b424348421a1d8952a8fc9d499d313107789" [[package]] name = "concurrent-queue" @@ -1144,6 +1144,15 @@ dependencies = [ "libc", ] +[[package]] +name = "cpufeatures" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b2a41393f66f16b0823bb79094d54ac5fbd34ab292ddafb9a0456ac9f87d201" +dependencies = [ + "libc", +] + [[package]] name = "crc32c" version = "0.6.8" @@ -1302,9 +1311,9 @@ dependencies = [ [[package]] name = "datafusion" -version = "52.4.0" +version = "52.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "43c18ba387f9c05ac1f3be32a73f8f3cc6c1cfc43e5d4b7a8e5b0d3a5eb48dc7" +checksum = "7541353e77dc7262b71ca27be07d8393661737e3a73b5d1b1c6f7d814c64fa2a" dependencies = [ "arrow", "arrow-schema", @@ -1340,7 +1349,7 @@ dependencies = [ "log", "object_store", "parking_lot", - "rand 0.9.2", + "rand 0.9.4", "regex", "sqlparser", "tempfile", @@ -1351,9 +1360,9 @@ dependencies = [ [[package]] name = "datafusion-catalog" -version = "52.4.0" +version = "52.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3c75a4ce672b27fb8423810efb92a3600027717a1664d06a2c307eeeabcec694" +checksum = "9997731f90fa5398ef831ad0e69600f92c861b79c0d38bd1a29b6f0e3a0ce4c8" dependencies = [ "arrow", "async-trait", @@ -1376,9 +1385,9 @@ dependencies = [ [[package]] name = "datafusion-catalog-listing" -version = "52.4.0" +version = "52.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2c8b9a3795ffb46bf4957a34c67d89a67558b311ae455c8d4295ff2115eeea50" +checksum = "2b30a3dd50dec860c9559275c8d97d9de602e611237a6ecfbda0b3b63b872352" dependencies = [ "arrow", "async-trait", @@ -1399,9 +1408,9 @@ dependencies = [ [[package]] name = "datafusion-common" -version = "52.4.0" +version = "52.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "205dc1e20441973f470e6b7ef87626a3b9187970e5106058fef1b713047f770c" +checksum = "d551054acec0398ca604512310b77ce05c46f66e54b54d48200a686e385cca4e" dependencies = [ "ahash", "arrow", @@ -1421,9 +1430,9 @@ dependencies = [ [[package]] name = "datafusion-common-runtime" -version = "52.4.0" +version = "52.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8cf5880c02ff6f5f11fb5bc19211789fb32fd3c53d79b7d6cb2b12e401312ba0" +checksum = "567d40e285f5b79f8737b576605721cd6c1133b5d2b00bdbd5d9838d90d0812f" dependencies = [ "futures", "log", @@ -1432,9 +1441,9 @@ dependencies = [ [[package]] name = "datafusion-datasource" -version = "52.4.0" +version = "52.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bc614d6e709450e29b7b032a42c1bdb705f166a6b2edef7bed7c7897eb905499" +checksum = "27d2668f51b3b30befae2207472569e37807fdedd1d14da58acc6f8ca6257eae" dependencies = [ "arrow", "async-trait", @@ -1454,16 +1463,16 @@ dependencies = [ "itertools 0.14.0", "log", "object_store", - "rand 0.9.2", + "rand 0.9.4", "tokio", "url", ] [[package]] name = "datafusion-datasource-arrow" -version = "52.4.0" +version = "52.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6e497d5fc48dac7ce86f6b4fb09a3a494385774af301ff20ec91aebfae9b05b4" +checksum = "e02e1b3e3a8ec55f1f62de4252b0407c8567363d056078769a197e24fc834a0f" dependencies = [ "arrow", "arrow-ipc", @@ -1485,9 +1494,9 @@ dependencies = [ [[package]] name = "datafusion-datasource-csv" -version = "52.4.0" +version = "52.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0dfc250cad940d0327ca2e9109dc98830892d17a3d6b2ca11d68570e872cf379" +checksum = "b559d7bf87d4f900f847baba8509634f838d9718695389e903604cdcccdb01f3" dependencies = [ "arrow", "async-trait", @@ -1508,9 +1517,9 @@ dependencies = [ [[package]] name = "datafusion-datasource-json" -version = "52.4.0" +version = "52.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c91e9677ed62833b0e8129dec0d1a8f3c9bb7590bd6dd714a43e4c3b663e4aa0" +checksum = "250e2d7591ba8b638f063854650faa40bca4e8bd4059b2ece8836f6388d02db4" dependencies = [ "arrow", "async-trait", @@ -1530,15 +1539,15 @@ dependencies = [ [[package]] name = "datafusion-doc" -version = "52.4.0" +version = "52.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3e13e5fe3447baa0584b61ee8644086e007e1ef6e58f4be48bc8a72417854729" +checksum = "b9496cb0db222dbb9a3735760ceca7fc56f35e1d5502c38d0caa77a81e9c1f6a" [[package]] name = "datafusion-execution" -version = "52.4.0" +version = "52.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "48a6cc03e34899a54546b229235f7b192634c8e832f78a267f0989b18216c56d" +checksum = "dc45d23c516ed8d3637751e44e09e21b45b3f58b473c802dddd1f1ad4fe435ff" dependencies = [ "arrow", "async-trait", @@ -1550,16 +1559,16 @@ dependencies = [ "log", "object_store", "parking_lot", - "rand 0.9.2", + "rand 0.9.4", "tempfile", "url", ] [[package]] name = "datafusion-expr" -version = "52.4.0" +version = "52.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ee3315d87eca7a7df58e52a1fb43b4c4171b545fd30ffc3102945c162a9f6ddb" +checksum = "63dd30526d2db4fda6440806a41e4676334a94bc0596cc9cc2a0efed20ef2c44" dependencies = [ "arrow", "async-trait", @@ -1579,9 +1588,9 @@ dependencies = [ [[package]] name = "datafusion-expr-common" -version = "52.4.0" +version = "52.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "98c6d83feae0753799f933a2c47dfd15980c6947960cb95ed60f5c1f885548b3" +checksum = "1b486b5f6255d40976b88bb83813b0d035a8333e0ec39864824e78068cf42fa6" dependencies = [ "arrow", "datafusion-common", @@ -1592,9 +1601,9 @@ dependencies = [ [[package]] name = "datafusion-functions" -version = "52.4.0" +version = "52.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "49b82962015cc3db4d7662459c9f7fcda0591b5edacb8af1cf3bc3031f274800" +checksum = "07356c94118d881130dd0ffbff127540407d969c8978736e324edcd6c41cd48f" dependencies = [ "arrow", "arrow-buffer", @@ -1614,7 +1623,7 @@ dependencies = [ "log", "md-5", "num-traits", - "rand 0.9.2", + "rand 0.9.4", "regex", "sha2", "unicode-segmentation", @@ -1623,9 +1632,9 @@ dependencies = [ [[package]] name = "datafusion-functions-aggregate" -version = "52.4.0" +version = "52.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4e42c227d9e55a6c8041785d4a8a117e4de531033d480aae10984247ac62e27e" +checksum = "b644f9cf696df9233ce6958b9807666d78563b56f923267474dd6c07795f1f8f" dependencies = [ "ahash", "arrow", @@ -1644,9 +1653,9 @@ dependencies = [ [[package]] name = "datafusion-functions-aggregate-common" -version = "52.4.0" +version = "52.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cead3cfed825b0b688700f4338d281cd7857e4907775a5b9554c083edd5f3f95" +checksum = "c1de2deaaabe8923ce9ea9f29c47bbb4ee14f67ea2fe1ab5398d9bbebcf86e56" dependencies = [ "ahash", "arrow", @@ -1657,9 +1666,9 @@ dependencies = [ [[package]] name = "datafusion-functions-nested" -version = "52.4.0" +version = "52.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "62ea99612970aebab8cf864d02eb3d296bbab7f4881e1023d282b57fe431b201" +checksum = "552f8d92e4331ee91d23c02d12bb6acf32cbfd5215117e01c0fb63cd4b15af1a" dependencies = [ "arrow", "arrow-ord", @@ -1680,9 +1689,9 @@ dependencies = [ [[package]] name = "datafusion-functions-table" -version = "52.4.0" +version = "52.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d83dbf3ab8b9af6f209b068825a7adbd3b88bf276f2a1ec14ba09567b97f5674" +checksum = "970fd0cdd3df8802b9a9975ff600998289ba9d46682a4f7285cba4820c9ada78" dependencies = [ "arrow", "async-trait", @@ -1696,9 +1705,9 @@ dependencies = [ [[package]] name = "datafusion-functions-window" -version = "52.4.0" +version = "52.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "732edabe07496e2fc5a1e57a284d7a36edcea445a2821119770a0dea624b472c" +checksum = "40b4c21a7c8a986a1866c0a87ab756d0bbf7b5f41f306009fa2d9af79c52ed31" dependencies = [ "arrow", "datafusion-common", @@ -1714,9 +1723,9 @@ dependencies = [ [[package]] name = "datafusion-functions-window-common" -version = "52.4.0" +version = "52.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e0c6e30e09700799bd52adce8c377ab03dda96e73a623e4803a31ad94fe7ce14" +checksum = "b1210ad73b8b3211aeaf4a42bef9bd7a2b7fce3ec119a478831f18c6ff7f7b93" dependencies = [ "datafusion-common", "datafusion-physical-expr-common", @@ -1724,9 +1733,9 @@ dependencies = [ [[package]] name = "datafusion-macros" -version = "52.4.0" +version = "52.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "402f2a8ed70fb99a18f71580a1fe338604222a3d32ddeac6e72c5b34feea2d4d" +checksum = "aaa566a963013a38681ad82a727a654bc7feb19632426aea8c3412d415d200c5" dependencies = [ "datafusion-doc", "quote", @@ -1735,9 +1744,9 @@ dependencies = [ [[package]] name = "datafusion-optimizer" -version = "52.4.0" +version = "52.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "99f32edb8ba12f08138f86c09b80fae3d4a320551262fa06b91d8a8cb3065a5b" +checksum = "ff9aa82b240252a88dee118372f9b9757c545ab9e53c0736bebab2e7da0ef1f2" dependencies = [ "arrow", "chrono", @@ -1754,9 +1763,9 @@ dependencies = [ [[package]] name = "datafusion-physical-expr" -version = "52.4.0" +version = "52.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "987c5e29e96186589301b42e25aa7d11bbe319a73eb02ef8d755edc55b5b89fc" +checksum = "7d48022b8af9988c1d852644f9e8b5584c490659769a550c5e8d39457a1da0a5" dependencies = [ "ahash", "arrow", @@ -1777,9 +1786,9 @@ dependencies = [ [[package]] name = "datafusion-physical-expr-adapter" -version = "52.4.0" +version = "52.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1de89d0afa08b6686697bd8a6bac4ba2cd44c7003356e1bce6114d5a93f94b5c" +checksum = "ae7a8abc0b4fe624000972a9b145b30b7f1b680bffaa950ea53f78d9b21c27c3" dependencies = [ "arrow", "datafusion-common", @@ -1792,9 +1801,9 @@ dependencies = [ [[package]] name = "datafusion-physical-expr-common" -version = "52.4.0" +version = "52.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "602d1970c0fe87f1c3a36665d131fbfe1c4379d35f8fc5ec43a362229ad2954d" +checksum = "147253ca3e6b9d59c162de64c02800973018660e13340dd1886dd038d17ac429" dependencies = [ "ahash", "arrow", @@ -1809,9 +1818,9 @@ dependencies = [ [[package]] name = "datafusion-physical-optimizer" -version = "52.4.0" +version = "52.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b24d704b6385ebe27c756a12e5ba15684576d3b47aeca79cc9fb09480236dc32" +checksum = "689156bb2282107b6239db8d7ef44b4dab10a9b33d3491a0c74acac5e4fedd72" dependencies = [ "arrow", "datafusion-common", @@ -1827,9 +1836,9 @@ dependencies = [ [[package]] name = "datafusion-physical-plan" -version = "52.4.0" +version = "52.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c21d94141ea5043e98793f170798e9c1887095813b8291c5260599341e383a38" +checksum = "68253dc0ee5330aa558b2549c9b0da5af9fc17d753ae73022939014ad616fc28" dependencies = [ "ahash", "arrow", @@ -1858,9 +1867,9 @@ dependencies = [ [[package]] name = "datafusion-pruning" -version = "52.4.0" +version = "52.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1a68cce43d18c0dfac95cacd74e70565f7e2fb12b9ed41e2d312f0fa837626b1" +checksum = "0fcad240a54d0b1d3e8f668398900260a53122d522b2102ab57218590decacd6" dependencies = [ "arrow", "datafusion-common", @@ -1875,9 +1884,9 @@ dependencies = [ [[package]] name = "datafusion-session" -version = "52.4.0" +version = "52.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6b4e1c40a0b1896aed4a4504145c2eb7fa9b9da13c2d04b40a4767a09f076199" +checksum = "f58e83a68bb67007a8fcbf005c44cefe441270c7ee7f6dee10c0e0109b556f6d" dependencies = [ "async-trait", "datafusion-common", @@ -1889,9 +1898,9 @@ dependencies = [ [[package]] name = "datafusion-sql" -version = "52.4.0" +version = "52.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2f1891e5b106d1d73c7fe403bd8a265d19c3977edc17f60808daf26c2fe65ffb" +checksum = "be53e9eb55db0fbb8980bb6d87f2435b0524acf4c718ed54a57cabbb299b2ab3" dependencies = [ "arrow", "bigdecimal", @@ -2053,9 +2062,9 @@ dependencies = [ [[package]] name = "ethnum" -version = "1.5.2" +version = "1.5.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ca81e6b4777c89fd810c25a4be2b1bd93ea034fbe58e6a75216a34c6b82c539b" +checksum = "40404c3f5f511ec4da6fe866ddf6a717c309fdbb69fbbad7b0f3edab8f2e835f" [[package]] name = "event-listener" @@ -2092,9 +2101,9 @@ checksum = "9afc2bd4d5a73106dd53d10d73d3401c2f32730ba2c0b93ddb888a8983680471" [[package]] name = "fastrand" -version = "2.3.0" +version = "2.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "37909eebbb50d72f9059c3b6d82c0463f2ff062c9e95845c43a6c9c0355411be" +checksum = "9f1f227452a390804cdb637b74a86990f2a7d7ba4b7d5693aac9b4dd6defd8d6" [[package]] name = "find-msvc-tools" @@ -2179,12 +2188,11 @@ checksum = "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c" [[package]] name = "fsst" -version = "3.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a32ddfc5478379cd1782bdd9d7d1411063f563e5b338fc73bafe5916451a5b9d" +version = "5.0.0-beta.5" +source = "git+https://github.com/lance-format/lance.git?rev=d630106d#d630106da5a238b3adfb8c5dea3b3921f3519945" dependencies = [ "arrow-array", - "rand 0.9.2", + "rand 0.9.4", ] [[package]] @@ -2344,9 +2352,9 @@ dependencies = [ [[package]] name = "geo-types" -version = "0.7.18" +version = "0.7.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "24f8647af4005fa11da47cd56252c6ef030be8fa97bdbf355e7dfb6348f0a82c" +checksum = "94776032c45f950d30a13af6113c2ad5625316c9abfbccee4dd5a6695f8fe0f5" dependencies = [ "approx", "num-traits", @@ -2563,6 +2571,12 @@ dependencies = [ "foldhash 0.2.0", ] +[[package]] +name = "hashbrown" +version = "0.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4f467dd6dccf739c208452f8014c75c18bb8301b050ad1cfb27153803edb0f51" + [[package]] name = "heapless" version = "0.8.0" @@ -2684,9 +2698,9 @@ checksum = "135b12329e5e3ce057a9f972339ea52bc954fe1e9358ef27f95e89716fbc5424" [[package]] name = "hyper" -version = "1.8.1" +version = "1.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2ab2d4f250c3d7b1c9fcdff1cece94ea4e2dfbec68614f7b87cb205f24ca9d11" +checksum = "6299f016b246a94207e63da54dbe807655bf9e00044f73ded42c3ac5305fbcca" dependencies = [ "atomic-waker", "bytes", @@ -2698,7 +2712,6 @@ dependencies = [ "httparse", "itoa", "pin-project-lite", - "pin-utils", "smallvec", "tokio", "want", @@ -2706,16 +2719,15 @@ dependencies = [ [[package]] name = "hyper-rustls" -version = "0.27.7" +version = "0.27.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e3c93eb611681b207e1fe55d5a71ecf91572ec8a6705cdb6857f7d8d5242cf58" +checksum = "33ca68d021ef39cf6463ab54c1d0f5daf03377b70561305bb89a8f83aab66e0f" dependencies = [ "http 1.4.0", "hyper", "hyper-util", "rustls", "rustls-native-certs", - "rustls-pki-types", "tokio", "tokio-rustls", "tower-service", @@ -2823,12 +2835,13 @@ dependencies = [ [[package]] name = "icu_collections" -version = "2.1.1" +version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4c6b649701667bbe825c3b7e6388cb521c23d88644678e83c0c4d0a621a34b43" +checksum = "2984d1cd16c883d7935b9e07e44071dca8d917fd52ecc02c04d5fa0b5a3f191c" dependencies = [ "displaydoc", "potential_utf", + "utf8_iter", "yoke", "zerofrom", "zerovec", @@ -2836,9 +2849,9 @@ dependencies = [ [[package]] name = "icu_locale_core" -version = "2.1.1" +version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "edba7861004dd3714265b4db54a3c390e880ab658fec5f7db895fae2046b5bb6" +checksum = "92219b62b3e2b4d88ac5119f8904c10f8f61bf7e95b640d25ba3075e6cac2c29" dependencies = [ "displaydoc", "litemap", @@ -2849,9 +2862,9 @@ dependencies = [ [[package]] name = "icu_normalizer" -version = "2.1.1" +version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5f6c8828b67bf8908d82127b2054ea1b4427ff0230ee9141c54251934ab1b599" +checksum = "c56e5ee99d6e3d33bd91c5d85458b6005a22140021cc324cea84dd0e72cff3b4" dependencies = [ "icu_collections", "icu_normalizer_data", @@ -2863,15 +2876,15 @@ dependencies = [ [[package]] name = "icu_normalizer_data" -version = "2.1.1" +version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7aedcccd01fc5fe81e6b489c15b247b8b0690feb23304303a9e560f37efc560a" +checksum = "da3be0ae77ea334f4da67c12f149704f19f81d1adf7c51cf482943e84a2bad38" [[package]] name = "icu_properties" -version = "2.1.2" +version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "020bfc02fe870ec3a66d93e677ccca0562506e5872c650f893269e08615d74ec" +checksum = "bee3b67d0ea5c2cca5003417989af8996f8604e34fb9ddf96208a033901e70de" dependencies = [ "icu_collections", "icu_locale_core", @@ -2883,15 +2896,15 @@ dependencies = [ [[package]] name = "icu_properties_data" -version = "2.1.2" +version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "616c294cf8d725c6afcd8f55abc17c56464ef6211f9ed59cccffe534129c77af" +checksum = "8e2bbb201e0c04f7b4b3e14382af113e17ba4f63e2c9d2ee626b720cbce54a14" [[package]] name = "icu_provider" -version = "2.1.1" +version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "85962cf0ce02e1e0a629cc34e7ca3e373ce20dda4c4d7294bbd0bf1fdb59e614" +checksum = "139c4cf31c8b5f33d7e199446eff9c1e02decfc2f0eec2c8d71f65befa45b421" dependencies = [ "displaydoc", "icu_locale_core", @@ -2937,12 +2950,12 @@ dependencies = [ [[package]] name = "indexmap" -version = "2.13.0" +version = "2.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7714e70437a7dc3ac8eb7e6f8df75fd8eb422675fc7678aff7364301092b1017" +checksum = "d466e9454f08e4a911e14806c24e16fba1b4c121d1ea474396f396069cf949d9" dependencies = [ "equivalent", - "hashbrown 0.16.1", + "hashbrown 0.17.0", "serde", "serde_core", ] @@ -2957,6 +2970,17 @@ dependencies = [ "generic-array", ] +[[package]] +name = "io-uring" +version = "0.7.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4d09b98f7eace8982db770e4408e7470b028ce513ac28fecdc6bf4c30fe92b62" +dependencies = [ + "bitflags", + "cfg-if", + "libc", +] + [[package]] name = "ipnet" version = "2.12.0" @@ -2965,9 +2989,9 @@ checksum = "d98f6fed1fde3f8c21bc40a1abb88dd75e67924f9cffc3ef95607bad8017f8e2" [[package]] name = "iri-string" -version = "0.7.11" +version = "0.7.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d8e7418f59cc01c88316161279a7f665217ae316b388e58a0d10e29f54f1e5eb" +checksum = "25e659a4bb38e810ebc252e53b5814ff908a8c58c2a9ce2fae1bbec24cbf4e20" dependencies = [ "memchr", "serde", @@ -3008,9 +3032,9 @@ checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" [[package]] name = "jiff" -version = "0.2.23" +version = "0.2.24" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1a3546dc96b6d42c5f24902af9e2538e82e39ad350b0c766eb3fbf2d8f3d8359" +checksum = "f00b5dbd620d61dfdcb6007c9c1f6054ebd75319f163d886a9055cec1155073d" dependencies = [ "jiff-static", "jiff-tzdb-platform", @@ -3023,9 +3047,9 @@ dependencies = [ [[package]] name = "jiff-static" -version = "0.2.23" +version = "0.2.24" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2a8c8b344124222efd714b73bb41f8b5120b27a7cc1c75593a6ff768d9d05aa4" +checksum = "e000de030ff8022ea1da3f466fbb0f3a809f5e51ed31f6dd931c35181ad8e6d7" dependencies = [ "proc-macro2", "quote", @@ -3059,19 +3083,21 @@ dependencies = [ [[package]] name = "js-sys" -version = "0.3.91" +version = "0.3.95" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b49715b7073f385ba4bc528e5747d02e66cb39c6146efb66b781f131f0fb399c" +checksum = "2964e92d1d9dc3364cae4d718d93f227e3abb088e747d92e0395bfdedf1c12ca" dependencies = [ + "cfg-if", + "futures-util", "once_cell", "wasm-bindgen", ] [[package]] name = "jsonb" -version = "0.5.5" +version = "0.5.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2a901f06163d352fbe41c3c2ff5e08b75330a003cc941e988fb501022f5421e6" +checksum = "eb98fb29636087c40ad0d1274d9a30c0c1e83e03ae93f6e7e89247b37fcc6953" dependencies = [ "byteorder", "ethnum", @@ -3081,10 +3107,10 @@ dependencies = [ "nom 8.0.0", "num-traits", "ordered-float", - "rand 0.9.2", - "ryu", + "rand 0.9.4", "serde", "serde_json", + "zmij", ] [[package]] @@ -3104,9 +3130,8 @@ dependencies = [ [[package]] name = "lance" -version = "3.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "95c5ce428fda0721f5c48bfde17a1921c4da2d2142b2f46a16c89abf5fce8003" +version = "5.0.0-beta.5" +source = "git+https://github.com/lance-format/lance.git?rev=d630106d#d630106da5a238b3adfb8c5dea3b3921f3519945" dependencies = [ "arrow", "arrow-arith", @@ -3155,7 +3180,7 @@ dependencies = [ "pin-project", "prost", "prost-types", - "rand 0.9.2", + "rand 0.9.4", "roaring", "semver", "serde", @@ -3172,9 +3197,8 @@ dependencies = [ [[package]] name = "lance-arrow" -version = "3.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c9fdaf99863fa0d631e422881e88be4837d8b82f36a87143d723a9d285acec4b" +version = "5.0.0-beta.5" +source = "git+https://github.com/lance-format/lance.git?rev=d630106d#d630106da5a238b3adfb8c5dea3b3921f3519945" dependencies = [ "arrow-array", "arrow-buffer", @@ -3189,14 +3213,13 @@ dependencies = [ "half", "jsonb", "num-traits", - "rand 0.9.2", + "rand 0.9.4", ] [[package]] name = "lance-bitpacking" -version = "3.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "866b1634d38d94e8ab86fbcf238ac82dc8a5f72a4a6a90525f29899772e7cc7f" +version = "5.0.0-beta.5" +source = "git+https://github.com/lance-format/lance.git?rev=d630106d#d630106da5a238b3adfb8c5dea3b3921f3519945" dependencies = [ "arrayref", "paste", @@ -3225,13 +3248,13 @@ dependencies = [ "snafu", "tempfile", "tokio", + "uuid", ] [[package]] name = "lance-core" -version = "3.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "977c29f4e48c201c2806fe6ae117b65d0287eda236acd07357b556a54b0d5c5a" +version = "5.0.0-beta.5" +source = "git+https://github.com/lance-format/lance.git?rev=d630106d#d630106da5a238b3adfb8c5dea3b3921f3519945" dependencies = [ "arrow-array", "arrow-buffer", @@ -3254,7 +3277,7 @@ dependencies = [ "object_store", "pin-project", "prost", - "rand 0.9.2", + "rand 0.9.4", "roaring", "serde_json", "snafu", @@ -3268,9 +3291,8 @@ dependencies = [ [[package]] name = "lance-datafusion" -version = "3.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0ccc72695473f4207df4c6df3b347a63e84c32c0bc36bf42a7d86e8a7c0c67e2" +version = "5.0.0-beta.5" +source = "git+https://github.com/lance-format/lance.git?rev=d630106d#d630106da5a238b3adfb8c5dea3b3921f3519945" dependencies = [ "arrow", "arrow-array", @@ -3301,9 +3323,8 @@ dependencies = [ [[package]] name = "lance-datagen" -version = "3.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8fe84d76944acd834ded14d7562663af995556e0c6594f4b4ac69b0183f99c1a" +version = "5.0.0-beta.5" +source = "git+https://github.com/lance-format/lance.git?rev=d630106d#d630106da5a238b3adfb8c5dea3b3921f3519945" dependencies = [ "arrow", "arrow-array", @@ -3313,7 +3334,7 @@ dependencies = [ "futures", "half", "hex", - "rand 0.9.2", + "rand 0.9.4", "rand_distr 0.5.1", "rand_xoshiro", "random_word", @@ -3321,9 +3342,8 @@ dependencies = [ [[package]] name = "lance-encoding" -version = "3.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "be1007242188e5d53c98717e7f2cb340dc80eb9c94c2b935587598919b3a36bd" +version = "5.0.0-beta.5" +source = "git+https://github.com/lance-format/lance.git?rev=d630106d#d630106da5a238b3adfb8c5dea3b3921f3519945" dependencies = [ "arrow-arith", "arrow-array", @@ -3349,7 +3369,7 @@ dependencies = [ "prost", "prost-build", "prost-types", - "rand 0.9.2", + "rand 0.9.4", "snafu", "strum", "tokio", @@ -3360,9 +3380,8 @@ dependencies = [ [[package]] name = "lance-file" -version = "3.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f80088e418941f39cf5599d166ae1a6ef498cc2d967652a0692477d4871a9277" +version = "5.0.0-beta.5" +source = "git+https://github.com/lance-format/lance.git?rev=d630106d#d630106da5a238b3adfb8c5dea3b3921f3519945" dependencies = [ "arrow-arith", "arrow-array", @@ -3394,9 +3413,8 @@ dependencies = [ [[package]] name = "lance-geo" -version = "3.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3706c383d6d7a51d0a0f0b32e3bd55f71e0f913e61839edd78eb38f37507e5b3" +version = "5.0.0-beta.5" +source = "git+https://github.com/lance-format/lance.git?rev=d630106d#d630106da5a238b3adfb8c5dea3b3921f3519945" dependencies = [ "datafusion", "geo-traits", @@ -3410,9 +3428,8 @@ dependencies = [ [[package]] name = "lance-index" -version = "3.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e0011daf1ddde99becffd2ae235ad324576736a526c54ffbc4d7e583872f1215" +version = "5.0.0-beta.5" +source = "git+https://github.com/lance-format/lance.git?rev=d630106d#d630106da5a238b3adfb8c5dea3b3921f3519945" dependencies = [ "arrow", "arrow-arith", @@ -3426,6 +3443,7 @@ dependencies = [ "bitpacking", "bitvec", "bytes", + "chrono", "crossbeam-queue", "datafusion", "datafusion-common", @@ -3460,7 +3478,7 @@ dependencies = [ "prost", "prost-build", "prost-types", - "rand 0.9.2", + "rand 0.9.4", "rand_distr 0.5.1", "rangemap", "rayon", @@ -3479,9 +3497,8 @@ dependencies = [ [[package]] name = "lance-io" -version = "3.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cfa8a74e93753d19a27ce3adaeb99e31227df13ad5926dd43572be76b43dd284" +version = "5.0.0-beta.5" +source = "git+https://github.com/lance-format/lance.git?rev=d630106d#d630106da5a238b3adfb8c5dea3b3921f3519945" dependencies = [ "arrow", "arrow-arith", @@ -3501,17 +3518,20 @@ dependencies = [ "deepsize", "futures", "http 1.4.0", + "io-uring", "lance-arrow", "lance-core", "lance-namespace", + "libc", "log", + "moka", "object_store", "object_store_opendal", "opendal", "path_abs", "pin-project", "prost", - "rand 0.9.2", + "rand 0.9.4", "serde", "snafu", "tempfile", @@ -3522,9 +3542,8 @@ dependencies = [ [[package]] name = "lance-linalg" -version = "3.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6e2d8da8f6b8dd37ab3b8199896ee265817f86232e3727c0b0eeb3c9093b64d9" +version = "5.0.0-beta.5" +source = "git+https://github.com/lance-format/lance.git?rev=d630106d#d630106da5a238b3adfb8c5dea3b3921f3519945" dependencies = [ "arrow-array", "arrow-buffer", @@ -3535,28 +3554,28 @@ dependencies = [ "lance-arrow", "lance-core", "num-traits", - "rand 0.9.2", + "rand 0.9.4", ] [[package]] name = "lance-namespace" -version = "3.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f176e427d9c35938d8a7097876114bc35dfd280b06077779753f2effe3e86aab" +version = "5.0.0-beta.5" +source = "git+https://github.com/lance-format/lance.git?rev=d630106d#d630106da5a238b3adfb8c5dea3b3921f3519945" dependencies = [ "arrow", "async-trait", "bytes", "lance-core", "lance-namespace-reqwest-client", + "serde", "snafu", ] [[package]] name = "lance-namespace-reqwest-client" -version = "0.5.4" +version = "0.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "df9008f9825066088178c10599130c8bb0b9c79a39a479e8c51201620c43864a" +checksum = "ee2e48de899e2931afb67fcddd0a08e439bf5d8b6ea2a2ed9cb8f4df669bd5cc" dependencies = [ "reqwest", "serde", @@ -3567,9 +3586,8 @@ dependencies = [ [[package]] name = "lance-table" -version = "3.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aa189b3081481a97b64cf1161297947a63b8adb941b1950989d0269858703a43" +version = "5.0.0-beta.5" +source = "git+https://github.com/lance-format/lance.git?rev=d630106d#d630106da5a238b3adfb8c5dea3b3921f3519945" dependencies = [ "arrow", "arrow-array", @@ -3591,7 +3609,7 @@ dependencies = [ "prost", "prost-build", "prost-types", - "rand 0.9.2", + "rand 0.9.4", "rangemap", "roaring", "semver", @@ -3684,9 +3702,9 @@ dependencies = [ [[package]] name = "libc" -version = "0.2.183" +version = "0.2.186" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b5b646652bf6661599e1da8901b3b9522896f01e736bad5f723fe7a3a27f899d" +checksum = "68ab91017fe16c622486840e4c83c9a37afeff978bd239b5293d61ece587de66" [[package]] name = "libm" @@ -3696,9 +3714,9 @@ checksum = "b6d2cec3eae94f9f509c767b45932f1ada8350c4bdb85af2fcab4a3c14807981" [[package]] name = "libredox" -version = "0.1.15" +version = "0.1.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7ddbf48fd451246b1f8c2610bd3b4ac0cc6e149d89832867093ab69a17194f08" +checksum = "e02f3bb43d335493c96bf3fd3a321600bf6bd07ed34bc64118e9293bdffea46c" dependencies = [ "libc", ] @@ -3717,9 +3735,9 @@ checksum = "32a66949e030da00e8c7d4434b251670a91556f4144941d37452769c25d58a53" [[package]] name = "litemap" -version = "0.8.1" +version = "0.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6373607a59f0be73a39b6fe456b8192fcc3585f602af20751600e974dd455e77" +checksum = "92daf443525c4cce67b150400bc2316076100ce0b3686209eb8cf3c31612e6f0" [[package]] name = "lock_api" @@ -3888,9 +3906,9 @@ dependencies = [ [[package]] name = "mio" -version = "1.1.1" +version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a69bcab0ad47271a0234d9422b131806bf3968021e5dc9328caf2d4cd58557fc" +checksum = "50b7e5b27aa02a74bac8c3f23f448f8d87ff11f92d3aac1a6ed369ee08cc56c1" dependencies = [ "libc", "wasi", @@ -3999,7 +4017,7 @@ dependencies = [ "num-integer", "num-iter", "num-traits", - "rand 0.8.5", + "rand 0.8.6", "smallvec", "zeroize", ] @@ -4015,9 +4033,9 @@ dependencies = [ [[package]] name = "num-conv" -version = "0.2.0" +version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cf97ec579c3c42f953ef76dbf8d55ac91fb219dde70e49aa4a6b7d74e9919050" +checksum = "c6673768db2d862beb9b39a78fdcb1a69439615d5794a1be50caa9bc92c81967" [[package]] name = "num-integer" @@ -4103,7 +4121,7 @@ dependencies = [ "parking_lot", "percent-encoding", "quick-xml 0.38.4", - "rand 0.9.2", + "rand 0.9.4", "reqwest", "ring", "rustls-pemfile", @@ -4191,9 +4209,9 @@ checksum = "04744f49eae99ab78e0d5c0b603ab218f515ea8cfe5a456d7629ad883a3b6e7d" [[package]] name = "ordered-float" -version = "5.1.0" +version = "5.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7f4779c6901a562440c3786d08192c6fbda7c1c2060edd10006b05ee35d10f2d" +checksum = "b7d950ca161dc355eaf28f82b11345ed76c6e1f6eb1f4f4479e0323b9e2fbd0e" dependencies = [ "num-traits", ] @@ -4413,9 +4431,9 @@ dependencies = [ [[package]] name = "pkg-config" -version = "0.3.32" +version = "0.3.33" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7edddbd0b52d732b21ad9a5fab5c704c14cd949e5e9a1ec5929a24fded1b904c" +checksum = "19f132c84eca552bf34cab8ec81f1c1dcc229b811638f9d283dceabe58c5569e" [[package]] name = "portable-atomic" @@ -4425,18 +4443,18 @@ checksum = "c33a9471896f1c69cecef8d20cbe2f7accd12527ce60845ff44c153bb2a21b49" [[package]] name = "portable-atomic-util" -version = "0.2.6" +version = "0.2.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "091397be61a01d4be58e7841595bd4bfedb15f1cd54977d79b8271e94ed799a3" +checksum = "c2a106d1259c23fac8e543272398ae0e3c0b8d33c88ed73d0cc71b0f1d902618" dependencies = [ "portable-atomic", ] [[package]] name = "potential_utf" -version = "0.1.4" +version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b73949432f5e2a09657003c25bca5e19a0e9c84f8058ca374f49e0ebe605af77" +checksum = "0103b1cef7ec0cf76490e969665504990193874ea05c85ff9bab8b911d0a0564" dependencies = [ "zerovec", ] @@ -4584,7 +4602,7 @@ dependencies = [ "bytes", "getrandom 0.3.4", "lru-slab", - "rand 0.9.2", + "rand 0.9.4", "ring", "rustc-hash", "rustls", @@ -4639,9 +4657,9 @@ checksum = "dc33ff2d4973d518d823d61aa239014831e521c75da58e3df4840d3f47749d09" [[package]] name = "rand" -version = "0.8.5" +version = "0.8.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "34af8d1a0e25924bc5b7c43c079c942339d8f0a8b57c39049bef581b46327404" +checksum = "5ca0ecfa931c29007047d1bc58e623ab12e5590e8c7cc53200d5202b69266d8a" dependencies = [ "libc", "rand_chacha 0.3.1", @@ -4650,9 +4668,9 @@ dependencies = [ [[package]] name = "rand" -version = "0.9.2" +version = "0.9.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6db2770f06117d490610c7488547d543617b21bfa07796d7a12f6f1bd53850d1" +checksum = "44c5af06bb1b7d3216d91932aed5265164bf384dc89cd6ba05cf59a35f5f76ea" dependencies = [ "rand_chacha 0.9.0", "rand_core 0.9.5", @@ -4703,7 +4721,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "32cb0b9bc82b0a0876c2dd994a7e7a2683d3e7390ca40e6886785ef0c7e3ee31" dependencies = [ "num-traits", - "rand 0.8.5", + "rand 0.8.6", ] [[package]] @@ -4713,7 +4731,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6a8615d50dcf34fa31f7ab52692afec947c4dd0ab803cc87cb3b0b4570ff7463" dependencies = [ "num-traits", - "rand 0.9.2", + "rand 0.9.4", ] [[package]] @@ -4734,7 +4752,7 @@ dependencies = [ "ahash", "brotli", "paste", - "rand 0.9.2", + "rand 0.9.4", "unicase", ] @@ -4752,9 +4770,9 @@ checksum = "60a357793950651c4ed0f3f52338f53b2f809f32d83a07f72909fa13e4c6c1e3" [[package]] name = "rayon" -version = "1.11.0" +version = "1.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "368f01d005bf8fd9b1206fb6fa653e6c4a81ceb1466406b81792d87c5677a58f" +checksum = "fb39b166781f92d482534ef4b4b1b2568f42613b53e5b6c160e24cfbfa30926d" dependencies = [ "either", "rayon-core", @@ -4846,7 +4864,7 @@ dependencies = [ "once_cell", "percent-encoding", "quick-xml 0.37.5", - "rand 0.8.5", + "rand 0.8.6", "reqwest", "rsa", "rust-ini", @@ -4919,9 +4937,9 @@ dependencies = [ [[package]] name = "roaring" -version = "0.11.3" +version = "0.11.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8ba9ce64a8f45d7fc86358410bb1a82e8c987504c0d4900e9141d69a9f26c885" +checksum = "1dedc5658c6ecb3bdb5ef5f3295bb9253f42dcf3fd1402c03f6b1f7659c3c4a9" dependencies = [ "bytemuck", "byteorder", @@ -4987,9 +5005,9 @@ dependencies = [ [[package]] name = "rustc-hash" -version = "2.1.1" +version = "2.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "357703d41365b4b27c590e3ed91eabb1b663f07c4c084095e60cbed4362dff0d" +checksum = "94300abf3f1ae2e2b8ffb7b58043de3d399c73fa6f4b73826402a5c457614dbe" [[package]] name = "rustc_version" @@ -5028,9 +5046,9 @@ dependencies = [ [[package]] name = "rustls" -version = "0.23.37" +version = "0.23.39" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "758025cb5fccfd3bc2fd74708fd4682be41d99e5dff73c377c0646c6012c73a4" +checksum = "7c2c118cb077cca2822033836dfb1b975355dfb784b5e8da48f7b6c5db74e60e" dependencies = [ "aws-lc-rs", "once_cell", @@ -5064,9 +5082,9 @@ dependencies = [ [[package]] name = "rustls-pki-types" -version = "1.14.0" +version = "1.14.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "be040f8b0a225e40375822a563fa9524378b9d63112f53e19ffff34df5d33fdd" +checksum = "30a7197ae7eb376e574fe940d068c30fe0462554a3ddbe4eca7838e049c937a9" dependencies = [ "web-time", "zeroize", @@ -5074,9 +5092,9 @@ dependencies = [ [[package]] name = "rustls-webpki" -version = "0.103.10" +version = "0.103.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "df33b2b81ac578cabaf06b89b0631153a3f416b0a886e8a7a1707fb51abbd1ef" +checksum = "61c429a8649f110dddef65e2a5ad240f747e85f7758a6bccc7e5777bd33f756e" dependencies = [ "aws-lc-rs", "ring", @@ -5171,9 +5189,9 @@ dependencies = [ [[package]] name = "semver" -version = "1.0.27" +version = "1.0.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d767eb0aabc880b29956c35734170f26ed551a859dbd361d140cdbeca61ab1e2" +checksum = "8a7852d02fc848982e0c167ef163aaff9cd91dc640ba85e263cb1ce46fae51cd" [[package]] name = "seq-macro" @@ -5254,7 +5272,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e3bf829a2d51ab4a5ddf1352d8470c140cadc8301b2ae1789db023f01cedd6ba" dependencies = [ "cfg-if", - "cpufeatures", + "cpufeatures 0.2.17", "digest", ] @@ -5265,7 +5283,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283" dependencies = [ "cfg-if", - "cpufeatures", + "cpufeatures 0.2.17", "digest", ] @@ -5306,9 +5324,9 @@ dependencies = [ [[package]] name = "simd-adler32" -version = "0.3.8" +version = "0.3.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e320a6c5ad31d271ad523dcf3ad13e2767ad8b1cb8f047f75a8aeaf8da139da2" +checksum = "703d5c7ef118737c72f1af64ad2f6f8c5e1921f818cdcb97b8fe6fc69bf66214" [[package]] name = "simdutf8" @@ -5388,11 +5406,11 @@ dependencies = [ [[package]] name = "spade" -version = "2.15.0" +version = "2.15.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fb313e1c8afee5b5647e00ee0fe6855e3d529eb863a0fdae1d60006c4d1e9990" +checksum = "9699399fd9349b00b184f5635b074f9ec93afffef30c853f8c875b32c0f8c7fa" dependencies = [ - "hashbrown 0.15.5", + "hashbrown 0.16.1", "num-traits", "robust", "smallvec", @@ -5800,9 +5818,9 @@ dependencies = [ [[package]] name = "tinystr" -version = "0.8.2" +version = "0.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "42d3e9c45c09de15d06dd8acf5f4e0e399e85927b7f00711024eb7ae10fa4869" +checksum = "c8323304221c2a851516f22236c5722a72eaa19749016521d6dff0824447d96d" dependencies = [ "displaydoc", "zerovec", @@ -5825,9 +5843,9 @@ checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" [[package]] name = "tokio" -version = "1.50.0" +version = "1.52.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "27ad5e34374e03cfffefc301becb44e9dc3c17584f414349ebe29ed26661822d" +checksum = "b67dee974fe86fd92cc45b7a95fdd2f99a36a6d7b0d431a231178d3d670bbcc6" dependencies = [ "bytes", "libc", @@ -5841,9 +5859,9 @@ dependencies = [ [[package]] name = "tokio-macros" -version = "2.6.1" +version = "2.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5c55a2eff8b69ce66c84f85e1da1c233edc36ceb85a2058d11b0d6a3c7e7569c" +checksum = "385a6cb71ab9ab790c5fe8d67f1645e6c450a7ce006a33de03daa956cf70a496" dependencies = [ "proc-macro2", "quote", @@ -5886,18 +5904,18 @@ dependencies = [ [[package]] name = "toml_datetime" -version = "1.1.0+spec-1.1.0" +version = "1.1.1+spec-1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "97251a7c317e03ad83774a8752a7e81fb6067740609f75ea2b585b569a59198f" +checksum = "3165f65f62e28e0115a00b2ebdd37eb6f3b641855f9d636d3cd4103767159ad7" dependencies = [ "serde_core", ] [[package]] name = "toml_edit" -version = "0.25.8+spec-1.1.0" +version = "0.25.11+spec-1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "16bff38f1d86c47f9ff0647e6838d7bb362522bdf44006c7068c2b1e606f1f3c" +checksum = "0b59c4d22ed448339746c59b905d24568fcbb3ab65a500494f7b8c3e97739f2b" dependencies = [ "indexmap", "toml_datetime", @@ -5907,9 +5925,9 @@ dependencies = [ [[package]] name = "toml_parser" -version = "1.1.0+spec-1.1.0" +version = "1.1.2+spec-1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2334f11ee363607eb04df9b8fc8a13ca1715a72ba8662a26ac285c98aabb4011" +checksum = "a2abe9b86193656635d2411dc43050282ca48aa31c2451210f4202550afb7526" dependencies = [ "winnow", ] @@ -6037,14 +6055,14 @@ version = "2.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9ea3136b675547379c4bd395ca6b938e5ad3c3d20fad76e7fe85f9e0d011419c" dependencies = [ - "rand 0.9.2", + "rand 0.9.4", ] [[package]] name = "typenum" -version = "1.19.0" +version = "1.20.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "562d481066bde0658276a35467c4af00bdc6ee726305698a55b86e61d7ad82bb" +checksum = "40ce102ab67701b8526c123c1bab5cbe42d7040ccfd0f64af1a385808d2f43de" [[package]] name = "unicase" @@ -6060,9 +6078,9 @@ checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" [[package]] name = "unicode-segmentation" -version = "1.13.0" +version = "1.13.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a559e63b5d8004e12f9bce88af5c6d939c58de839b7532cfe9653846cedd2a9e" +checksum = "9629274872b2bfaf8d66f5f15725007f635594914870f65218920345aa11aa8c" [[package]] name = "unicode-width" @@ -6114,9 +6132,9 @@ checksum = "b6c140620e7ffbb22c2dee59cafe6084a59b5ffc27a8859a5f0d494b5d52b6be" [[package]] name = "uuid" -version = "1.22.0" +version = "1.23.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a68d3c8f01c0cfa54a75291d83601161799e4a89a39e0929f4b0354d88757a37" +checksum = "ddd74a9687298c6858e9b88ec8935ec45d22e8fd5e6394fa1bd4e99a87789c76" dependencies = [ "getrandom 0.4.2", "js-sys", @@ -6169,11 +6187,11 @@ checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b" [[package]] name = "wasip2" -version = "1.0.2+wasi-0.2.9" +version = "1.0.3+wasi-0.2.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9517f9239f02c069db75e65f174b3da828fe5f5b945c4dd26bd25d89c03ebcf5" +checksum = "20064672db26d7cdc89c7798c48a0fdfac8213434a1186e5ef29fd560ae223d6" dependencies = [ - "wit-bindgen", + "wit-bindgen 0.57.1", ] [[package]] @@ -6182,14 +6200,14 @@ version = "0.4.0+wasi-0.3.0-rc-2026-01-06" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5428f8bf88ea5ddc08faddef2ac4a67e390b88186c703ce6dbd955e1c145aca5" dependencies = [ - "wit-bindgen", + "wit-bindgen 0.51.0", ] [[package]] name = "wasm-bindgen" -version = "0.2.114" +version = "0.2.118" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6532f9a5c1ece3798cb1c2cfdba640b9b3ba884f5db45973a6f442510a87d38e" +checksum = "0bf938a0bacb0469e83c1e148908bd7d5a6010354cf4fb73279b7447422e3a89" dependencies = [ "cfg-if", "once_cell", @@ -6200,23 +6218,19 @@ dependencies = [ [[package]] name = "wasm-bindgen-futures" -version = "0.4.64" +version = "0.4.68" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e9c5522b3a28661442748e09d40924dfb9ca614b21c00d3fd135720e48b67db8" +checksum = "f371d383f2fb139252e0bfac3b81b265689bf45b6874af544ffa4c975ac1ebf8" dependencies = [ - "cfg-if", - "futures-util", "js-sys", - "once_cell", "wasm-bindgen", - "web-sys", ] [[package]] name = "wasm-bindgen-macro" -version = "0.2.114" +version = "0.2.118" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "18a2d50fcf105fb33bb15f00e7a77b772945a2ee45dcf454961fd843e74c18e6" +checksum = "eeff24f84126c0ec2db7a449f0c2ec963c6a49efe0698c4242929da037ca28ed" dependencies = [ "quote", "wasm-bindgen-macro-support", @@ -6224,9 +6238,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro-support" -version = "0.2.114" +version = "0.2.118" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "03ce4caeaac547cdf713d280eda22a730824dd11e6b8c3ca9e42247b25c631e3" +checksum = "9d08065faf983b2b80a79fd87d8254c409281cf7de75fc4b773019824196c904" dependencies = [ "bumpalo", "proc-macro2", @@ -6237,9 +6251,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-shared" -version = "0.2.114" +version = "0.2.118" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "75a326b8c223ee17883a4251907455a2431acc2791c98c26279376490c378c16" +checksum = "5fd04d9e306f1907bd13c6361b5c6bfc7b3b3c095ed3f8a9246390f8dbdee129" dependencies = [ "unicode-ident", ] @@ -6293,9 +6307,9 @@ dependencies = [ [[package]] name = "web-sys" -version = "0.3.91" +version = "0.3.95" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "854ba17bb104abfb26ba36da9729addc7ce7f06f5c0f90f3c391f8461cca21f9" +checksum = "4f2dfbb17949fa2088e5d39408c48368947b86f7834484e87b73de55bc14d97d" dependencies = [ "js-sys", "wasm-bindgen", @@ -6313,9 +6327,9 @@ dependencies = [ [[package]] name = "webpki-roots" -version = "1.0.6" +version = "1.0.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "22cfaf3c063993ff62e73cb4311efde4db1efb31ab78a3e5c457939ad5cc0bed" +checksum = "52f5ee44c96cf55f1b349600768e3ece3a8f26010c05265ab73f945bb1a2eb9d" dependencies = [ "rustls-pki-types", ] @@ -6577,9 +6591,9 @@ checksum = "d6bbff5f0aada427a1e5a6da5f1f98158182f26556f345ac9e04d36d0ebed650" [[package]] name = "winnow" -version = "1.0.0" +version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a90e88e4667264a994d34e6d1ab2d26d398dcdca8b7f52bec8668957517fc7d8" +checksum = "2ee1708bef14716a11bae175f579062d4554d95be2c6829f518df847b7b3fdd0" dependencies = [ "memchr", ] @@ -6593,6 +6607,12 @@ dependencies = [ "wit-bindgen-rust-macro", ] +[[package]] +name = "wit-bindgen" +version = "0.57.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ebf944e87a7c253233ad6766e082e3cd714b5d03812acc24c318f549614536e" + [[package]] name = "wit-bindgen-core" version = "0.51.0" @@ -6699,9 +6719,9 @@ dependencies = [ [[package]] name = "writeable" -version = "0.6.2" +version = "0.6.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9edde0db4769d2dc68579893f2306b26c6ecfbe0ef499b013d731b7b9247e0b9" +checksum = "1ffae5123b2d3fc086436f8834ae3ab053a283cfac8fe0a0b8eaae044768a4c4" [[package]] name = "wyz" @@ -6726,9 +6746,9 @@ checksum = "fdd20c5420375476fbd4394763288da7eb0cc0b8c11deed431a91562af7335d3" [[package]] name = "yoke" -version = "0.8.1" +version = "0.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "72d6e5c6afb84d73944e5cedb052c4680d5657337201555f9f2a16b7406d4954" +checksum = "abe8c5fda708d9ca3df187cae8bfb9ceda00dd96231bed36e445a1a48e66f9ca" dependencies = [ "stable_deref_trait", "yoke-derive", @@ -6737,9 +6757,9 @@ dependencies = [ [[package]] name = "yoke-derive" -version = "0.8.1" +version = "0.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b659052874eb698efe5b9e8cf382204678a0086ebf46982b79d6ca3182927e5d" +checksum = "de844c262c8848816172cef550288e7dc6c7b7814b4ee56b3e1553f275f1858e" dependencies = [ "proc-macro2", "quote", @@ -6749,18 +6769,18 @@ dependencies = [ [[package]] name = "zerocopy" -version = "0.8.47" +version = "0.8.48" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "efbb2a062be311f2ba113ce66f697a4dc589f85e78a4aea276200804cea0ed87" +checksum = "eed437bf9d6692032087e337407a86f04cd8d6a16a37199ed57949d415bd68e9" dependencies = [ "zerocopy-derive", ] [[package]] name = "zerocopy-derive" -version = "0.8.47" +version = "0.8.48" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0e8bc7269b54418e7aeeef514aa68f8690b8c0489a06b0136e5f57c4c5ccab89" +checksum = "70e3cd084b1788766f53af483dd21f93881ff30d7320490ec3ef7526d203bad4" dependencies = [ "proc-macro2", "quote", @@ -6769,18 +6789,18 @@ dependencies = [ [[package]] name = "zerofrom" -version = "0.1.6" +version = "0.1.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "50cc42e0333e05660c3587f3bf9d0478688e15d870fab3346451ce7f8c9fbea5" +checksum = "69faa1f2a1ea75661980b013019ed6687ed0e83d069bc1114e2cc74c6c04c4df" dependencies = [ "zerofrom-derive", ] [[package]] name = "zerofrom-derive" -version = "0.1.6" +version = "0.1.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d71e5d6e06ab090c67b5e44993ec16b72dcbaabc526db883a360057678b48502" +checksum = "11532158c46691caf0f2593ea8358fed6bbf68a0315e80aae9bd41fbade684a1" dependencies = [ "proc-macro2", "quote", @@ -6796,9 +6816,9 @@ checksum = "b97154e67e32c85465826e8bcc1c59429aaaf107c1e4a9e53c8d8ccd5eff88d0" [[package]] name = "zerotrie" -version = "0.2.3" +version = "0.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2a59c17a5562d507e4b54960e8569ebee33bee890c70aa3fe7b97e85a9fd7851" +checksum = "0f9152d31db0792fa83f70fb2f83148effb5c1f5b8c7686c3459e361d9bc20bf" dependencies = [ "displaydoc", "yoke", @@ -6807,9 +6827,9 @@ dependencies = [ [[package]] name = "zerovec" -version = "0.11.5" +version = "0.11.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6c28719294829477f525be0186d13efa9a3c602f7ec202ca9e353d310fb9a002" +checksum = "90f911cbc359ab6af17377d242225f4d75119aec87ea711a880987b18cd7b239" dependencies = [ "yoke", "zerofrom", @@ -6818,9 +6838,9 @@ dependencies = [ [[package]] name = "zerovec-derive" -version = "0.11.2" +version = "0.11.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eadce39539ca5cb3985590102671f2567e659fca9666581ad3411d59207951f3" +checksum = "625dc425cab0dca6dc3c3319506e6593dcb08a9f387ea3b284dbd52a92c40555" dependencies = [ "proc-macro2", "quote", diff --git a/Cargo.toml b/Cargo.toml index de53d6e..4d3b626 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -18,11 +18,11 @@ rust-version = "1.91.0" crate-type = ["cdylib", "staticlib", "rlib"] [dependencies] -lance = "3.0.1" -lance-core = "3.0.1" -lance-index = "3.0.1" -lance-io = "3.0.1" -lance-linalg = "3.0.1" +lance = { git = "https://github.com/lance-format/lance.git", rev = "d630106d" } +lance-core = { git = "https://github.com/lance-format/lance.git", rev = "d630106d" } +lance-index = { git = "https://github.com/lance-format/lance.git", rev = "d630106d" } +lance-io = { git = "https://github.com/lance-format/lance.git", rev = "d630106d" } +lance-linalg = { git = "https://github.com/lance-format/lance.git", rev = "d630106d" } arrow = { version = "57.0.0", features = ["prettyprint", "ffi"] } arrow-array = "57.0.0" arrow-schema = "57.0.0" @@ -32,12 +32,13 @@ futures = "0.3" log = "0.4" pin-project = "1.0" snafu = "0.9" +uuid = { version = "1", features = ["v4"] } [dev-dependencies] -lance = "3.0.1" -lance-datagen = "3.0.1" -lance-file = "3.0.1" -lance-table = "3.0.1" +lance = { git = "https://github.com/lance-format/lance.git", rev = "d630106d" } +lance-datagen = { git = "https://github.com/lance-format/lance.git", rev = "d630106d" } +lance-file = { git = "https://github.com/lance-format/lance.git", rev = "d630106d" } +lance-table = { git = "https://github.com/lance-format/lance.git", rev = "d630106d" } tokio = { version = "1", features = ["rt-multi-thread", "macros"] } arrow-array = "57.0.0" arrow-schema = "57.0.0" diff --git a/include/lance.h b/include/lance.h index 0a6ce0d..7dedd0b 100644 --- a/include/lance.h +++ b/include/lance.h @@ -431,6 +431,33 @@ uint64_t lance_dataset_index_count(const LanceDataset* dataset); */ const char* lance_dataset_index_list_json(const LanceDataset* dataset); +/* ─── Distributed vector search: index segment enumeration ─── */ + +/** + * Count the segments that make up a logical vector index. + * + * A logical index is a set of physical segments (one per distributed-build + * worker, or one per fragment range). Each segment has a stable UUID. Returns + * 0 if the index does not exist (also sets `LANCE_ERR_NOT_FOUND`) or on error. + */ +uint64_t lance_dataset_index_segment_count( + const LanceDataset* dataset, + const char* index_name +); + +/** + * Fill `out_uuids` with the UUIDs of the segments that make up a logical index. + * Each UUID is written as 16 raw bytes (RFC 4122 layout). The caller must + * allocate at least `lance_dataset_index_segment_count() * 16` bytes. + * + * Returns 0 on success, -1 on error. + */ +int32_t lance_dataset_index_segments( + const LanceDataset* dataset, + const char* index_name, + uint8_t* out_uuids +); + /* ─── Vector search (Phase 2) ─── */ /** @@ -460,6 +487,26 @@ int32_t lance_scanner_set_metric(LanceScanner* scanner, LanceMetricType metric); int32_t lance_scanner_set_use_index(LanceScanner* scanner, bool enable); int32_t lance_scanner_set_prefilter(LanceScanner* scanner, bool enable); +/** + * Restrict the next k-NN query to a specific subset of vector index segments. + * + * Used by distributed query engines (e.g. Velox) to fan a single k-NN query + * out across workers, each handling a slice of segments. The coordinator gets + * the segment list via `lance_dataset_index_segments()`. + * + * @param segment_uuids Pointer to `len` 16-byte UUIDs concatenated end-to-end + * (total byte length = `len * 16`). Each UUID identifies + * one physical segment of a logical index. + * @param len Number of UUIDs. Pass 0 (and segment_uuids may be NULL) + * to clear any previously-set segment restriction. + * @return 0 on success, -1 on error. + */ +int32_t lance_scanner_set_index_segments( + LanceScanner* scanner, + const uint8_t* segment_uuids, + size_t len +); + #ifdef __cplusplus } /* extern "C" */ #endif diff --git a/src/index.rs b/src/index.rs index 84df8b3..c088bbc 100644 --- a/src/index.rs +++ b/src/index.rs @@ -8,9 +8,10 @@ use std::ffi::{CString, c_char}; +use lance::index::DatasetIndexExt; use lance_core::Result; +use lance_index::IndexType; use lance_index::scalar::{BuiltinIndexType, ScalarIndexParams}; -use lance_index::{DatasetIndexExt, IndexType}; use crate::dataset::LanceDataset; use crate::error::{LanceErrorCode, ffi_try, set_last_error}; diff --git a/src/scanner.rs b/src/scanner.rs index 08a7abb..237bf5d 100644 --- a/src/scanner.rs +++ b/src/scanner.rs @@ -17,6 +17,7 @@ use lance::dataset::scanner::DatasetRecordBatchStream; use lance_core::Result; use lance_io::ffi::to_ffi_arrow_array_stream; use lance_io::stream::RecordBatchStream; +use uuid::Uuid; use crate::async_dispatcher::{self, LanceCallback}; use crate::batch::LanceBatch; @@ -46,6 +47,7 @@ pub struct LanceScanner { batch_size: Option, with_row_id: bool, fragment_ids: Option>, + index_segments: Option>, nearest: Option, nprobes: Option, refine_factor: Option, @@ -95,6 +97,7 @@ impl LanceScanner { batch_size: None, with_row_id: false, fragment_ids: None, + index_segments: None, nearest: None, nprobes: None, refine_factor: None, @@ -161,6 +164,9 @@ impl LanceScanner { if self.prefilter { scanner.prefilter(true); } + if let Some(segments) = &self.index_segments { + scanner.with_index_segments(segments.clone())?; + } } let stream = block_on(scanner.try_into_stream())?; self.schema = Some(stream.schema()); @@ -207,6 +213,9 @@ impl LanceScanner { if self.prefilter { scanner.prefilter(true); } + if let Some(segments) = &self.index_segments { + scanner.with_index_segments(segments.clone())?; + } } Ok(scanner) } @@ -727,6 +736,52 @@ pub unsafe extern "C" fn lance_scanner_set_prefilter( 0 } +/// Restrict the next `nearest()` query to a specific subset of vector index segments. +/// +/// Each segment is a 16-byte UUID (RFC 4122 layout). Pass an array of `len` +/// 16-byte buffers concatenated end-to-end (so the total byte length is `len * 16`). +/// Used by distributed query engines (e.g. Velox) to fan k-NN out across workers, +/// each handling a slice of segments. The coordinator gets the segment list via +/// `lance_dataset_index_segments()`. +/// +/// Calling with `len == 0` clears the segment restriction. +/// +/// Returns 0 on success, -1 on error. +#[unsafe(no_mangle)] +pub unsafe extern "C" fn lance_scanner_set_index_segments( + scanner: *mut LanceScanner, + segment_uuids: *const u8, + len: usize, +) -> i32 { + if scanner.is_null() { + set_last_error(LanceErrorCode::InvalidArgument, "scanner is NULL"); + return -1; + } + if segment_uuids.is_null() && len > 0 { + set_last_error( + LanceErrorCode::InvalidArgument, + "segment_uuids is NULL but len > 0", + ); + return -1; + } + let s = unsafe { &mut *scanner }; + if len == 0 { + s.index_segments = None; + } else { + let mut uuids = Vec::with_capacity(len); + for i in 0..len { + let mut bytes = [0u8; 16]; + unsafe { + std::ptr::copy_nonoverlapping(segment_uuids.add(i * 16), bytes.as_mut_ptr(), 16); + } + uuids.push(Uuid::from_bytes(bytes)); + } + s.index_segments = Some(uuids); + } + crate::error::clear_last_error(); + 0 +} + // --------------------------------------------------------------------------- // Vector search (Phase 2): k-NN query setter // --------------------------------------------------------------------------- From 9c75e1e7f89e5406c41da9921d2757d9498359bb Mon Sep 17 00:00:00 2001 From: "jianjian.xie" Date: Fri, 24 Apr 2026 16:03:35 -0700 Subject: [PATCH 2/4] feat(index): expose vector index segments through C ABI Adds lance_dataset_index_segment_count(name) and lance_dataset_index_segments(name, out_uuids) to enumerate the physical segments of a logical vector index. Each segment is identified by its 16-byte UUID (RFC 4122 layout, written as raw bytes into a caller-allocated buffer of len*16 bytes). The header also declares lance_scanner_set_index_segments (impl in the next commit). Both pieces together let a distributed query engine like Velox shard a single k-NN query across workers. --- src/index.rs | 107 +++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 107 insertions(+) diff --git a/src/index.rs b/src/index.rs index c088bbc..bde31ab 100644 --- a/src/index.rs +++ b/src/index.rs @@ -154,6 +154,113 @@ pub unsafe extern "C" fn lance_dataset_index_count(dataset: *const LanceDataset) } } +/// Count the segments that make up a logical index. +/// +/// A logical index is a set of physical segments (one per distributed-build worker +/// or one per fragment range). Each segment has a stable UUID. Returns 0 if the +/// index does not exist (also sets `LANCE_ERR_NOT_FOUND`). +#[unsafe(no_mangle)] +pub unsafe extern "C" fn lance_dataset_index_segment_count( + dataset: *const LanceDataset, + index_name: *const c_char, +) -> u64 { + if dataset.is_null() || index_name.is_null() { + set_last_error( + LanceErrorCode::InvalidArgument, + "dataset and index_name must not be NULL", + ); + return 0; + } + let ds = unsafe { &*dataset }; + let name = match unsafe { helpers::parse_c_string(index_name) } { + Ok(Some(s)) => s, + Ok(None) => { + set_last_error( + LanceErrorCode::InvalidArgument, + "index_name must not be empty", + ); + return 0; + } + Err(err) => { + crate::error::set_lance_error(&err); + return 0; + } + }; + let snap = ds.snapshot(); + match block_on(snap.load_indices()) { + Ok(indices) => { + let count = indices.iter().filter(|i| i.name == name).count(); + if count == 0 { + set_last_error( + LanceErrorCode::NotFound, + format!("index '{}' not found", name), + ); + return 0; + } + crate::error::clear_last_error(); + count as u64 + } + Err(err) => { + crate::error::set_lance_error(&err); + 0 + } + } +} + +/// Fill `out_uuids` with the UUIDs of the segments that make up a logical index. +/// +/// Each UUID is written as 16 raw bytes (RFC 4122 layout). The caller must +/// allocate at least `lance_dataset_index_segment_count() * 16` bytes. +/// +/// Returns 0 on success, -1 on error. +#[unsafe(no_mangle)] +pub unsafe extern "C" fn lance_dataset_index_segments( + dataset: *const LanceDataset, + index_name: *const c_char, + out_uuids: *mut u8, +) -> i32 { + ffi_try!( + unsafe { dataset_index_segments_inner(dataset, index_name, out_uuids) }, + neg + ) +} + +unsafe fn dataset_index_segments_inner( + dataset: *const LanceDataset, + index_name: *const c_char, + out_uuids: *mut u8, +) -> Result { + if dataset.is_null() || index_name.is_null() || out_uuids.is_null() { + return Err(lance_core::Error::InvalidInput { + source: "dataset, index_name, and out_uuids must not be NULL".into(), + location: snafu::location!(), + }); + } + let ds = unsafe { &*dataset }; + let name = unsafe { helpers::parse_c_string(index_name)? }.ok_or_else(|| { + lance_core::Error::InvalidInput { + source: "index_name must not be empty".into(), + location: snafu::location!(), + } + })?; + let snap = ds.snapshot(); + let indices = block_on(snap.load_indices())?; + let segments: Vec<_> = indices.iter().filter(|i| i.name == name).collect(); + if segments.is_empty() { + return Err(lance_core::Error::IndexNotFound { + identity: format!("name='{}'", name), + location: snafu::location!(), + }); + } + for (i, seg) in segments.iter().enumerate() { + let bytes = seg.uuid.as_bytes(); + unsafe { + std::ptr::copy_nonoverlapping(bytes.as_ptr(), out_uuids.add(i * 16), 16); + } + } + Ok(0) +} + /// Drop an index by name. #[unsafe(no_mangle)] pub unsafe extern "C" fn lance_dataset_drop_index( From dd6dd799a81d51c17bf8ac5856614aced83b7d73 Mon Sep 17 00:00:00 2001 From: "jianjian.xie" Date: Fri, 24 Apr 2026 16:04:17 -0700 Subject: [PATCH 3/4] test: distributed search via index segments Five tests covering the new segment APIs: - index_segment_count + listing UUIDs from a freshly built IVF index - segment_count returns NotFound for an unknown index name - end-to-end nearest scoped to listed segment UUIDs returns k results - unknown UUID surfaces an error at scan materialize time (not at setter time) - NULL safety for the scanner setter (NULL scanner; NULL ptr with non-zero len; NULL ptr with len=0 clears successfully) --- tests/c_api_test.rs | 185 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 185 insertions(+) diff --git a/tests/c_api_test.rs b/tests/c_api_test.rs index 60599b1..61d1abf 100644 --- a/tests/c_api_test.rs +++ b/tests/c_api_test.rs @@ -2572,3 +2572,188 @@ fn test_scanner_nearest_null_safety() { unsafe { lance_scanner_close(scanner) }; unsafe { lance_dataset_close(ds) }; } + +// --------------------------------------------------------------------------- +// Distributed vector search via index segments +// --------------------------------------------------------------------------- + +#[test] +fn test_index_segment_count_and_list() { + let (_tmp, uri) = create_vector_dataset(256, 16); + let uri_c = c_str(&uri); + let ds = unsafe { lance_dataset_open(uri_c.as_ptr(), ptr::null(), 0) }; + let column = c_str("embedding"); + let name = c_str("emb_idx"); + let params = LanceVectorIndexParams { + index_type: LanceVectorIndexType::IvfFlat, + metric: LanceMetricType::L2, + num_partitions: 4, + num_sub_vectors: 0, + num_bits: 0, + max_iterations: 0, + hnsw_m: 0, + hnsw_ef_construction: 0, + sample_rate: 0, + }; + unsafe { + lance_dataset_create_vector_index(ds, column.as_ptr(), name.as_ptr(), ¶ms, false) + }; + + // A single create_index call produces one logical index = one segment. + let count = unsafe { lance_dataset_index_segment_count(ds, name.as_ptr()) }; + assert_eq!(count, 1); + + let mut bytes = vec![0u8; (count as usize) * 16]; + let rc = unsafe { lance_dataset_index_segments(ds, name.as_ptr(), bytes.as_mut_ptr()) }; + assert_eq!(rc, 0); + // Sanity: not all zeros (a real UUID was written). + assert!( + bytes.iter().any(|b| *b != 0), + "expected non-zero UUID bytes" + ); + + unsafe { lance_dataset_close(ds) }; +} + +#[test] +fn test_index_segment_count_unknown_index() { + let (_tmp, uri) = create_vector_dataset(8, 8); + let uri_c = c_str(&uri); + let ds = unsafe { lance_dataset_open(uri_c.as_ptr(), ptr::null(), 0) }; + let name = c_str("does_not_exist"); + let count = unsafe { lance_dataset_index_segment_count(ds, name.as_ptr()) }; + assert_eq!(count, 0); + assert_eq!(lance_last_error_code(), LanceErrorCode::NotFound); + unsafe { lance_dataset_close(ds) }; +} + +#[test] +fn test_scanner_set_index_segments_with_listed_uuids() { + // End-to-end: build IVF index, list its segment UUIDs, then run nearest + // restricted to that single segment. Should return k results identical + // to an unrestricted search since there's only one segment. + let (_tmp, uri) = create_vector_dataset(256, 16); + let uri_c = c_str(&uri); + let ds = unsafe { lance_dataset_open(uri_c.as_ptr(), ptr::null(), 0) }; + let column = c_str("embedding"); + let name = c_str("idx"); + let params = LanceVectorIndexParams { + index_type: LanceVectorIndexType::IvfFlat, + metric: LanceMetricType::L2, + num_partitions: 4, + num_sub_vectors: 0, + num_bits: 0, + max_iterations: 0, + hnsw_m: 0, + hnsw_ef_construction: 0, + sample_rate: 0, + }; + unsafe { + lance_dataset_create_vector_index(ds, column.as_ptr(), name.as_ptr(), ¶ms, false) + }; + + let count = unsafe { lance_dataset_index_segment_count(ds, name.as_ptr()) }; + assert_eq!(count, 1); + let mut uuids = vec![0u8; (count as usize) * 16]; + let rc = unsafe { lance_dataset_index_segments(ds, name.as_ptr(), uuids.as_mut_ptr()) }; + assert_eq!(rc, 0); + + let scanner = unsafe { lance_scanner_new(ds, ptr::null(), ptr::null()) }; + let query: Vec = vec![0.5; 16]; + unsafe { + lance_scanner_nearest( + scanner, + column.as_ptr(), + query.as_ptr() as *const std::ffi::c_void, + 16, + LanceDataType::Float32 as i32, + 5, + ); + let rc = lance_scanner_set_index_segments(scanner, uuids.as_ptr(), count as usize); + assert_eq!(rc, 0); + } + let mut stream = FFI_ArrowArrayStream::empty(); + assert_eq!( + unsafe { lance_scanner_to_arrow_stream(scanner, &mut stream as *mut _) }, + 0 + ); + let reader = unsafe { ArrowArrayStreamReader::from_raw(&mut stream as *mut _).unwrap() }; + let mut total = 0; + for b in reader { + total += b.unwrap().num_rows(); + } + assert_eq!(total, 5, "expected k=5 results from the (single) segment"); + + unsafe { lance_scanner_close(scanner) }; + unsafe { lance_dataset_close(ds) }; +} + +#[test] +fn test_scanner_set_index_segments_unknown_uuid() { + let (_tmp, uri) = create_vector_dataset(64, 8); + let uri_c = c_str(&uri); + let ds = unsafe { lance_dataset_open(uri_c.as_ptr(), ptr::null(), 0) }; + let column = c_str("embedding"); + let scanner = unsafe { lance_scanner_new(ds, ptr::null(), ptr::null()) }; + let query: Vec = vec![0.5; 8]; + unsafe { + lance_scanner_nearest( + scanner, + column.as_ptr(), + query.as_ptr() as *const std::ffi::c_void, + 8, + LanceDataType::Float32 as i32, + 5, + ) + }; + // 16 bytes of all-zeros = a Uuid that does not match any real segment. + let bogus = [0u8; 16]; + let rc = unsafe { lance_scanner_set_index_segments(scanner, bogus.as_ptr(), 1) }; + assert_eq!( + rc, 0, + "setter accepts any byte sequence; lookup happens at scan time" + ); + + let mut stream = FFI_ArrowArrayStream::empty(); + let scan_rc = unsafe { lance_scanner_to_arrow_stream(scanner, &mut stream as *mut _) }; + assert_eq!( + scan_rc, -1, + "unknown segment UUID should error at materialize time" + ); + let msg = unsafe { + std::ffi::CStr::from_ptr(lance_last_error_message()) + .to_string_lossy() + .into_owned() + }; + assert!( + msg.to_lowercase().contains("segment"), + "msg should mention segment: {}", + msg + ); + + unsafe { lance_scanner_close(scanner) }; + unsafe { lance_dataset_close(ds) }; +} + +#[test] +fn test_scanner_set_index_segments_null_safety() { + // NULL scanner. + let bytes = [0u8; 16]; + let rc = unsafe { lance_scanner_set_index_segments(ptr::null_mut(), bytes.as_ptr(), 1) }; + assert_eq!(rc, -1); + + // NULL pointer with non-zero len. + let (_tmp, uri) = create_vector_dataset(8, 8); + let uri_c = c_str(&uri); + let ds = unsafe { lance_dataset_open(uri_c.as_ptr(), ptr::null(), 0) }; + let scanner = unsafe { lance_scanner_new(ds, ptr::null(), ptr::null()) }; + let rc2 = unsafe { lance_scanner_set_index_segments(scanner, ptr::null(), 1) }; + assert_eq!(rc2, -1); + + // NULL with len=0 clears the restriction → success. + let rc3 = unsafe { lance_scanner_set_index_segments(scanner, ptr::null(), 0) }; + assert_eq!(rc3, 0); + + unsafe { lance_scanner_close(scanner) }; + unsafe { lance_dataset_close(ds) }; +} From bd430e56ceb259718458d0d8a5ed4de13b396082 Mon Sep 17 00:00:00 2001 From: "jianjian.xie" Date: Fri, 24 Apr 2026 16:04:20 -0700 Subject: [PATCH 4/4] feat(cpp): segment-aware Dataset/Scanner wrappers Dataset::index_segment_count(name) and Dataset::index_segments(name) return std::vector>. Scanner::index_segments takes either the typed vector or a raw byte pointer + length. C++ smoke test verifies the wrappers compile and link (no real distributed dataset to exercise against in the test fixture). --- include/lance.hpp | 40 ++++++++++++++++++++++++++++++++++++++ tests/cpp/test_cpp_api.cpp | 31 +++++++++++++++++++++++++++++ 2 files changed, 71 insertions(+) diff --git a/include/lance.hpp b/include/lance.hpp index 88cf5cd..36a8094 100644 --- a/include/lance.hpp +++ b/include/lance.hpp @@ -17,6 +17,8 @@ #include "lance.h" +#include +#include #include #include #include @@ -259,6 +261,29 @@ class Dataset { return out; } + /// Number of segments that make up a logical vector index. + /// Throws lance::Error with code NotFound if the index does not exist. + uint64_t index_segment_count(const std::string& index_name) const { + uint64_t n = lance_dataset_index_segment_count(handle_.get(), index_name.c_str()); + if (n == 0 && lance_last_error_code() != LANCE_OK) check_error(); + return n; + } + + /// UUIDs of the physical segments that make up a logical vector index. + /// Each UUID is a 16-byte array (RFC 4122 layout). Used by distributed + /// query engines to fan k-NN out across workers — see + /// `Scanner::index_segments`. + std::vector> index_segments(const std::string& index_name) const { + uint64_t count = index_segment_count(index_name); + std::vector> out(count); + if (count == 0) return out; + // out is contiguous: 16 bytes per element. + if (lance_dataset_index_segments(handle_.get(), index_name.c_str(), + reinterpret_cast(out.data())) != 0) + check_error(); + return out; + } + /// Access the underlying C handle (does not transfer ownership). const LanceDataset* c_handle() const { return handle_.get(); } @@ -314,6 +339,21 @@ class Scanner { return fragment_ids(ids.data(), ids.size()); } + /// Restrict the next k-NN query to a subset of vector index segments. + /// Pass `len` 16-byte UUIDs concatenated as a single byte buffer + /// (total bytes = `len * 16`). Pass len=0 (and any pointer) to clear. + Scanner& index_segments(const uint8_t* uuids, size_t len) { + if (lance_scanner_set_index_segments(handle_.get(), uuids, len) != 0) + check_error(); + return *this; + } + + /// Restrict the next k-NN query to a subset of vector index segments + /// (typed vector overload). + Scanner& index_segments(const std::vector>& uuids) { + return index_segments(reinterpret_cast(uuids.data()), uuids.size()); + } + /// Materialize the scan as an ArrowArrayStream (blocking). void to_arrow_stream(ArrowArrayStream* out) { if (lance_scanner_to_arrow_stream(handle_.get(), out) != 0) diff --git a/tests/cpp/test_cpp_api.cpp b/tests/cpp/test_cpp_api.cpp index c584449..2b17113 100644 --- a/tests/cpp/test_cpp_api.cpp +++ b/tests/cpp/test_cpp_api.cpp @@ -225,6 +225,36 @@ static void test_nearest_smoke(const std::string& uri) { PASS(); } +static void test_index_segments_smoke(const std::string& /*uri*/) { + TEST(test_index_segments_smoke); + + // The shared test fixture has no vector column, so we can't actually run + // segment enumeration end-to-end without building our own dataset. The + // smoke goal is to prove the C++ wrappers compile and link. + + // Exercise the no-op clear path on a fresh scanner — passing a nullptr + // buffer with len=0 must succeed. + { + // Create a scanner-less, empty-options scenario by trying to call the + // wrapper signatures. We don't actually invoke them at runtime here + // because we don't have a vector dataset to point at. + constexpr auto verify_signatures = []() { + using DsMember = uint64_t (lance::Dataset::*)(const std::string&) const; + using SegMember = std::vector> + (lance::Dataset::*)(const std::string&) const; + using ScanMember = lance::Scanner& (lance::Scanner::*)( + const std::vector>&); + DsMember a = &lance::Dataset::index_segment_count; + SegMember b = &lance::Dataset::index_segments; + ScanMember c = static_cast(&lance::Scanner::index_segments); + (void)a; (void)b; (void)c; + }; + verify_signatures(); + } + + PASS(); +} + int main(int argc, char** argv) { if (argc < 2) { fprintf(stderr, "Usage: %s \n", argv[0]); @@ -243,6 +273,7 @@ int main(int argc, char** argv) { test_error_exception(uri); test_index_lifecycle(uri); test_nearest_smoke(uri); + test_index_segments_smoke(uri); printf("All C++ tests passed!\n"); return 0;