diff --git a/docs/docs.json b/docs/docs.json index 36c7c88..990e11f 100644 --- a/docs/docs.json +++ b/docs/docs.json @@ -85,6 +85,7 @@ "tables/schema", "tables/update", "tables/versioning", + "tables/branching", "tables/consistency" ] }, diff --git a/docs/snippets/tables.mdx b/docs/snippets/tables.mdx index 4650408..e91355e 100644 --- a/docs/snippets/tables.mdx +++ b/docs/snippets/tables.mdx @@ -30,7 +30,17 @@ export const PyAlterVectorColumn = "vector_dim = 768 # Your embedding dimension export const PyBatchDataInsertion = "import pyarrow as pa\n\ndef make_batches():\n for i in range(5): # Create 5 batches\n yield pa.RecordBatch.from_arrays(\n [\n pa.array([[3.1, 4.1], [5.9, 26.5]], pa.list_(pa.float32(), 2)),\n pa.array([f\"item{i * 2 + 1}\", f\"item{i * 2 + 2}\"]),\n pa.array([float((i * 2 + 1) * 10), float((i * 2 + 2) * 10)]),\n ],\n [\"vector\", \"item\", \"price\"],\n )\n\nschema = pa.schema(\n [\n pa.field(\"vector\", pa.list_(pa.float32(), 2)),\n pa.field(\"item\", pa.utf8()),\n pa.field(\"price\", pa.float32()),\n ]\n)\n# Create table with batches\ntable_name = \"batch_ingestion_example\"\ntable = db.create_table(table_name, make_batches(), schema=schema, mode=\"overwrite\")\n"; -export const PyBranches = "# Fork an isolated, writable branch from main's latest version.\n# The returned handle is scoped to the branch; writes on it do not\n# affect main.\nbranch = table.branches.create(\"exp\")\nbranch.add([{\"id\": 4, \"author\": \"Lancelot\", \"quote\": \"For the realm!\"}])\nprint(branch.count_rows()) # 4 rows on the branch\nprint(table.count_rows()) # 3 rows; main is untouched\n\n# List all branches, mapping name to branch metadata.\nprint(table.branches.list())\n\n# Reopen the branch later by name, or open it directly from the\n# database connection.\nchecked_out = table.branches.checkout(\"exp\")\nbranch_handle = db.open_table(\"quotes_branches_example\", branch=\"exp\")\nprint(checked_out.count_rows(), branch_handle.count_rows()) # both 4\n\n# Delete a branch when you're done with it.\ntable.branches.delete(\"exp\")\n"; +export const PyBranchCreate = "# Fork an isolated, writable branch from main's latest version.\n# `create` returns a table handle scoped to the new branch.\nbranch = table.branches.create(\"exp\")\n"; + +export const PyBranchDelete = "# Delete the branch and its branch-local history. Data on main is safe.\ntable.branches.delete(\"exp\")\n"; + +export const PyBranchIndex = "# Build and validate indexes on a branch before promoting them to main.\ndev = products.branches.create(\"index-dev\")\n\n# A vector (ANN) index and a full-text search index, both branch-scoped.\ndev.create_index(\n \"vector\",\n config=IvfPq(distance_type=\"cosine\", num_partitions=1, num_sub_vectors=2),\n)\ndev.create_index(\"text\", config=FTS())\n\n# Both indexes live only on the branch; main still has none.\nprint([ix.name for ix in dev.list_indices()]) # branch: two indexes\nprint([ix.name for ix in products.list_indices()]) # main: [] (untouched)\n"; + +export const PyBranchPromote = "# There is no built-in merge yet, so promote a branch by writing its rows\n# back to main with a normal ingestion call. `merge_insert` keys on a\n# unique column, so rows that already exist on main are updated in place and\n# new rows are appended — exactly what an upsert-style ingestion job does.\npromoted = promo.to_arrow() # or filter down to just the rows you changed\n(\n table.merge_insert(\"id\")\n .when_matched_update_all() # update rows that already exist on main\n .when_not_matched_insert_all() # insert rows that are new on the branch\n .execute(promoted)\n)\n"; + +export const PyBranchReopen = "# Reopen an existing branch by name from the table handle...\nchecked_out = table.branches.checkout(\"exp\")\n# ...or open it directly from the database connection.\nbranch_handle = db.open_table(\"quotes_branches_example\", branch=\"exp\")\nprint(checked_out.count_rows(), branch_handle.count_rows()) # both 4\n"; + +export const PyBranchWrite = "# Writes land on the branch handle only; main is left untouched.\nbranch.add([{\"id\": 4, \"author\": \"Lancelot\", \"quote\": \"For the realm!\"}])\nprint(branch.count_rows()) # 4 rows on the branch\nprint(table.count_rows()) # 3 rows; main is unaffected\n\n# List every branch, each mapped to its metadata (including its fork point).\nprint(table.branches.list())\n"; export const PyConsistencyCheckoutLatest = "uri = str(tmp_db.uri)\nwriter_db = lancedb.connect(uri)\nreader_db = lancedb.connect(uri)\nwriter_table = writer_db.create_table(\n \"consistency_checkout_latest_table\", [{\"id\": 1}], mode=\"overwrite\"\n)\nreader_table = reader_db.open_table(\"consistency_checkout_latest_table\")\n\nwriter_table.add([{\"id\": 2}])\nrows_before_refresh = reader_table.count_rows()\nprint(f\"Rows before checkout_latest: {rows_before_refresh}\")\n\nreader_table.checkout_latest()\nrows_after_refresh = reader_table.count_rows()\nprint(f\"Rows after checkout_latest: {rows_after_refresh}\")\n"; @@ -150,7 +160,17 @@ export const TsAlterColumnsWithExpression = "// For custom transforms, create a export const TsAlterVectorColumn = "const oldDim = 384;\nconst newDim = 1024;\nconst vectorSchema = new arrow.Schema([\n new arrow.Field(\"id\", new arrow.Int64()),\n new arrow.Field(\n \"embedding\",\n new arrow.FixedSizeList(\n oldDim,\n new arrow.Field(\"item\", new arrow.Float16(), true),\n ),\n true,\n ),\n]);\nconst vectorData = lancedb.makeArrowTable(\n [{ id: 1, embedding: Array.from({ length: oldDim }, () => Math.random()) }],\n { schema: vectorSchema },\n);\nconst vectorTable = await db.createTable(\"vector_alter_example\", vectorData, {\n mode: \"overwrite\",\n});\n\n// Changing FixedSizeList dimensions (384 -> 1024) is not supported via alterColumns.\n// Use addColumns + dropColumns + alterColumns(rename) to replace the column.\nawait vectorTable.addColumns([\n {\n name: \"embedding_v2\",\n valueSql: `arrow_cast(NULL, 'FixedSizeList(${newDim}, Float16)')`,\n },\n]);\nawait vectorTable.dropColumns([\"embedding\"]);\nawait vectorTable.alterColumns([{ path: \"embedding_v2\", rename: \"embedding\" }]);\n"; -export const TsBranches = "const branches = await table.branches();\n\n// Fork an isolated, writable branch from main's latest version.\n// The returned handle is scoped to the branch; writes on it do not\n// affect main.\nconst branch = await branches.create(\"exp\");\nawait branch.add([{ id: 4, author: \"Lancelot\", quote: \"For the realm!\" }]);\nconsole.log(await branch.countRows()); // 4 rows on the branch\nconsole.log(await table.countRows()); // 3 rows; main is untouched\n\n// List all branches, mapping name to branch metadata.\nconsole.log(await branches.list());\n\n// Reopen the branch later by name, or open it directly from the\n// database connection.\nconst checkedOut = await branches.checkout(\"exp\");\nconst branchHandle = await db.openTable(\n \"quotes_branches_example\",\n undefined,\n { branch: \"exp\" },\n);\nconsole.log(await checkedOut.countRows()); // 4\nconsole.log(await branchHandle.countRows()); // 4\n\n// Delete a branch when you're done with it.\nawait branches.delete(\"exp\");\n"; +export const TsBranchCreate = "// Fork an isolated, writable branch from main's latest version.\n// `create` returns a table handle scoped to the new branch.\nconst branch = await branches.create(\"exp\");\n"; + +export const TsBranchDelete = "// Delete the branch and its branch-local history. Data on main is safe.\nawait branches.delete(\"exp\");\n"; + +export const TsBranchIndex = "// Build and validate indexes on a branch before promoting them to main.\nconst dev = await productBranches.create(\"index-dev\");\n\n// A vector (ANN) index and a full-text search index, both branch-scoped.\nawait dev.createIndex(\"vector\", {\n config: lancedb.Index.ivfPq({\n distanceType: \"cosine\",\n numPartitions: 1,\n numSubVectors: 2,\n }),\n});\nawait dev.createIndex(\"text\", { config: lancedb.Index.fts() });\n\n// Both indexes live only on the branch; main still has none.\nconsole.log((await dev.listIndices()).map((ix) => ix.name)); // branch: two indexes\nconsole.log((await products.listIndices()).map((ix) => ix.name)); // main: [] (untouched)\n"; + +export const TsBranchPromote = "// There is no built-in merge yet, so promote a branch by writing its rows\n// back to main with a normal ingestion call. `mergeInsert` keys on a\n// unique column, so rows that already exist on main are updated in place and\n// new rows are appended — exactly what an upsert-style ingestion job does.\nconst promoted = await promo.toArrow(); // or filter down to just the changed rows\nawait table\n .mergeInsert(\"id\")\n .whenMatchedUpdateAll() // update rows that already exist on main\n .whenNotMatchedInsertAll() // insert rows that are new on the branch\n .execute(promoted);\n"; + +export const TsBranchReopen = "// Reopen an existing branch by name from the table handle...\nconst checkedOut = await branches.checkout(\"exp\");\n// ...or open it directly from the database connection.\nconst branchHandle = await db.openTable(\n \"quotes_branches_example\",\n undefined,\n { branch: \"exp\" },\n);\nconsole.log(await checkedOut.countRows(), await branchHandle.countRows()); // both 4\n"; + +export const TsBranchWrite = "// Writes land on the branch handle only; main is left untouched.\nawait branch.add([{ id: 4, author: \"Lancelot\", quote: \"For the realm!\" }]);\nconsole.log(await branch.countRows()); // 4 rows on the branch\nconsole.log(await table.countRows()); // 3 rows; main is unaffected\n\n// List every branch, each mapped to its metadata (including its fork point).\nconsole.log(await branches.list());\n"; export const TsConsistencyCheckoutLatest = "const checkoutWriterDb = await lancedb.connect(databaseDir);\nconst checkoutReaderDb = await lancedb.connect(databaseDir);\nconst checkoutWriterTable = await checkoutWriterDb.createTable(\n \"consistency_checkout_latest_table\",\n [{ id: 1 }],\n { mode: \"overwrite\" },\n);\nconst checkoutReaderTable = await checkoutReaderDb.openTable(\n \"consistency_checkout_latest_table\",\n);\nawait checkoutWriterTable.add([{ id: 2 }]);\nconst rowsBeforeRefresh = await checkoutReaderTable.countRows();\nconsole.log(`Rows before checkoutLatest: ${rowsBeforeRefresh}`);\nawait checkoutReaderTable.checkoutLatest();\nconst rowsAfterRefresh = await checkoutReaderTable.countRows();\nconsole.log(`Rows after checkoutLatest: ${rowsAfterRefresh}`);\n"; @@ -252,7 +272,17 @@ export const RsAlterColumnsWithExpression = "// For custom transforms, create a export const RsAlterVectorColumn = "let old_dim = 384;\nlet new_dim = 1024;\nlet vector_schema = Arc::new(Schema::new(vec![\n Field::new(\"id\", DataType::Int64, false),\n Field::new(\n \"embedding\",\n DataType::FixedSizeList(\n Arc::new(Field::new(\"item\", DataType::Float32, true)),\n old_dim,\n ),\n true,\n ),\n]));\nlet vector_batch = RecordBatch::try_new(\n vector_schema.clone(),\n vec![\n Arc::new(Int64Array::from(vec![1])),\n Arc::new(\n FixedSizeListArray::from_iter_primitive::(\n vec![Some(vec![Some(0.1_f32); old_dim as usize])],\n old_dim,\n ),\n ),\n ],\n)\n.unwrap();\nlet vector_reader: Box =\n Box::new(RecordBatchIterator::new(vec![Ok(vector_batch)].into_iter(), vector_schema.clone()));\nlet vector_table = db\n .create_table(\"vector_alter_example\", vector_reader)\n .mode(CreateTableMode::Overwrite)\n .execute()\n .await\n .unwrap();\n\n// Changing FixedSizeList dimensions (384 -> 1024) is not supported via alter_columns.\n// Use add_columns + drop_columns + alter_columns(rename) to replace the column.\nvector_table\n .add_columns(\n NewColumnTransform::SqlExpressions(vec![(\n \"embedding_v2\".to_string(),\n format!(\"arrow_cast(NULL, 'FixedSizeList({}, Float32)')\", new_dim),\n )]),\n None,\n )\n .await\n .unwrap();\nvector_table.drop_columns(&[\"embedding\"]).await.unwrap();\nvector_table\n .alter_columns(&[ColumnAlteration::new(\"embedding_v2\".to_string())\n .rename(\"embedding\".to_string())])\n .await\n .unwrap();\n"; -export const RsBranches = "use lancedb::table::Ref;\n\n// Fork an isolated, writable branch from main's latest version.\n// The returned handle is scoped to the branch; writes on it do not\n// affect main.\nlet branch = branches_table\n .create_branch(\"exp\", Ref::Version(None, None))\n .await\n .unwrap();\nbranch\n .add(make_quotes_reader(vec![(4, \"Lancelot\", \"For the realm!\")]))\n .execute()\n .await\n .unwrap();\nlet branch_rows = branch.count_rows(None).await.unwrap();\nlet main_rows = branches_table.count_rows(None).await.unwrap();\nprintln!(\"Branch rows: {}\", branch_rows); // 4\nprintln!(\"Main rows: {}\", main_rows); // 3; main is untouched\n\n// List all branches, mapping name to branch metadata.\nlet all_branches = branches_table.list_branches().await.unwrap();\nprintln!(\"Branches: {:?}\", all_branches);\n\n// Reopen the branch later by name, or open it directly via the builder.\nlet checked_out = branches_table.checkout_branch(\"exp\").await.unwrap();\nlet opened = db\n .open_table(\"quotes_branches_example\")\n .branch(\"exp\")\n .execute()\n .await\n .unwrap();\nlet checked_out_rows = checked_out.count_rows(None).await.unwrap();\nlet opened_rows = opened.count_rows(None).await.unwrap();\nprintln!(\"Reopened rows: {}, {}\", checked_out_rows, opened_rows); // both 4\n\n// Delete a branch when you're done with it.\nbranches_table.delete_branch(\"exp\").await.unwrap();\n"; +export const RsBranchCreate = "// Fork an isolated, writable branch from main's latest version.\n// `create_branch` returns a table handle scoped to the new branch.\nlet branch = branches_table\n .create_branch(\"exp\", Ref::Version(None, None))\n .await\n .unwrap();\n"; + +export const RsBranchDelete = "// Delete the branch and its branch-local history. Data on main is safe.\nbranches_table.delete_branch(\"exp\").await.unwrap();\n"; + +export const RsBranchIndex = "use lancedb::index::scalar::FtsIndexBuilder;\nuse lancedb::index::Index;\n\n// Build and validate indexes on a branch before promoting them to main.\nlet dev = products\n .create_branch(\"index-dev\", Ref::Version(None, None))\n .await\n .unwrap();\n\n// A vector (ANN) index and a full-text search index, both branch-scoped.\ndev.create_index(&[\"vector\"], Index::Auto)\n .execute()\n .await\n .unwrap();\ndev.create_index(&[\"text\"], Index::FTS(FtsIndexBuilder::default()))\n .execute()\n .await\n .unwrap();\n\n// Both indexes live only on the branch; main still has none.\nprintln!(\"Branch indexes: {}\", dev.list_indices().await.unwrap().len()); // 2\nprintln!(\"Main indexes: {}\", products.list_indices().await.unwrap().len()); // 0\n"; + +export const RsBranchPromote = "// There is no built-in merge yet, so promote a branch by writing its rows\n// back to main with a normal ingestion call. `merge_insert` keys on a\n// unique column, so rows that already exist on main are updated in place and\n// new rows are appended — exactly what an upsert-style ingestion job does.\nlet schema = promo.schema().await.unwrap();\nlet batches = promo\n .query()\n .execute()\n .await\n .unwrap()\n .try_collect::>()\n .await\n .unwrap();\nlet promoted = RecordBatchIterator::new(batches.into_iter().map(Ok), schema);\n\nlet mut merge = branches_table.merge_insert(&[\"id\"]);\nmerge\n .when_matched_update_all(None) // update rows that already exist on main\n .when_not_matched_insert_all(); // insert rows that are new on the branch\nmerge.execute(Box::new(promoted)).await.unwrap();\n"; + +export const RsBranchReopen = "// Reopen an existing branch by name from the table handle...\nlet checked_out = branches_table.checkout_branch(\"exp\", None).await.unwrap();\n// ...or open it directly via the connection's builder.\nlet opened = db\n .open_table(\"quotes_branches_example\")\n .branch(\"exp\")\n .execute()\n .await\n .unwrap();\nprintln!(\n \"Reopened rows: {}, {}\",\n checked_out.count_rows(None).await.unwrap(),\n opened.count_rows(None).await.unwrap()\n); // both 4\n"; + +export const RsBranchWrite = "// Writes land on the branch handle only; main is left untouched.\nbranch\n .add(make_quotes_reader(vec![(4, \"Lancelot\", \"For the realm!\")]))\n .execute()\n .await\n .unwrap();\nprintln!(\"Branch rows: {}\", branch.count_rows(None).await.unwrap()); // 4\nprintln!(\"Main rows: {}\", branches_table.count_rows(None).await.unwrap()); // 3\n\n// List every branch, each mapped to its metadata (including its fork point).\nprintln!(\"Branches: {:?}\", branches_table.list_branches().await.unwrap());\n"; export const RsConsistencyCheckoutLatest = "let checkout_writer_db = connect(&db_uri).execute().await.unwrap();\nlet checkout_reader_db = connect(&db_uri).execute().await.unwrap();\nlet checkout_writer_table = checkout_writer_db\n .create_table(\n \"consistency_checkout_latest_table\",\n make_users_reader(vec![1], vec![\"Alice\"], None),\n )\n .mode(CreateTableMode::Overwrite)\n .execute()\n .await\n .unwrap();\nlet checkout_reader_table = checkout_reader_db\n .open_table(\"consistency_checkout_latest_table\")\n .execute()\n .await\n .unwrap();\ncheckout_writer_table\n .add(make_users_reader(vec![2], vec![\"Bob\"], None))\n .execute()\n .await\n .unwrap();\nlet rows_before_refresh = checkout_reader_table.count_rows(None).await.unwrap();\nprintln!(\"Rows before checkout_latest: {}\", rows_before_refresh);\ncheckout_reader_table.checkout_latest().await.unwrap();\nlet rows_after_refresh = checkout_reader_table.count_rows(None).await.unwrap();\nprintln!(\"Rows after checkout_latest: {}\", rows_after_refresh);\n"; diff --git a/docs/tables/branching.mdx b/docs/tables/branching.mdx new file mode 100644 index 0000000..7b5157f --- /dev/null +++ b/docs/tables/branching.mdx @@ -0,0 +1,282 @@ +--- +title: "Branches" +sidebarTitle: "Branches" +description: "Fork isolated, writable lines of table history in LanceDB. Run experiments, backfills, and index rebuilds without disturbing production reads on main." +icon: "code-branch" +--- +import { + PyConnect, + TsConnect, + RsConnect, + PyConnectEnterpriseQuickstart, + TsConnectEnterpriseQuickstart, + RsConnectEnterpriseQuickstart, +} from '/snippets/connection.mdx'; +import { + PyBranchCreate as BranchCreate, + TsBranchCreate, + RsBranchCreate, + PyBranchWrite as BranchWrite, + TsBranchWrite, + RsBranchWrite, + PyBranchReopen as BranchReopen, + TsBranchReopen, + RsBranchReopen, + PyBranchDelete as BranchDelete, + TsBranchDelete, + RsBranchDelete, + PyBranchPromote as BranchPromote, + TsBranchPromote, + RsBranchPromote, + PyBranchIndex as BranchIndex, + TsBranchIndex, + RsBranchIndex, +} from '/snippets/tables.mdx'; + +A branch is an isolated, writable line of history forked from `main` (or from any +other branch). Anything you do on a branch (e.g., adding rows, changing the schema, +building an index) stays on that branch, so `main` keeps serving production +reads exactly as before. Branches are a natural fit when you want to: + +- Experiment with a new index, schema change, or reprocessing step without + affecting live queries on `main`. +- Run a backfill or migration you'd like to review before promoting it. +- Hand a collaborator a frozen point-in-time fork while you keep writing to `main`. + +## How branches relate to versions and tags + +Every LanceDB table already tracks a linear history of [versions](/tables/versioning), +and you can [tag](/tables/versioning#tag-based-versioning) a version or `checkout` +one to read it. Branches add the missing piece: a *separate, writable* line of +history. Where a tag is a read-only label and `checkout` is a read-only view, a +branch forks from a point in history and then evolves on its own. Creating or +checking out a branch hands you a new table handle whose reads and writes are +scoped to that branch. The [comparison table](#branches-vs-tags-vs-checkout) at +the end of this page lays out when to reach for each. + + +Branches are supported on local and namespace-backed tables in LanceDB OSS, as +well as on LanceDB Enterprise (remote) tables. + + +## Connect to a table + +The branch API is identical no matter how you connect — only the connection +itself differs between OSS and Enterprise. Establish a connection (`db`) and open +a `table` (see [Create a table](/tables/create)), then use the same branch calls +in every example that follows. + +### LanceDB OSS + +Point LanceDB at a local directory (or an object-storage URI) to use it as an +embedded library. + + + + {PyConnect} + + + + {TsConnect} + + + + { "use lancedb::connect;\n\n" } + {RsConnect} + + + +### LanceDB Enterprise + +Enterprise + +Branching on LanceDB Enterprise works the same way as on OSS, but you connect to your +Enterprise deployment with a `db://` URI, an API key, and your +region. Once you have a connection, open a table and use the same branch calls in +every example that follows. + + + + {PyConnectEnterpriseQuickstart} + + + + {TsConnectEnterpriseQuickstart} + + + + {RsConnectEnterpriseQuickstart} + + + +## Work with branches + +The lifecycle of a branch is short and predictable: fork it, write to it, reopen +it whenever you need it, and delete it once you're done. The examples below use a +small `quotes` table with three rows on `main`. + +### Create a branch + +Forking from `main` returns a table handle scoped to the new branch. `main` is +the reserved default source, so `create` needs only a name; to fork from +somewhere else, pass a branch name, a specific version, or both. + + + + {BranchCreate} + + + + {TsBranchCreate} + + + + {RsBranchCreate} + + + +### Write to a branch + +Writes go through the branch handle and stay there — the `main` handle keeps +reporting its original row count. Listing branches returns a mapping of each +branch name to its metadata, including the version it was forked from. + + + + {BranchWrite} + + + + {TsBranchWrite} + + + + {RsBranchWrite} + + + +### Reopen a branch + +A branch outlives the handle that created it. Reopen it later by name — either +from an existing table handle or straight from the connection when you open the +table. Both routes give you a writable handle tracking the branch's latest state. + + + + {BranchReopen} + + + + {TsBranchReopen} + + + + {RsBranchReopen} + + + +### Delete a branch + +Deleting a branch removes it and its branch-local history; `main` is untouched. +Delete a branch only once you've promoted anything worth keeping — see +[Merging a branch into `main`](#merging-a-branch-into-main) below. + + + + {BranchDelete} + + + + {TsBranchDelete} + + + + {RsBranchDelete} + + + +## Merge a branch into `main` + + +LanceDB doesn't yet support an automatic merge of a branch into `main`, with full conflict-handling. +There's no single `merge()` call that reconciles the two histories for you. + + +Until full merge support is available, you can promote a branch the same way any other data lands in a table: by +running an ingestion workload against `main`. The only real question is how each +branch row lines up with the row it should replace on `main`. This is exactly +what [`merge_insert`](/tables/update#merge-incoming-rows-by-key) handles. Keyed on +a unique column, it updates rows that already exist and inserts the ones that +don't (an *upsert*), leaving everything else on `main` untouched. + +Read the rows you want out of the branch, then upsert them into `main` on your +key column — `id` in this example: + + + + {BranchPromote} + + + + {TsBranchPromote} + + + + {RsBranchPromote} + + + +Since this is just a keyed write against `main`, it looks like a normal +ingestion job: point the job's source at the branch and its destination at +`main`. Reading the whole branch and upserting it is the simplest approach and is +safe to re-run, because `merge_insert` is idempotent on the key. To promote only +what changed, filter the branch read down to the rows you touched before the +upsert. + +## Build indexes on a branch + +One of the most useful things a branch buys you is a safe place to build and +validate an index without affecting what's in production on the `main` branch. +Fork a branch, create your vector (ANN) and full-text search (FTS) +indexes on it, check recall and latency, and only then promote the change to +`main`. Because the indexes live on the branch, queries against `main` never see +a half-built index and are never slowed down by the build. + +This pattern is especially valuable on Enterprise +deployments, where `main` is typically serving production traffic while you tune +an index configuration on the side. + + + + {BranchIndex} + + + + {TsBranchIndex} + + + + {RsBranchIndex} + + + +Schema changes such as adding, altering, or dropping columns are branch-scoped in +the same way, so you can stage a larger reshaping of a table and review it before +it ever reaches `main`. + +## Branches vs. tags vs. versions + +Now that you're familiar with branches, you can see how they complement the other ways LanceDB +give you to work with table history. Choose these approaches based on whether you need to +*label*, *read*, or *write* data at a point in history: + +| Feature | Writable? | Purpose | +| --- | --- | --- | +| **[Branch](#work-with-branches)** | ✅ Yes | Write on top of a point in history without touching `main`. | +| **[Tag](/tables/versioning#tag-based-versioning)** | ❌ No | Attach a human-readable label to an existing version; protects it from cleanup. | +| **[`checkout(version)`](/tables/versioning#rollback-to-previous-versions)** | ❌ No | Read a historical version of `main` without forking. Read-only until you `restore`. | + + +For linear version history — creating versions, listing them, rolling back, and +tagging — see the [Versioning and Reproducibility](/tables/versioning) guide. + diff --git a/docs/tables/versioning.mdx b/docs/tables/versioning.mdx index 8879036..e2b961e 100644 --- a/docs/tables/versioning.mdx +++ b/docs/tables/versioning.mdx @@ -35,9 +35,6 @@ import { PyVersioningTags as VersioningTags, TsVersioningTags as TsVersioningTags, RsVersioningTags as RsVersioningTags, - PyBranches as Branches, - TsBranches as TsBranches, - RsBranches as RsBranches, RsVersioningMakeQuotesReader as RsVersioningMakeQuotesReader, } from '/snippets/tables.mdx'; @@ -223,66 +220,13 @@ deletion, the underlying table version becomes eligible for cleanup again. ## Branches -Branches are isolated, writable lines of history forked from another branch (or -a specific version). Writes to a branch do **not** affect `main`, which makes -branches ideal for: +Beyond linear history, LanceDB also supports **branches** — isolated, writable +lines of history forked from `main` (or a specific version). Whereas tags and +`checkout` give you read-only views of existing versions, a branch has its own +writable history, making it ideal for experiments, backfills, and migrations +that you want to keep separate from production reads on `main`. -- Running experiments (new indexes, schema changes, reprocessing) without - disturbing production reads on `main`. -- Backfills or migrations that you want to review before promoting. -- Sharing a frozen point-in-time fork with a collaborator while continuing to - write to `main`. - -Unlike tags, which are read-only labels on existing versions, a branch has its -own writable history. Creating or checking out a branch returns a new table -handle whose reads and writes are scoped to that branch. - - -Branches are currently supported on local and namespace-backed tables in -LanceDB OSS. Branch support for LanceDB Enterprise (remote) tables is not yet -available. - - -### Create, Write, and Reopen a Branch - -Create a branch from `main` (the default source) and write to it. The base table -continues to point at `main` and is unaffected by writes on the branch. You can -also reopen an existing branch either from a table handle or directly from the -database connection when opening the table. - - - - {Branches} - - - - {TsBranches} - - - - {RsBranches} - - - -`list` returns a mapping of branch names to branch metadata, including which -branch each was forked from. `main` is the reserved default branch: wherever a -branch is optional, leaving it out means `main`. To fork from somewhere else, -pass a branch name, a specific version, or both when creating the branch. - -Deleting a branch removes that branch and its branch-local history; data on -`main` is not affected. There is no automatic merge operation. To promote -changes, copy the desired data back explicitly. - - -**Branches vs. tags vs. checkout** - -- Use a **tag** to bookmark an existing version with a human-readable name. Tags - are read-only and protect that version from cleanup. -- Use **`checkout(version)`** to read from a historical version of `main` - without forking. The handle is read-only until you `restore`. -- Use a **branch** when you need to *write* on top of a point in history - without touching `main`. - +Branches are covered in their own guide: see [Branches](/tables/branching). ## Delete Data From the Table diff --git a/pyproject.toml b/pyproject.toml index 9bb160b..f70d712 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -5,7 +5,7 @@ description = "Add your description here" readme = "README.md" requires-python = ">=3.12,<3.14" dependencies = [ - "lancedb>=0.33.0", + "lancedb>=0.34.0", "pyarrow>=23.0.1", "lance-namespace>=0.6.1", "pandas>=3.0.1", diff --git a/tests/py/test_tables.py b/tests/py/test_tables.py index 38ef8c6..7d285c3 100644 --- a/tests/py/test_tables.py +++ b/tests/py/test_tables.py @@ -1367,7 +1367,9 @@ def test_versioning_tags(tmp_db): def test_branches(tmp_db): + import numpy as np import pyarrow as pa + from lancedb.index import FTS, IvfPq db = tmp_db schema = pa.schema( @@ -1388,30 +1390,94 @@ def test_branches(tmp_db): mode="overwrite", ) - # --8<-- [start:branches] + # --8<-- [start:branch_create] # Fork an isolated, writable branch from main's latest version. - # The returned handle is scoped to the branch; writes on it do not - # affect main. + # `create` returns a table handle scoped to the new branch. branch = table.branches.create("exp") + # --8<-- [end:branch_create] + + # --8<-- [start:branch_write] + # Writes land on the branch handle only; main is left untouched. branch.add([{"id": 4, "author": "Lancelot", "quote": "For the realm!"}]) print(branch.count_rows()) # 4 rows on the branch - print(table.count_rows()) # 3 rows; main is untouched + print(table.count_rows()) # 3 rows; main is unaffected - # List all branches, mapping name to branch metadata. + # List every branch, each mapped to its metadata (including its fork point). print(table.branches.list()) + # --8<-- [end:branch_write] - # Reopen the branch later by name, or open it directly from the - # database connection. + # --8<-- [start:branch_reopen] + # Reopen an existing branch by name from the table handle... checked_out = table.branches.checkout("exp") + # ...or open it directly from the database connection. branch_handle = db.open_table("quotes_branches_example", branch="exp") print(checked_out.count_rows(), branch_handle.count_rows()) # both 4 + # --8<-- [end:branch_reopen] - # Delete a branch when you're done with it. + # --8<-- [start:branch_delete] + # Delete the branch and its branch-local history. Data on main is safe. table.branches.delete("exp") - # --8<-- [end:branches] + # --8<-- [end:branch_delete] + assert table.count_rows() == 3 assert "exp" not in table.branches.list() + # Setup: a branch that diverges from main, ready to promote back. + promo = table.branches.create("promote") + promo.update(where="id = 1", values={"quote": "Revised on the branch"}) + promo.add([{"id": 4, "author": "Galahad", "quote": "The grail awaits."}]) + + # --8<-- [start:branch_promote] + # There is no built-in merge yet, so promote a branch by writing its rows + # back to main with a normal ingestion call. `merge_insert` keys on a + # unique column, so rows that already exist on main are updated in place and + # new rows are appended — exactly what an upsert-style ingestion job does. + promoted = promo.to_arrow() # or filter down to just the rows you changed + ( + table.merge_insert("id") + .when_matched_update_all() # update rows that already exist on main + .when_not_matched_insert_all() # insert rows that are new on the branch + .execute(promoted) + ) + # --8<-- [end:branch_promote] + + promoted_main = {row["id"]: row["quote"] for row in table.to_arrow().to_pylist()} + assert table.count_rows() == 4 + assert promoted_main[1] == "Revised on the branch" + assert 4 in promoted_main + table.branches.delete("promote") + + # Setup: a larger table with a vector and a text column to index. + rng = np.random.default_rng(0) + products = db.create_table( + "products_branch_index", + [ + {"id": i, "vector": rng.random(4).tolist(), "text": f"product number {i}"} + for i in range(512) + ], + mode="overwrite", + ) + + # --8<-- [start:branch_index] + # Build and validate indexes on a branch before promoting them to main. + dev = products.branches.create("index-dev") + + # A vector (ANN) index and a full-text search index, both branch-scoped. + dev.create_index( + "vector", + config=IvfPq(distance_type="cosine", num_partitions=1, num_sub_vectors=2), + ) + dev.create_index("text", config=FTS()) + + # Both indexes live only on the branch; main still has none. + print([ix.name for ix in dev.list_indices()]) # branch: two indexes + print([ix.name for ix in products.list_indices()]) # main: [] (untouched) + # --8<-- [end:branch_index] + + assert len(dev.list_indices()) == 2 + assert len(products.list_indices()) == 0 + products.branches.delete("index-dev") + # ============================================================================ # Consistency Examples diff --git a/tests/rs/Cargo.lock b/tests/rs/Cargo.lock index 7a24be5..f646196 100644 --- a/tests/rs/Cargo.lock +++ b/tests/rs/Cargo.lock @@ -2,6 +2,15 @@ # It is not intended for manual editing. version = 4 +[[package]] +name = "addr2line" +version = "0.25.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1b5d307320b3181d6d7954e663bd7c774a838b8220fe0593c86d9fb09f498b4b" +dependencies = [ + "gimli", +] + [[package]] name = "adler2" version = "2.0.1" @@ -32,18 +41,21 @@ dependencies = [ ] [[package]] -name = "alloc-no-stdlib" -version = "2.0.4" +name = "aligned-vec" +version = "0.6.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cc7bb162ec39d46ab1ca8c77bf72e890535becd1751bb45f64c597edb4c8c6b3" +checksum = "dc890384c8602f339876ded803c97ad529f3842aba97f6392b3dba0dd171769b" +dependencies = [ + "equator", +] [[package]] -name = "alloc-stdlib" -version = "0.2.2" +name = "alloca" +version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "94fb8275041c72129eb51b7d0322c29b8387a0386127718b096429201a5d6ece" +checksum = "e5a7d05ea6aea7e9e64d25b9156ba2fee3fdd659e34e41063cd2fc7cd020d7f4" dependencies = [ - "alloc-no-stdlib", + "cc", ] [[package]] @@ -61,6 +73,18 @@ dependencies = [ "libc", ] +[[package]] +name = "anes" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4b46cbb362ab8752921c97e041f5e366ee6297bd428a31275b9fcf1e380f7299" + +[[package]] +name = "anstyle" +version = "1.0.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "940b3a0ca603d1eade50a4846a2afffd5ef57a9feac2c0e2ec2e14f9ead76000" + [[package]] name = "anyhow" version = "1.0.102" @@ -73,7 +97,7 @@ version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f0c269894b6fe5e9d7ada0cf69b5bf847ff35bc25fc271f08e1d080fce80339a" dependencies = [ - "object", + "object 0.32.2", ] [[package]] @@ -428,6 +452,21 @@ version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c08606f8c3cbf4ce6ec8e28fb0014a2c086708fe954eaa885384a6165172e7e8" +[[package]] +name = "backtrace" +version = "0.3.76" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bb531853791a215d7c62a30daf0dde835f381ab5de4589cfe7c649d2cbe92bd6" +dependencies = [ + "addr2line", + "cfg-if", + "libc", + "miniz_oxide", + "object 0.37.3", + "rustc-demangle", + "windows-link 0.2.1", +] + [[package]] name = "base64" version = "0.21.7" @@ -517,27 +556,6 @@ dependencies = [ "generic-array", ] -[[package]] -name = "brotli" -version = "8.0.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4bd8b9603c7aa97359dbd97ecf258968c95f3adddd6db2f7e7a5bef101c84560" -dependencies = [ - "alloc-no-stdlib", - "alloc-stdlib", - "brotli-decompressor", -] - -[[package]] -name = "brotli-decompressor" -version = "5.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "874bb8112abecc98cbd6d81ea4fa7e94fb9449648c93cc89aa40c81c24d7de03" -dependencies = [ - "alloc-no-stdlib", - "alloc-stdlib", -] - [[package]] name = "bumpalo" version = "3.19.0" @@ -567,6 +585,12 @@ dependencies = [ "syn 2.0.117", ] +[[package]] +name = "bytecount" +version = "0.6.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "175812e0be2bccb6abe50bb8d566126198344f707e304f45c648fd8f2cc0365e" + [[package]] name = "bytemuck" version = "1.24.0" @@ -599,6 +623,12 @@ version = "1.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1e748733b7cbc798e1434b6ac524f0c1ff2ab456fe201501e6497c8417a4fc33" +[[package]] +name = "cast" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "37b2a672a2cb129a2e41c10b1224bb368f9f37a2b16b612598138befd7b37eb5" + [[package]] name = "cc" version = "1.2.49" @@ -613,9 +643,9 @@ dependencies = [ [[package]] name = "cedarwood" -version = "0.4.6" +version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6d910bedd62c24733263d0bed247460853c9d22e8956bd4cd964302095e04e90" +checksum = "c0524a528a6a0288df1863c3c20fe92c301875b4941e7b6c4b394ab08c5a4c55" dependencies = [ "smallvec", ] @@ -678,6 +708,58 @@ dependencies = [ "phf_codegen 0.11.3", ] +[[package]] +name = "ciborium" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42e69ffd6f0917f5c029256a24d0161db17cea3997d185db0d35926308770f0e" +dependencies = [ + "ciborium-io", + "ciborium-ll", + "serde", +] + +[[package]] +name = "ciborium-io" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05afea1e0a06c9be33d539b876f1ce3692f4afea2cb41f740e7743225ed1c757" + +[[package]] +name = "ciborium-ll" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "57663b653d948a338bfb3eeba9bb2fd5fcfaecb9e199e87e1eda4d9e8b240fd9" +dependencies = [ + "ciborium-io", + "half", +] + +[[package]] +name = "clap" +version = "4.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ddb117e43bbf7dacf0a4190fef4d345b9bad68dfc649cb349e7d17d28428e51" +dependencies = [ + "clap_builder", +] + +[[package]] +name = "clap_builder" +version = "4.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "714a53001bf66416adb0e2ef5ac857140e7dc3a0c48fb28b2f10762fc4b5069f" +dependencies = [ + "anstyle", + "clap_lex", +] + +[[package]] +name = "clap_lex" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8d4a3bb8b1e0c1050499d1815f5ab16d04f0959b233085fb31653fbfc9d98f9" + [[package]] name = "comfy-table" version = "7.1.2" @@ -759,6 +841,15 @@ version = "0.8.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" +[[package]] +name = "cpp_demangle" +version = "0.4.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2bb79cb74d735044c972aae58ed0aaa9a837e85b01106a54c39e42e97f62253" +dependencies = [ + "cfg-if", +] + [[package]] name = "cpufeatures" version = "0.2.17" @@ -777,6 +868,42 @@ dependencies = [ "cfg-if", ] +[[package]] +name = "criterion" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "950046b2aa2492f9a536f5f4f9a3de7b9e2476e575e05bd6c333371add4d98f3" +dependencies = [ + "alloca", + "anes", + "cast", + "ciborium", + "clap", + "criterion-plot", + "itertools 0.13.0", + "num-traits", + "oorandom", + "page_size", + "plotters", + "rayon", + "regex", + "serde", + "serde_json", + "tinytemplate", + "tokio", + "walkdir", +] + +[[package]] +name = "criterion-plot" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d8d80a2f4f5b554395e47b5d8305bc3d27813bacb73493eb1001e8f76dae29ea" +dependencies = [ + "cast", + "itertools 0.13.0", +] + [[package]] name = "crossbeam-channel" version = "0.5.15" @@ -1605,6 +1732,15 @@ dependencies = [ "sqlparser 0.61.0", ] +[[package]] +name = "debugid" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bef552e6f588e446098f6ba40d89ac146c8c7b64aade83c051ee00bb5d2bc18d" +dependencies = [ + "uuid", +] + [[package]] name = "deepsize" version = "0.2.0" @@ -1751,6 +1887,26 @@ dependencies = [ "syn 2.0.117", ] +[[package]] +name = "equator" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4711b213838dfee0117e3be6ac926007d7f433d7bbe33595975d4190cb07e6fc" +dependencies = [ + "equator-macro", +] + +[[package]] +name = "equator-macro" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "44f23cf4b44bfce11a86ace86f8a73ffdec849c9fd00a386a53d278bd9e81fb3" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.117", +] + [[package]] name = "equivalent" version = "1.0.2" @@ -1824,6 +1980,18 @@ version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3a3076410a55c90011c298b04d0cfa770b00fa04e1e3c97d3f6c9de105a03844" +[[package]] +name = "findshlibs" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "40b9e59cd0f7e0806cca4be089683ecb6434e602038df21fe6bf6711b2f07f64" +dependencies = [ + "cc", + "lazy_static", + "libc", + "winapi", +] + [[package]] name = "fixedbitset" version = "0.5.7" @@ -1885,9 +2053,9 @@ dependencies = [ [[package]] name = "fsst" -version = "7.0.0" +version = "8.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bcd0ce0249ac12fd44fcde62d435c36d881952c2f0df4d1de24b45e1dbba5ddb" +checksum = "7af6e24ed12cf382082d5a7f365df2b8e3d6b1518f615d54c85165e9b9718250" dependencies = [ "arrow-array", "rand 0.9.2", @@ -2061,6 +2229,12 @@ dependencies = [ "wasip3", ] +[[package]] +name = "gimli" +version = "0.32.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e629b9b98ef3dd8afe6ca2bd0f89306cec16d43d907889945bc5d6687f2f13c7" + [[package]] name = "glob" version = "0.3.3" @@ -2317,35 +2491,58 @@ dependencies = [ [[package]] name = "icu_collections" -version = "2.1.1" +version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4c6b649701667bbe825c3b7e6388cb521c23d88644678e83c0c4d0a621a34b43" +checksum = "2984d1cd16c883d7935b9e07e44071dca8d917fd52ecc02c04d5fa0b5a3f191c" dependencies = [ "displaydoc", "potential_utf", + "utf8_iter", "yoke", "zerofrom", "zerovec", ] +[[package]] +name = "icu_locale" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d5a396343c7208121dc86e35623d3dfe19814a7613cfd14964994cdc9c9a2e26" +dependencies = [ + "icu_collections", + "icu_locale_core", + "icu_locale_data", + "icu_provider", + "potential_utf", + "tinystr", + "zerovec", +] + [[package]] name = "icu_locale_core" -version = "2.1.1" +version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "edba7861004dd3714265b4db54a3c390e880ab658fec5f7db895fae2046b5bb6" +checksum = "92219b62b3e2b4d88ac5119f8904c10f8f61bf7e95b640d25ba3075e6cac2c29" dependencies = [ "displaydoc", "litemap", + "serde", "tinystr", "writeable", "zerovec", ] +[[package]] +name = "icu_locale_data" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d5fdcc9ac77c6d74ff5cf6e65ef3181d6af32003b16fce3a77fb451d2f695993" + [[package]] name = "icu_normalizer" -version = "2.1.1" +version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5f6c8828b67bf8908d82127b2054ea1b4427ff0230ee9141c54251934ab1b599" +checksum = "c56e5ee99d6e3d33bd91c5d85458b6005a22140021cc324cea84dd0e72cff3b4" dependencies = [ "icu_collections", "icu_normalizer_data", @@ -2357,15 +2554,15 @@ dependencies = [ [[package]] name = "icu_normalizer_data" -version = "2.1.1" +version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7aedcccd01fc5fe81e6b489c15b247b8b0690feb23304303a9e560f37efc560a" +checksum = "da3be0ae77ea334f4da67c12f149704f19f81d1adf7c51cf482943e84a2bad38" [[package]] name = "icu_properties" -version = "2.1.2" +version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "020bfc02fe870ec3a66d93e677ccca0562506e5872c650f893269e08615d74ec" +checksum = "bee3b67d0ea5c2cca5003417989af8996f8604e34fb9ddf96208a033901e70de" dependencies = [ "icu_collections", "icu_locale_core", @@ -2377,18 +2574,20 @@ dependencies = [ [[package]] name = "icu_properties_data" -version = "2.1.2" +version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "616c294cf8d725c6afcd8f55abc17c56464ef6211f9ed59cccffe534129c77af" +checksum = "8e2bbb201e0c04f7b4b3e14382af113e17ba4f63e2c9d2ee626b720cbce54a14" [[package]] name = "icu_provider" -version = "2.1.1" +version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "85962cf0ce02e1e0a629cc34e7ca3e373ce20dda4c4d7294bbd0bf1fdb59e614" +checksum = "139c4cf31c8b5f33d7e199446eff9c1e02decfc2f0eec2c8d71f65befa45b421" dependencies = [ "displaydoc", "icu_locale_core", + "serde", + "stable_deref_trait", "writeable", "yoke", "zerofrom", @@ -2396,6 +2595,27 @@ dependencies = [ "zerovec", ] +[[package]] +name = "icu_segmenter" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c0794db0b1a86193ac9c48768d0e6c52c54448e0870ad87907d456ee0dac964" +dependencies = [ + "icu_collections", + "icu_locale", + "icu_provider", + "icu_segmenter_data", + "potential_utf", + "utf8_iter", + "zerovec", +] + +[[package]] +name = "icu_segmenter_data" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e4a2c462a4d927d512f5f882a033ddd62f33a05bb9f230d98f736ac3dc85938f" + [[package]] name = "id-arena" version = "2.3.0" @@ -2452,6 +2672,24 @@ dependencies = [ "serde_core", ] +[[package]] +name = "inferno" +version = "0.11.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "232929e1d75fe899576a3d5c7416ad0d88dbfbb3c3d6aa00873a7408a50ddb88" +dependencies = [ + "ahash", + "indexmap 2.14.0", + "is-terminal", + "itoa", + "log", + "num-format", + "once_cell", + "quick-xml", + "rgb", + "str_stack", +] + [[package]] name = "io-uring" version = "0.7.12" @@ -2479,6 +2717,17 @@ dependencies = [ "serde", ] +[[package]] +name = "is-terminal" +version = "0.4.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3640c1c38b8e4e43584d8df18be5fc6b0aa314ce6ebf51b53313d4306cca8e46" +dependencies = [ + "hermit-abi", + "libc", + "windows-sys 0.61.2", +] + [[package]] name = "itertools" version = "0.13.0" @@ -2511,19 +2760,20 @@ checksum = "9028f49264629065d057f340a86acb84867925865f73bbf8d47b4d149a7e88b8" [[package]] name = "jieba-macros" -version = "0.9.0" +version = "0.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a29cfc5dcd898604c6f80363411fa6b6b08e27d1d253d6225b9cb6702ea02fc0" +checksum = "46adade69b634535a8f495cf87710ed893cff53e1dbc9dd750c2ab81c5defb82" dependencies = [ "phf_codegen 0.13.1", ] [[package]] name = "jieba-rs" -version = "0.9.0" +version = "0.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3245d6e9d1d5facbd6a23848d6b67e3439738ccbb4fa5a3d65da315ba1a910a2" +checksum = "11b53580aaa8ec8b713da271da434f8947409242c537a9ab3f7b76bdbb19e8a9" dependencies = [ + "bytecount", "cedarwood", "jieba-macros", "phf 0.13.1", @@ -2623,9 +2873,9 @@ dependencies = [ [[package]] name = "lance" -version = "7.0.0" +version = "8.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3944aca86f4c78f4da04af1c2bf33e664a2826b7af72972ad200d6b9de59019f" +checksum = "e2122e9f5f5f4b38bb9f0c4991c8d5171696402b3cc6187885c8385875f6df2e" dependencies = [ "arc-swap", "arrow", @@ -2653,21 +2903,22 @@ dependencies = [ "datafusion-functions", "datafusion-physical-expr", "datafusion-physical-plan", - "deepsize", "either", + "fst", "futures", "half", "humantime", "itertools 0.13.0", - "lance-arrow", - "lance-core", + "lance-arrow 8.0.0", + "lance-core 8.0.0", "lance-datafusion", "lance-encoding", "lance-file", "lance-index", "lance-io", "lance-linalg", - "lance-namespace", + "lance-namespace 8.0.0", + "lance-select", "lance-table", "lance-tokenizer", "log", @@ -2681,6 +2932,7 @@ dependencies = [ "rand 0.9.2", "rayon", "roaring", + "rustc-hash", "semver", "serde", "serde_json", @@ -2715,11 +2967,60 @@ dependencies = [ "rand 0.9.2", ] +[[package]] +name = "lance-arrow" +version = "8.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "95f45fb7e0822cc0233d686222ab451f6ec58879620029e0b7bae8192c2f7c7e" +dependencies = [ + "arrow-array", + "arrow-buffer", + "arrow-data", + "arrow-ipc", + "arrow-ord", + "arrow-schema", + "arrow-select", + "bytes", + "futures", + "getrandom 0.2.16", + "half", + "jsonb", + "num-traits", + "rand 0.9.2", +] + +[[package]] +name = "lance-arrow-scalar" +version = "58.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "771f68b04b47f3addf781116f65061808de94b05e1e9411c23c18f32d14ebe79" +dependencies = [ + "arrow-array", + "arrow-buffer", + "arrow-cast", + "arrow-data", + "arrow-row", + "arrow-schema", + "half", +] + +[[package]] +name = "lance-arrow-stats" +version = "58.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dd47ec33c90bf29f688fd02118e37d3a5ad5c339caa3163f89e417dc0867001f" +dependencies = [ + "arrow-array", + "arrow-schema", + "half", + "lance-arrow-scalar", +] + [[package]] name = "lance-bitpacking" -version = "7.0.0" +version = "8.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "80c4d12521b1945041dd515a56aa0854973138e7ac12111c92572e33e4ecb593" +checksum = "c56c21a39860ca7bae712b4e24b9fd066029c570a72428a579faf697de5b8822" dependencies = [ "arrayref", "paste", @@ -2728,23 +3029,60 @@ dependencies = [ [[package]] name = "lance-core" -version = "7.0.0" +version = "7.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13f84020da5a484e2f07dd1796e09785ed7cd889857ebc4cb77e32ef214ee594" +dependencies = [ + "arrow-array", + "arrow-buffer", + "arrow-schema", + "async-trait", + "byteorder", + "bytes", + "deepsize", + "futures", + "itertools 0.13.0", + "lance-arrow 7.0.0", + "libc", + "log", + "moka", + "num_cpus", + "object_store", + "pin-project", + "prost", + "rand 0.9.2", + "roaring", + "serde_json", + "snafu 0.9.1", + "tempfile", + "tokio", + "tokio-stream", + "tokio-util", + "tracing", + "url", +] + +[[package]] +name = "lance-core" +version = "8.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "13f84020da5a484e2f07dd1796e09785ed7cd889857ebc4cb77e32ef214ee594" +checksum = "4ccc7b0b61c11bdb74f929111214553b36b1ee43c88445edda17bc9a2bda75e9" dependencies = [ "arrow-array", "arrow-buffer", + "arrow-data", "arrow-schema", "async-trait", "byteorder", "bytes", "datafusion-common", "datafusion-sql", - "deepsize", "futures", "itertools 0.13.0", - "lance-arrow", + "lance-arrow 8.0.0", + "lance-derive", "libc", + "libm", "log", "moka", "num_cpus", @@ -2760,14 +3098,15 @@ dependencies = [ "tokio-stream", "tokio-util", "tracing", + "twox-hash", "url", ] [[package]] name = "lance-datafusion" -version = "7.0.0" +version = "8.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7460597a66534a75987993d4dac5bc330586d99c5b79ae73367dbcbd4e29e576" +checksum = "f0932140306faa6c3c4e17f0478c031e2be659ea2721311a530fb7c664e1b9a0" dependencies = [ "arrow", "arrow-array", @@ -2784,8 +3123,8 @@ dependencies = [ "datafusion-physical-expr", "futures", "jsonb", - "lance-arrow", - "lance-core", + "lance-arrow 8.0.0", + "lance-core 8.0.0", "lance-datagen", "log", "pin-project", @@ -2797,9 +3136,9 @@ dependencies = [ [[package]] name = "lance-datagen" -version = "7.0.0" +version = "8.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "046f5506ed2271cd941a050de7bf535dd3aedc291aadec836a63fa56c5926e3b" +checksum = "075c8154e6b27ff6859f90886efd8cbac348cca255c8b855da630994b4fed714" dependencies = [ "arrow", "arrow-array", @@ -2812,14 +3151,24 @@ dependencies = [ "rand 0.9.2", "rand_distr 0.5.1", "rand_xoshiro", - "random_word", +] + +[[package]] +name = "lance-derive" +version = "8.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7056da2e742fd466f6bdfaa876c91d3e0e99bb4f756be058e374ceae2630ec2f" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.117", ] [[package]] name = "lance-encoding" -version = "7.0.0" +version = "8.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7af54edf43dcf9d6a56cc636eb35d457e68373c6448dca3f0891b3325b4a24e6" +checksum = "5e165c668ae61fce336d5f6f9a999ee99b963bb395a3fb818737c533fc9528d1" dependencies = [ "arrow-arith", "arrow-array", @@ -2836,9 +3185,9 @@ dependencies = [ "hex", "hyperloglogplus", "itertools 0.13.0", - "lance-arrow", + "lance-arrow 8.0.0", "lance-bitpacking", - "lance-core", + "lance-core 8.0.0", "log", "lz4", "num-traits", @@ -2854,9 +3203,9 @@ dependencies = [ [[package]] name = "lance-file" -version = "7.0.0" +version = "8.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0772ae2d6207995dc1eb28aff9507f78e90b3362b58f311da001e9dc25f3d736" +checksum = "1691bd5c37bc5e07bde00e32950b5526c2e5653bd2493a3773712574366a37a1" dependencies = [ "arrow-arith", "arrow-array", @@ -2869,10 +3218,9 @@ dependencies = [ "byteorder", "bytes", "datafusion-common", - "deepsize", "futures", - "lance-arrow", - "lance-core", + "lance-arrow 8.0.0", + "lance-core 8.0.0", "lance-encoding", "lance-io", "log", @@ -2887,9 +3235,9 @@ dependencies = [ [[package]] name = "lance-index" -version = "7.0.0" +version = "8.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e71fbfb51096a903cb524fe0da716f5f15fbc4a6b6f84cd6dec21abf319c5e84" +checksum = "9b4792b56f0e047eed706d05a1eedc5f549090913fb4527585bb3a331b83416b" dependencies = [ "arc-swap", "arrow", @@ -2910,7 +3258,6 @@ dependencies = [ "datafusion-common", "datafusion-expr", "datafusion-physical-expr", - "deepsize", "dirs", "fst", "futures", @@ -2918,17 +3265,19 @@ dependencies = [ "itertools 0.13.0", "jieba-rs", "jsonb", - "lance-arrow", - "lance-core", + "lance-arrow 8.0.0", + "lance-arrow-stats", + "lance-core 8.0.0", "lance-datafusion", "lance-datagen", "lance-encoding", "lance-file", "lance-io", "lance-linalg", + "lance-select", "lance-table", "lance-tokenizer", - "libm", + "libsais-rs", "log", "ndarray", "num-traits", @@ -2940,6 +3289,7 @@ dependencies = [ "rand_distr 0.5.1", "rangemap", "rayon", + "regex-syntax", "roaring", "serde", "serde_json", @@ -2947,15 +3297,14 @@ dependencies = [ "tempfile", "tokio", "tracing", - "twox-hash", "uuid", ] [[package]] name = "lance-io" -version = "7.0.0" +version = "8.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bab8c98ef1b870b20541d27f3ca4efdf7c9f5c25214233be07d231ba88900219" +checksum = "6c81dda99d5b8c8741f413d65650a28ff203d94eda3cbbdda6fd3af3ea9b2a69" dependencies = [ "arrow", "arrow-arith", @@ -2970,13 +3319,12 @@ dependencies = [ "byteorder", "bytes", "chrono", - "deepsize", "futures", "http", "io-uring", - "lance-arrow", - "lance-core", - "lance-namespace", + "lance-arrow 8.0.0", + "lance-core 8.0.0", + "lance-namespace 8.0.0", "log", "moka", "object_store", @@ -2993,18 +3341,17 @@ dependencies = [ [[package]] name = "lance-linalg" -version = "7.0.0" +version = "8.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6b4c51cad0ac780b02dc4da48528479e7693c03e8d05390510bbc69ca2a9a1f1" +checksum = "5c0a8d2a2bc7e6b6229abe320ec6d9e2049a0e65e084684cba01cf032fc71896" dependencies = [ "arrow-array", "arrow-buffer", "arrow-schema", "cc", - "deepsize", "half", - "lance-arrow", - "lance-core", + "lance-arrow 8.0.0", + "lance-core 8.0.0", "num-traits", "rand 0.9.2", ] @@ -3018,36 +3365,55 @@ dependencies = [ "arrow", "async-trait", "bytes", - "lance-core", - "lance-namespace-reqwest-client", + "lance-core 7.0.0", + "lance-namespace-reqwest-client 0.7.7", + "snafu 0.9.1", +] + +[[package]] +name = "lance-namespace" +version = "8.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3ee2d75929caec41747e58ce090b1a061b650a16cb10d09998128d7912203b1d" +dependencies = [ + "arrow", + "async-trait", + "bytes", + "lance-core 8.0.0", + "lance-namespace-reqwest-client 0.8.6", "snafu 0.9.1", ] [[package]] name = "lance-namespace-impls" -version = "7.0.0" +version = "8.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e8d1231906a3cf92dd3dcda7d14a09c4835af6cd2bcd76dfd2481e87f20a282d" +checksum = "c92dd5dcb98562a91650da0b5fa63726d435fad8b03327625cc3de445b7e191a" dependencies = [ "arrow", "arrow-ipc", "arrow-schema", "async-trait", "bytes", + "datafusion-common", + "datafusion-physical-plan", "futures", "lance", - "lance-core", + "lance-core 8.0.0", "lance-index", "lance-io", "lance-linalg", - "lance-namespace", + "lance-namespace 8.0.0", "lance-table", "log", "object_store", "rand 0.9.2", + "roaring", "serde_json", + "time", "tokio", "url", + "uuid", ] [[package]] @@ -3064,11 +3430,42 @@ dependencies = [ "url", ] +[[package]] +name = "lance-namespace-reqwest-client" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba3f0a235e3ed5f8805205649ccc7d7d0f3df23ce1294242c9265ad488d7f19d" +dependencies = [ + "reqwest", + "serde", + "serde_json", + "serde_repr", + "serde_with", + "url", +] + +[[package]] +name = "lance-select" +version = "8.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf3a2162385a4392376ed7a732e070afd1432bb6f86b0ebcb7c4304c7196953b" +dependencies = [ + "arrow-array", + "arrow-buffer", + "arrow-schema", + "byteorder", + "bytes", + "itertools 0.13.0", + "lance-core 8.0.0", + "roaring", + "tracing", +] + [[package]] name = "lance-table" -version = "7.0.0" +version = "8.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b16f1355904aea4ebb04ffc70c58c97901e10bde44452b4b021de4a1f329250d" +checksum = "73da3932a85489e80d5458d4bbc1d340e15c72b89bab529723f575d4d8fda5eb" dependencies = [ "arrow", "arrow-array", @@ -3079,12 +3476,12 @@ dependencies = [ "byteorder", "bytes", "chrono", - "deepsize", "futures", - "lance-arrow", - "lance-core", + "lance-arrow 8.0.0", + "lance-core 8.0.0", "lance-file", "lance-io", + "lance-select", "log", "object_store", "prost", @@ -3105,39 +3502,44 @@ dependencies = [ [[package]] name = "lance-testing" -version = "7.0.0" +version = "8.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3094c2aacbd1fa093d809fc54fa911e3498671ba451041341d7caaa18460e6b2" +checksum = "f0a5a7b5dbda917170b705301d0c6a8753a8c02f501b20d8b7067b5463ba071d" dependencies = [ "arrow-array", "arrow-schema", - "lance-arrow", + "criterion", + "lance-arrow 8.0.0", "num-traits", + "pprof", "rand 0.9.2", ] [[package]] name = "lance-tokenizer" -version = "7.0.0" +version = "8.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b39b7f5ed9d0c0b716bf599b559d888267ed1dfe4c4e29d3648b51d2a28940cf" +checksum = "d16326f6b6d3b736aac40a0ffa78d8aa17d3a53a3766cd0af6d23db87472bd5e" dependencies = [ + "icu_segmenter", "jieba-rs", "lindera", "rust-stemmers", "serde", + "stop-words", "unicode-normalization", ] [[package]] name = "lancedb" -version = "0.30.0" +version = "0.31.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9db595d8da6c1fbf41ef6a547f27a73cc1105d83a837e8c76ce08743a3c09260" +checksum = "2bd0b54bb1cdd075efa5a8827ec16dcf5c0781253cd88e63988c174915c53fe2" dependencies = [ "ahash", "arrow", "arrow-array", + "arrow-buffer", "arrow-cast", "arrow-data", "arrow-ipc", @@ -3159,8 +3561,8 @@ dependencies = [ "futures", "half", "lance", - "lance-arrow", - "lance-core", + "lance-arrow 8.0.0", + "lance-core 8.0.0", "lance-datafusion", "lance-datagen", "lance-encoding", @@ -3168,7 +3570,7 @@ dependencies = [ "lance-index", "lance-io", "lance-linalg", - "lance-namespace", + "lance-namespace 8.0.0", "lance-namespace-impls", "lance-table", "lance-testing", @@ -3284,6 +3686,15 @@ dependencies = [ "libc", ] +[[package]] +name = "libsais-rs" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "40fe164dbd47ea0c20e78a121c980ef673326905f1d4fba55e3645a20ef6717f" +dependencies = [ + "rayon", +] + [[package]] name = "lindera" version = "3.0.7" @@ -3595,6 +4006,17 @@ dependencies = [ "rawpointer", ] +[[package]] +name = "nix" +version = "0.26.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "598beaf3cc6fdd9a5dfb1630c2800c7acd31df7aaf0f565796fba2b53ca1af1b" +dependencies = [ + "bitflags 1.3.2", + "cfg-if", + "libc", +] + [[package]] name = "nom" version = "8.0.0" @@ -3652,9 +4074,19 @@ dependencies = [ [[package]] name = "num-conv" -version = "0.1.0" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "521739c6d2bac4aa25192232afe6841231376b2b26d4d9fae5ecf8ca5772e441" + +[[package]] +name = "num-format" +version = "0.4.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "51d515d32fb182ee37cda2ccdcb92950d6a3c2893aa280e540671c2cd0f3b1d9" +checksum = "a652d9771a63711fd3c3deb670acfbe5c30a4072e664d7a3bf5a9e1056ac72c3" +dependencies = [ + "arrayvec", + "itoa", +] [[package]] name = "num-integer" @@ -3694,6 +4126,15 @@ dependencies = [ "memchr", ] +[[package]] +name = "object" +version = "0.37.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff76201f031d8863c38aa7f905eca4f53abbfa15f609db4277d44cd8938f33fe" +dependencies = [ + "memchr", +] + [[package]] name = "object_store" version = "0.13.2" @@ -3726,6 +4167,12 @@ version = "1.21.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" +[[package]] +name = "oorandom" +version = "11.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6790f58c7ff633d8771f42965289203411a5e5c68388703c06e14f24770b41e" + [[package]] name = "openssl-probe" version = "0.1.6" @@ -3747,6 +4194,16 @@ dependencies = [ "num-traits", ] +[[package]] +name = "page_size" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "30d5b2194ed13191c1999ae0704b7839fb18384fa22e49b57eeaa97d79ce40da" +dependencies = [ + "libc", + "winapi", +] + [[package]] name = "parking" version = "2.2.1" @@ -3975,6 +4432,34 @@ dependencies = [ "array-init-cursor", ] +[[package]] +name = "plotters" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5aeb6f403d7a4911efb1e33402027fc44f29b5bf6def3effcc22d7bb75f2b747" +dependencies = [ + "num-traits", + "plotters-backend", + "plotters-svg", + "wasm-bindgen", + "web-sys", +] + +[[package]] +name = "plotters-backend" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df42e13c12958a16b3f7f4386b9ab1f3e7933914ecea48da7139435263a4172a" + +[[package]] +name = "plotters-svg" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "51bae2ac328883f7acdfea3d66a7c35751187f870bc81f94563733a154d7a670" +dependencies = [ + "plotters-backend", +] + [[package]] name = "polars" version = "0.39.2" @@ -4345,6 +4830,8 @@ version = "0.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b73949432f5e2a09657003c25bca5e19a0e9c84f8058ca374f49e0ebe605af77" dependencies = [ + "serde_core", + "writeable", "zerovec", ] @@ -4354,6 +4841,28 @@ version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "439ee305def115ba05938db6eb1644ff94165c5ab5e9420d1c1bcedbba909391" +[[package]] +name = "pprof" +version = "0.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "38a01da47675efa7673b032bf8efd8214f1917d89685e07e395ab125ea42b187" +dependencies = [ + "aligned-vec", + "backtrace", + "cfg-if", + "findshlibs", + "inferno", + "libc", + "log", + "nix", + "once_cell", + "smallvec", + "spin", + "symbolic-demangle", + "tempfile", + "thiserror 2.0.18", +] + [[package]] name = "ppv-lite86" version = "0.2.21" @@ -4463,6 +4972,15 @@ dependencies = [ "syn 2.0.117", ] +[[package]] +name = "quick-xml" +version = "0.26.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f50b1c63b38611e7d4d7f68b82d3ad0cc71a2ad2e7f61fc10f1328d917c93cd" +dependencies = [ + "memchr", +] + [[package]] name = "quinn" version = "0.11.9" @@ -4642,19 +5160,6 @@ dependencies = [ "rand_core 0.9.3", ] -[[package]] -name = "random_word" -version = "0.5.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e47a395bdb55442b883c89062d6bcff25dc90fa5f8369af81e0ac6d49d78cf81" -dependencies = [ - "ahash", - "brotli", - "paste", - "rand 0.9.2", - "unicase", -] - [[package]] name = "rangemap" version = "1.7.0" @@ -4839,6 +5344,15 @@ dependencies = [ "web-sys", ] +[[package]] +name = "rgb" +version = "0.8.53" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "47b34b781b31e5d73e9fbc8689c70551fd1ade9a19e3e28cfec8580a79290cc4" +dependencies = [ + "bytemuck", +] + [[package]] name = "ring" version = "0.17.14" @@ -4902,7 +5416,7 @@ dependencies = [ "arrow-schema", "futures", "futures-util", - "lance-namespace", + "lance-namespace 7.0.0", "lancedb", "polars", "polars-arrow", @@ -4922,6 +5436,12 @@ dependencies = [ "serde_derive", ] +[[package]] +name = "rustc-demangle" +version = "0.1.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b50b8869d9fc858ce7266cce0194bd74df58b9d0e3f6df3a9fc8eb470d95c09d" + [[package]] name = "rustc-hash" version = "2.1.1" @@ -5221,6 +5741,12 @@ dependencies = [ "unsafe-libyaml", ] +[[package]] +name = "sha1_smol" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbfa15b3dddfee50a0fff136974b3e1bde555604ba463834a7eb7deb6417705d" + [[package]] name = "sha2" version = "0.10.9" @@ -5349,6 +5875,15 @@ dependencies = [ "windows-sys 0.60.2", ] +[[package]] +name = "spin" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d5fe4ccb98d9c292d56fec89a5e07da7fc4cf0dc11e156b41793132775d3e591" +dependencies = [ + "lock_api", +] + [[package]] name = "sqlparser" version = "0.39.0" @@ -5416,6 +5951,21 @@ version = "0.2.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e51f1e89f093f99e7432c491c382b88a6860a5adbe6bf02574bf0a08efff1978" +[[package]] +name = "stop-words" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d68df56303396bcfb639455b3c166804aeb7994005010aab5e9e8a1277b8871d" +dependencies = [ + "serde_json", +] + +[[package]] +name = "str_stack" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f446288b699d66d0fd2e30d1cfe7869194312524b3b9252594868ed26ef056a" + [[package]] name = "streaming-decompression" version = "0.1.2" @@ -5505,6 +6055,29 @@ version = "2.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292" +[[package]] +name = "symbolic-common" +version = "12.18.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "332615d90111d8eeaf86a84dc9bbe9f65d0d8c5cf11b4caccedc37754eb0dcfd" +dependencies = [ + "debugid", + "memmap2 0.9.10", + "stable_deref_trait", + "uuid", +] + +[[package]] +name = "symbolic-demangle" +version = "12.18.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "912017718eb4d21930546245af9a3475c9dccf15675a5c215664e76621afc471" +dependencies = [ + "cpp_demangle", + "rustc-demangle", + "symbolic-common", +] + [[package]] name = "syn" version = "1.0.109" @@ -5652,30 +6225,30 @@ dependencies = [ [[package]] name = "time" -version = "0.3.44" +version = "0.3.47" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "91e7d9e3bb61134e77bde20dd4825b97c010155709965fedf0f49bb138e52a9d" +checksum = "743bd48c283afc0388f9b8827b976905fb217ad9e647fae3a379a9283c4def2c" dependencies = [ "deranged", "itoa", "num-conv", "powerfmt", - "serde", + "serde_core", "time-core", "time-macros", ] [[package]] name = "time-core" -version = "0.1.6" +version = "0.1.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "40868e7c1d2f0b8d73e4a8c7f0ff63af4f6d19be117e90bd73eb1d62cf831c6b" +checksum = "7694e1cfe791f8d31026952abf09c69ca6f6fa4e1a1229e18988f06a04a12dca" [[package]] name = "time-macros" -version = "0.2.24" +version = "0.2.27" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "30cfb0125f12d9c277f35663a0a33f8c30190f4e4574868a330595412d34ebf3" +checksum = "2e70e4c5a0e0a8a4823ad65dfe1a6930e4f4d756dcd9dd7939022b5e8c501215" dependencies = [ "num-conv", "time-core", @@ -5692,14 +6265,25 @@ dependencies = [ [[package]] name = "tinystr" -version = "0.8.2" +version = "0.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "42d3e9c45c09de15d06dd8acf5f4e0e399e85927b7f00711024eb7ae10fa4869" +checksum = "c8323304221c2a851516f22236c5722a72eaa19749016521d6dff0824447d96d" dependencies = [ "displaydoc", + "serde_core", "zerovec", ] +[[package]] +name = "tinytemplate" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "be4d6b5f19ff7664e8c98d03e2139cb510db9b0a60b55f8e8709b689d939b6bc" +dependencies = [ + "serde", + "serde_json", +] + [[package]] name = "tinyvec" version = "1.10.0" @@ -6009,6 +6593,7 @@ dependencies = [ "getrandom 0.4.2", "js-sys", "serde_core", + "sha1_smol", "wasm-bindgen", ] @@ -6672,9 +7257,9 @@ checksum = "fdd20c5420375476fbd4394763288da7eb0cc0b8c11deed431a91562af7335d3" [[package]] name = "yoke" -version = "0.8.1" +version = "0.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "72d6e5c6afb84d73944e5cedb052c4680d5657337201555f9f2a16b7406d4954" +checksum = "709fe23a0424b6a435d82152b1bd3fdfb0833487d5fa90d05d42762a9891fef5" dependencies = [ "stable_deref_trait", "yoke-derive", @@ -6683,9 +7268,9 @@ dependencies = [ [[package]] name = "yoke-derive" -version = "0.8.1" +version = "0.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b659052874eb698efe5b9e8cf382204678a0086ebf46982b79d6ca3182927e5d" +checksum = "de844c262c8848816172cef550288e7dc6c7b7814b4ee56b3e1553f275f1858e" dependencies = [ "proc-macro2", "quote", @@ -6742,21 +7327,23 @@ checksum = "b97154e67e32c85465826e8bcc1c59429aaaf107c1e4a9e53c8d8ccd5eff88d0" [[package]] name = "zerotrie" -version = "0.2.3" +version = "0.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2a59c17a5562d507e4b54960e8569ebee33bee890c70aa3fe7b97e85a9fd7851" +checksum = "0f9152d31db0792fa83f70fb2f83148effb5c1f5b8c7686c3459e361d9bc20bf" dependencies = [ "displaydoc", "yoke", "zerofrom", + "zerovec", ] [[package]] name = "zerovec" -version = "0.11.5" +version = "0.11.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6c28719294829477f525be0186d13efa9a3c602f7ec202ca9e353d310fb9a002" +checksum = "90f911cbc359ab6af17377d242225f4d75119aec87ea711a880987b18cd7b239" dependencies = [ + "serde", "yoke", "zerofrom", "zerovec-derive", @@ -6764,9 +7351,9 @@ dependencies = [ [[package]] name = "zerovec-derive" -version = "0.11.2" +version = "0.11.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eadce39539ca5cb3985590102671f2567e659fca9666581ad3411d59207951f3" +checksum = "625dc425cab0dca6dc3c3319506e6593dcb08a9f387ea3b284dbd52a92c40555" dependencies = [ "proc-macro2", "quote", diff --git a/tests/rs/Cargo.toml b/tests/rs/Cargo.toml index 255bc1f..c8a4cc7 100644 --- a/tests/rs/Cargo.toml +++ b/tests/rs/Cargo.toml @@ -10,7 +10,7 @@ arrow-schema = "58.3.0" futures = "0.3.31" futures-util = "0.3.31" lance-namespace = "7.0.0" -lancedb = { version = "0.30.0", features = ["polars"] } +lancedb = { version = "0.31.0", features = ["polars"] } polars = ">=0.37,<0.40.0" polars-arrow = ">=0.37,<0.40.0" serde = { version = "1", features = ["derive"] } diff --git a/tests/rs/tables.rs b/tests/rs/tables.rs index 7221709..f9899be 100644 --- a/tests/rs/tables.rs +++ b/tests/rs/tables.rs @@ -10,7 +10,9 @@ use arrow_array::{ RecordBatchIterator, RecordBatchReader, StringArray, }; use arrow_schema::{DataType, Field, Schema}; +use futures::TryStreamExt; use lancedb::connect; +use lancedb::query::ExecutableQuery; use lancedb::database::CreateTableMode; use lancedb::table::{ ColumnAlteration, Duration, FieldMetadataUpdate, NewColumnTransform, OptimizeAction, @@ -67,6 +69,45 @@ fn make_quotes_reader(rows: Vec<(i64, &str, &str)>) -> Box Box { + const DIM: usize = 4; + let schema = Arc::new(Schema::new(vec![ + Field::new("id", DataType::Int32, false), + Field::new( + "vector", + DataType::FixedSizeList( + Arc::new(Field::new("item", DataType::Float32, true)), + DIM as i32, + ), + true, + ), + Field::new("text", DataType::Utf8, false), + ])); + + let ids = Int32Array::from_iter_values(0..n); + let vectors = FixedSizeListArray::from_iter_primitive::( + (0..n).map(|i| { + Some( + (0..DIM) + .map(|d| Some((((i as usize * 31 + d * 7) % 97) as f32) / 97.0)) + .collect::>>(), + ) + }), + DIM as i32, + ); + let texts = StringArray::from_iter_values((0..n).map(|i| format!("product number {i}"))); + + let batch = RecordBatch::try_new( + schema.clone(), + vec![Arc::new(ids), Arc::new(vectors), Arc::new(texts)], + ) + .unwrap(); + let reader = RecordBatchIterator::new(vec![Ok(batch)].into_iter(), schema); + Box::new(reader) +} + #[allow(dead_code)] async fn update_connect_enterprise_example() { // --8<-- [start:update_connect_enterprise] @@ -1338,45 +1379,134 @@ async fn main() { .await .unwrap(); - // --8<-- [start:branches] use lancedb::table::Ref; + // --8<-- [start:branch_create] // Fork an isolated, writable branch from main's latest version. - // The returned handle is scoped to the branch; writes on it do not - // affect main. + // `create_branch` returns a table handle scoped to the new branch. let branch = branches_table .create_branch("exp", Ref::Version(None, None)) .await .unwrap(); + // --8<-- [end:branch_create] + + // --8<-- [start:branch_write] + // Writes land on the branch handle only; main is left untouched. branch .add(make_quotes_reader(vec![(4, "Lancelot", "For the realm!")])) .execute() .await .unwrap(); - let branch_rows = branch.count_rows(None).await.unwrap(); - let main_rows = branches_table.count_rows(None).await.unwrap(); - println!("Branch rows: {}", branch_rows); // 4 - println!("Main rows: {}", main_rows); // 3; main is untouched + println!("Branch rows: {}", branch.count_rows(None).await.unwrap()); // 4 + println!("Main rows: {}", branches_table.count_rows(None).await.unwrap()); // 3 - // List all branches, mapping name to branch metadata. - let all_branches = branches_table.list_branches().await.unwrap(); - println!("Branches: {:?}", all_branches); + // List every branch, each mapped to its metadata (including its fork point). + println!("Branches: {:?}", branches_table.list_branches().await.unwrap()); + // --8<-- [end:branch_write] - // Reopen the branch later by name, or open it directly via the builder. - let checked_out = branches_table.checkout_branch("exp").await.unwrap(); + // --8<-- [start:branch_reopen] + // Reopen an existing branch by name from the table handle... + let checked_out = branches_table.checkout_branch("exp", None).await.unwrap(); + // ...or open it directly via the connection's builder. let opened = db .open_table("quotes_branches_example") .branch("exp") .execute() .await .unwrap(); - let checked_out_rows = checked_out.count_rows(None).await.unwrap(); - let opened_rows = opened.count_rows(None).await.unwrap(); - println!("Reopened rows: {}, {}", checked_out_rows, opened_rows); // both 4 - - // Delete a branch when you're done with it. + println!( + "Reopened rows: {}, {}", + checked_out.count_rows(None).await.unwrap(), + opened.count_rows(None).await.unwrap() + ); // both 4 + // --8<-- [end:branch_reopen] + + // --8<-- [start:branch_delete] + // Delete the branch and its branch-local history. Data on main is safe. branches_table.delete_branch("exp").await.unwrap(); - // --8<-- [end:branches] + // --8<-- [end:branch_delete] + assert_eq!(branches_table.count_rows(None).await.unwrap(), 3); assert!(!branches_table.list_branches().await.unwrap().contains_key("exp")); + + // Setup: a branch that diverges from main, ready to promote back. + let promo = branches_table + .create_branch("promote", Ref::Version(None, None)) + .await + .unwrap(); + promo + .update() + .only_if("id = 1") + .column("quote", "'Revised on the branch'") + .execute() + .await + .unwrap(); + promo + .add(make_quotes_reader(vec![(4, "Galahad", "The grail awaits.")])) + .execute() + .await + .unwrap(); + + // --8<-- [start:branch_promote] + // There is no built-in merge yet, so promote a branch by writing its rows + // back to main with a normal ingestion call. `merge_insert` keys on a + // unique column, so rows that already exist on main are updated in place and + // new rows are appended — exactly what an upsert-style ingestion job does. + let schema = promo.schema().await.unwrap(); + let batches = promo + .query() + .execute() + .await + .unwrap() + .try_collect::>() + .await + .unwrap(); + let promoted = RecordBatchIterator::new(batches.into_iter().map(Ok), schema); + + let mut merge = branches_table.merge_insert(&["id"]); + merge + .when_matched_update_all(None) // update rows that already exist on main + .when_not_matched_insert_all(); // insert rows that are new on the branch + merge.execute(Box::new(promoted)).await.unwrap(); + // --8<-- [end:branch_promote] + + assert_eq!(branches_table.count_rows(None).await.unwrap(), 4); + branches_table.delete_branch("promote").await.unwrap(); + + // Setup: a larger table with a vector and a text column to index. + let products = db + .create_table("products_branch_index", make_products_reader(512)) + .mode(CreateTableMode::Overwrite) + .execute() + .await + .unwrap(); + + // --8<-- [start:branch_index] + use lancedb::index::scalar::FtsIndexBuilder; + use lancedb::index::Index; + + // Build and validate indexes on a branch before promoting them to main. + let dev = products + .create_branch("index-dev", Ref::Version(None, None)) + .await + .unwrap(); + + // A vector (ANN) index and a full-text search index, both branch-scoped. + dev.create_index(&["vector"], Index::Auto) + .execute() + .await + .unwrap(); + dev.create_index(&["text"], Index::FTS(FtsIndexBuilder::default())) + .execute() + .await + .unwrap(); + + // Both indexes live only on the branch; main still has none. + println!("Branch indexes: {}", dev.list_indices().await.unwrap().len()); // 2 + println!("Main indexes: {}", products.list_indices().await.unwrap().len()); // 0 + // --8<-- [end:branch_index] + + assert_eq!(dev.list_indices().await.unwrap().len(), 2); + assert_eq!(products.list_indices().await.unwrap().len(), 0); + products.delete_branch("index-dev").await.unwrap(); } diff --git a/tests/ts/package-lock.json b/tests/ts/package-lock.json index 44285f7..8713b65 100644 --- a/tests/ts/package-lock.json +++ b/tests/ts/package-lock.json @@ -10,7 +10,7 @@ "license": "Apache-2.0", "dependencies": { "@huggingface/transformers": "^3.0.2", - "@lancedb/lancedb": "^0.30.0", + "@lancedb/lancedb": "^0.31.0", "openai": "^4.29.2", "sharp": "^0.33.5" }, @@ -1610,9 +1610,9 @@ } }, "node_modules/@lancedb/lancedb": { - "version": "0.30.0", - "resolved": "https://registry.npmjs.org/@lancedb/lancedb/-/lancedb-0.30.0.tgz", - "integrity": "sha512-d0FoEL6cthqgsulqAc7fck6kRXrSRGMTqlKYbhSGSazHU6vB2GEpD737Mu0HZd7fMyBUdhR9sD1W2C9uQZ5p0Q==", + "version": "0.31.0", + "resolved": "https://registry.npmjs.org/@lancedb/lancedb/-/lancedb-0.31.0.tgz", + "integrity": "sha512-EUEVpheKhaCNE6ybcW760OUyfeei2dKR2ZwgLWeC/ntHL4BBiBLIErh9fuEuUP3/mAx4B5UFraB2m5nDUx5XEA==", "cpu": [ "x64", "arm64" @@ -1630,22 +1630,24 @@ "node": ">= 18" }, "optionalDependencies": { - "@lancedb/lancedb-darwin-arm64": "0.30.0", - "@lancedb/lancedb-linux-arm64-gnu": "0.30.0", - "@lancedb/lancedb-linux-arm64-musl": "0.30.0", - "@lancedb/lancedb-linux-x64-gnu": "0.30.0", - "@lancedb/lancedb-linux-x64-musl": "0.30.0", - "@lancedb/lancedb-win32-arm64-msvc": "0.30.0", - "@lancedb/lancedb-win32-x64-msvc": "0.30.0" + "@huggingface/transformers": "3.0.2", + "@lancedb/lancedb-darwin-arm64": "0.31.0", + "@lancedb/lancedb-linux-arm64-gnu": "0.31.0", + "@lancedb/lancedb-linux-arm64-musl": "0.31.0", + "@lancedb/lancedb-linux-x64-gnu": "0.31.0", + "@lancedb/lancedb-linux-x64-musl": "0.31.0", + "@lancedb/lancedb-win32-arm64-msvc": "0.31.0", + "@lancedb/lancedb-win32-x64-msvc": "0.31.0", + "openai": "4.29.2" }, "peerDependencies": { "apache-arrow": ">=15.0.0 <=18.1.0" } }, "node_modules/@lancedb/lancedb-darwin-arm64": { - "version": "0.30.0", - "resolved": "https://registry.npmjs.org/@lancedb/lancedb-darwin-arm64/-/lancedb-darwin-arm64-0.30.0.tgz", - "integrity": "sha512-x6dmsjRIv0xumELYnFAEfyFDxqcO/n4rHYCJvC27RbRez0UmbByi6OMTgbSoSTatrQSPRCL7JJSa5pwNeawnIg==", + "version": "0.31.0", + "resolved": "https://registry.npmjs.org/@lancedb/lancedb-darwin-arm64/-/lancedb-darwin-arm64-0.31.0.tgz", + "integrity": "sha512-6n3VxAenwcNWQpk9Ta4NL/KhpSywskafarBakzFPGe/OXzdKXbmbXdrhGdl8oMacbfgql6wmW3DcAJAbsiThvw==", "cpu": [ "arm64" ], @@ -1659,9 +1661,9 @@ } }, "node_modules/@lancedb/lancedb-linux-arm64-gnu": { - "version": "0.30.0", - "resolved": "https://registry.npmjs.org/@lancedb/lancedb-linux-arm64-gnu/-/lancedb-linux-arm64-gnu-0.30.0.tgz", - "integrity": "sha512-CWUex7hRNiLkXFeTQlUGPyy5VktbXoUmQdZuiVCETZ6ggljEC7c7Qvzu2ge+jEZML+UE7tXL2lVC3klRFGczng==", + "version": "0.31.0", + "resolved": "https://registry.npmjs.org/@lancedb/lancedb-linux-arm64-gnu/-/lancedb-linux-arm64-gnu-0.31.0.tgz", + "integrity": "sha512-dWVHk5xhTpXQt08y3pxQFf5DK/O2saiWT8AZIIlonFRbGVhLm1KzBCfCgS1WUVbdleSkqAqY3NnMxZzu/dhx9Q==", "cpu": [ "arm64" ], @@ -1675,9 +1677,9 @@ } }, "node_modules/@lancedb/lancedb-linux-arm64-musl": { - "version": "0.30.0", - "resolved": "https://registry.npmjs.org/@lancedb/lancedb-linux-arm64-musl/-/lancedb-linux-arm64-musl-0.30.0.tgz", - "integrity": "sha512-2MHmAS4tKePNVbwfgDrjCFj6BVuAgXlL2c9iWk7TfcwL+jcSxo52LFx03O0+ArpVCF2sI6aoDqZaQNre5zMniQ==", + "version": "0.31.0", + "resolved": "https://registry.npmjs.org/@lancedb/lancedb-linux-arm64-musl/-/lancedb-linux-arm64-musl-0.31.0.tgz", + "integrity": "sha512-4y49+83R34IR75H2NPeDtHgSnUIKE9gJoSiELsGmCgICoFTZRVxq88atBEl0BfpsuuUc8S2bSx45mf3H4lZmWA==", "cpu": [ "arm64" ], @@ -1691,9 +1693,9 @@ } }, "node_modules/@lancedb/lancedb-linux-x64-gnu": { - "version": "0.30.0", - "resolved": "https://registry.npmjs.org/@lancedb/lancedb-linux-x64-gnu/-/lancedb-linux-x64-gnu-0.30.0.tgz", - "integrity": "sha512-0OpDNxsDE4OXD+PIFd7KdE42xhYIE+fZL+jCm1v3dTug4UEhumWBuSgbUIBP7t0yJZHwh62/QivVh/V1cPB2Bg==", + "version": "0.31.0", + "resolved": "https://registry.npmjs.org/@lancedb/lancedb-linux-x64-gnu/-/lancedb-linux-x64-gnu-0.31.0.tgz", + "integrity": "sha512-AFc0qTNdjoPor5bEHRuijft0BGNSiyHlF0cdUdsApLvjrl5jTBzw59+EqzjjCevWz5w/44BBjaTt6ZChWVTc/Q==", "cpu": [ "x64" ], @@ -1707,9 +1709,9 @@ } }, "node_modules/@lancedb/lancedb-linux-x64-musl": { - "version": "0.30.0", - "resolved": "https://registry.npmjs.org/@lancedb/lancedb-linux-x64-musl/-/lancedb-linux-x64-musl-0.30.0.tgz", - "integrity": "sha512-4Bq7VngQt+lyxIcu79EN8nYJs8gvUnrIT6I/t2MSlpG/BXWHZG2A+PRii1Zq4GCUlRTTG+RlieCv8CBGSPm8bw==", + "version": "0.31.0", + "resolved": "https://registry.npmjs.org/@lancedb/lancedb-linux-x64-musl/-/lancedb-linux-x64-musl-0.31.0.tgz", + "integrity": "sha512-VXtu/xTJXtkfTyUH+MXbMYCGVCAvHJvGDoPNIqDBtrnX7uHlgmxPbw7V97k18/5UVLU2v8NhsM5sCiMD4si+Tw==", "cpu": [ "x64" ], @@ -1723,9 +1725,9 @@ } }, "node_modules/@lancedb/lancedb-win32-arm64-msvc": { - "version": "0.30.0", - "resolved": "https://registry.npmjs.org/@lancedb/lancedb-win32-arm64-msvc/-/lancedb-win32-arm64-msvc-0.30.0.tgz", - "integrity": "sha512-N2DQg2XBWZirn5jS6kRJUxF679t3sKcIxBwP9zY4Idq5OVLAj0yfLueWIKhYxv8en7pBFYWdgw5j9dTS7XajyQ==", + "version": "0.31.0", + "resolved": "https://registry.npmjs.org/@lancedb/lancedb-win32-arm64-msvc/-/lancedb-win32-arm64-msvc-0.31.0.tgz", + "integrity": "sha512-AvbnX/24uPh6UP9IRdYZNLc+2iE4uLKIQcEbGXUMLsVIMWQvuHfRamZpIlWb50tz/C7Wo5aPiQIkolPJN5fiuw==", "cpu": [ "arm64" ], @@ -1739,9 +1741,9 @@ } }, "node_modules/@lancedb/lancedb-win32-x64-msvc": { - "version": "0.30.0", - "resolved": "https://registry.npmjs.org/@lancedb/lancedb-win32-x64-msvc/-/lancedb-win32-x64-msvc-0.30.0.tgz", - "integrity": "sha512-CDgN/ZmYqSlVX2nBJAF2PYEwqBBxotCVORjagmvrd0k5D7RBLlAQUEAR4gDMum2BpYsUkzdTYQpquLjRCVbwbQ==", + "version": "0.31.0", + "resolved": "https://registry.npmjs.org/@lancedb/lancedb-win32-x64-msvc/-/lancedb-win32-x64-msvc-0.31.0.tgz", + "integrity": "sha512-reFE43TCr5Lifni2fC5YiyOnjDTAXOY37+rAY01FFKZyhjZkhRy9NZN1WvWJ35XHC4eeUVOW4+UXOvSV0djyCA==", "cpu": [ "x64" ], @@ -1754,6 +1756,47 @@ "node": ">= 18" } }, + "node_modules/@lancedb/lancedb/node_modules/@types/node": { + "version": "18.19.130", + "resolved": "https://registry.npmjs.org/@types/node/-/node-18.19.130.tgz", + "integrity": "sha512-GRaXQx6jGfL8sKfaIDD6OupbIHBr9jv7Jnaml9tB7l4v068PAOXqfcujMMo5PhbIs6ggR1XODELqahT2R8v0fg==", + "license": "MIT", + "optional": true, + "dependencies": { + "undici-types": "~5.26.4" + } + }, + "node_modules/@lancedb/lancedb/node_modules/openai": { + "version": "4.29.2", + "resolved": "https://registry.npmjs.org/openai/-/openai-4.29.2.tgz", + "integrity": "sha512-cPkT6zjEcE4qU5OW/SoDDuXEsdOLrXlAORhzmaguj5xZSPlgKvLhi27sFWhLKj07Y6WKNWxcwIbzm512FzTBNQ==", + "license": "Apache-2.0", + "optional": true, + "dependencies": { + "@types/node": "^18.11.18", + "@types/node-fetch": "^2.6.4", + "abort-controller": "^3.0.0", + "agentkeepalive": "^4.2.1", + "digest-fetch": "^1.3.0", + "form-data-encoder": "1.7.2", + "formdata-node": "^4.3.2", + "node-fetch": "^2.6.7", + "web-streams-polyfill": "^3.2.1" + }, + "bin": { + "openai": "bin/cli" + } + }, + "node_modules/@lancedb/lancedb/node_modules/web-streams-polyfill": { + "version": "3.3.3", + "resolved": "https://registry.npmjs.org/web-streams-polyfill/-/web-streams-polyfill-3.3.3.tgz", + "integrity": "sha512-d2JWLCivmZYTSIoge9MsgFCZrt571BikcWGYkjC1khllbTeDlGqZ2D8vD8E/lJa8WGWbb7Plm8/XJYV7IJHZZw==", + "license": "MIT", + "optional": true, + "engines": { + "node": ">= 8" + } + }, "node_modules/@pkgjs/parseargs": { "version": "0.11.0", "resolved": "https://registry.npmjs.org/@pkgjs/parseargs/-/parseargs-0.11.0.tgz", @@ -2225,6 +2268,12 @@ "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.2.tgz", "integrity": "sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw==" }, + "node_modules/base-64": { + "version": "0.1.0", + "resolved": "https://registry.npmjs.org/base-64/-/base-64-0.1.0.tgz", + "integrity": "sha512-Y5gU45svrR5tI2Vt/X9GPd3L0HNIKzGu202EjxrXMpuc2V2CiKgemAbUUsqYmZJvPtCXoUKjNZwBJzsNScUbXA==", + "optional": true + }, "node_modules/brace-expansion": { "version": "1.1.12", "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.12.tgz", @@ -2397,6 +2446,16 @@ "node": ">=10" } }, + "node_modules/charenc": { + "version": "0.0.2", + "resolved": "https://registry.npmjs.org/charenc/-/charenc-0.0.2.tgz", + "integrity": "sha512-yrLQ/yVUFXkzg7EDQsPieE/53+0RlaWTs+wBrvW36cyilJ2SaDWfl4Yj7MtLTXleV9uEKefbAGUPv2/iWSooRA==", + "license": "BSD-3-Clause", + "optional": true, + "engines": { + "node": "*" + } + }, "node_modules/ci-info": { "version": "3.9.0", "resolved": "https://registry.npmjs.org/ci-info/-/ci-info-3.9.0.tgz", @@ -2591,6 +2650,16 @@ "node": ">= 8" } }, + "node_modules/crypt": { + "version": "0.0.2", + "resolved": "https://registry.npmjs.org/crypt/-/crypt-0.0.2.tgz", + "integrity": "sha512-mCxBlsHFYh9C+HVpiEacem8FEBnMXgU9gy4zmNC+SXAZNB/1idgp/aulFJ4FgCi7GPEVbfyng092GqL2k2rmow==", + "license": "BSD-3-Clause", + "optional": true, + "engines": { + "node": "*" + } + }, "node_modules/debug": { "version": "4.3.7", "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.7.tgz", @@ -2665,6 +2734,17 @@ "node": "^14.15.0 || ^16.10.0 || >=18.0.0" } }, + "node_modules/digest-fetch": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/digest-fetch/-/digest-fetch-1.3.0.tgz", + "integrity": "sha512-CGJuv6iKNM7QyZlM2T3sPAdZWd/p9zQiRNS9G+9COUCwzWFTs0Xp8NF5iePx7wtvhDykReiRRrSeNb4oMmB8lA==", + "license": "ISC", + "optional": true, + "dependencies": { + "base-64": "^0.1.0", + "md5": "^2.3.0" + } + }, "node_modules/dunder-proto": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/dunder-proto/-/dunder-proto-1.0.1.tgz", @@ -3292,6 +3372,13 @@ "resolved": "https://registry.npmjs.org/is-arrayish/-/is-arrayish-0.3.2.tgz", "integrity": "sha512-eVRqCvVlZbuw3GrM63ovNSNAeA1K16kaR/LRY/92w0zxQ5/1YzwblUX652i4Xs9RwAGjW9d9y6X88t8OaAJfWQ==" }, + "node_modules/is-buffer": { + "version": "1.1.6", + "resolved": "https://registry.npmjs.org/is-buffer/-/is-buffer-1.1.6.tgz", + "integrity": "sha512-NcdALwpXkTm5Zvvbk7owOUSvVvBKDgKP5/ewfXEznmQFfs4ZRmanOeKBTjRVjka3QFoN6XJ+9F3USqfHqTaU5w==", + "license": "MIT", + "optional": true + }, "node_modules/is-core-module": { "version": "2.15.1", "resolved": "https://registry.npmjs.org/is-core-module/-/is-core-module-2.15.1.tgz", @@ -4182,6 +4269,18 @@ "node": ">= 0.4" } }, + "node_modules/md5": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/md5/-/md5-2.3.0.tgz", + "integrity": "sha512-T1GITYmFaKuO91vxyoQMFETst+O71VUPEU3ze5GNzDm0OWdP8v1ziTaAEPUr/3kLsY3Sftgz242A1SetQiDL7g==", + "license": "BSD-3-Clause", + "optional": true, + "dependencies": { + "charenc": "0.0.2", + "crypt": "0.0.2", + "is-buffer": "~1.1.6" + } + }, "node_modules/merge-stream": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/merge-stream/-/merge-stream-2.0.0.tgz", diff --git a/tests/ts/package.json b/tests/ts/package.json index 4a8b6b5..1e1966c 100644 --- a/tests/ts/package.json +++ b/tests/ts/package.json @@ -18,7 +18,7 @@ "license": "Apache-2.0", "dependencies": { "@huggingface/transformers": "^3.0.2", - "@lancedb/lancedb": "^0.30.0", + "@lancedb/lancedb": "^0.31.0", "openai": "^4.29.2", "sharp": "^0.33.5" }, diff --git a/tests/ts/tables.test.ts b/tests/ts/tables.test.ts index 2a671d5..84e2b5e 100644 --- a/tests/ts/tables.test.ts +++ b/tests/ts/tables.test.ts @@ -862,36 +862,99 @@ test("branch snippets (async)", async () => { { mode: "overwrite" }, ); - // --8<-- [start:branches] const branches = await table.branches(); + // --8<-- [start:branch_create] // Fork an isolated, writable branch from main's latest version. - // The returned handle is scoped to the branch; writes on it do not - // affect main. + // `create` returns a table handle scoped to the new branch. const branch = await branches.create("exp"); + // --8<-- [end:branch_create] + + // --8<-- [start:branch_write] + // Writes land on the branch handle only; main is left untouched. await branch.add([{ id: 4, author: "Lancelot", quote: "For the realm!" }]); console.log(await branch.countRows()); // 4 rows on the branch - console.log(await table.countRows()); // 3 rows; main is untouched + console.log(await table.countRows()); // 3 rows; main is unaffected - // List all branches, mapping name to branch metadata. + // List every branch, each mapped to its metadata (including its fork point). console.log(await branches.list()); + // --8<-- [end:branch_write] - // Reopen the branch later by name, or open it directly from the - // database connection. + // --8<-- [start:branch_reopen] + // Reopen an existing branch by name from the table handle... const checkedOut = await branches.checkout("exp"); + // ...or open it directly from the database connection. const branchHandle = await db.openTable( "quotes_branches_example", undefined, { branch: "exp" }, ); - console.log(await checkedOut.countRows()); // 4 - console.log(await branchHandle.countRows()); // 4 + console.log(await checkedOut.countRows(), await branchHandle.countRows()); // both 4 + // --8<-- [end:branch_reopen] - // Delete a branch when you're done with it. + // --8<-- [start:branch_delete] + // Delete the branch and its branch-local history. Data on main is safe. await branches.delete("exp"); - // --8<-- [end:branches] + // --8<-- [end:branch_delete] + expect(await table.countRows()).toBe(3); expect(await branches.list()).not.toHaveProperty("exp"); + + // Setup: a branch that diverges from main, ready to promote back. + const promo = await branches.create("promote"); + await promo.update({ where: "id = 1", values: { quote: "Revised on the branch" } }); + await promo.add([{ id: 4, author: "Galahad", quote: "The grail awaits." }]); + + // --8<-- [start:branch_promote] + // There is no built-in merge yet, so promote a branch by writing its rows + // back to main with a normal ingestion call. `mergeInsert` keys on a + // unique column, so rows that already exist on main are updated in place and + // new rows are appended — exactly what an upsert-style ingestion job does. + const promoted = await promo.toArrow(); // or filter down to just the changed rows + await table + .mergeInsert("id") + .whenMatchedUpdateAll() // update rows that already exist on main + .whenNotMatchedInsertAll() // insert rows that are new on the branch + .execute(promoted); + // --8<-- [end:branch_promote] + + expect(await table.countRows()).toBe(4); + await branches.delete("promote"); + + // Setup: a larger table with a vector and a text column to index. + const products = await db.createTable( + "products_branch_index", + Array.from({ length: 512 }, (_, i) => ({ + id: i, + vector: Array.from({ length: 4 }, () => Math.random()), + text: `product number ${i}`, + })), + { mode: "overwrite" }, + ); + const productBranches = await products.branches(); + + // --8<-- [start:branch_index] + // Build and validate indexes on a branch before promoting them to main. + const dev = await productBranches.create("index-dev"); + + // A vector (ANN) index and a full-text search index, both branch-scoped. + await dev.createIndex("vector", { + config: lancedb.Index.ivfPq({ + distanceType: "cosine", + numPartitions: 1, + numSubVectors: 2, + }), + }); + await dev.createIndex("text", { config: lancedb.Index.fts() }); + + // Both indexes live only on the branch; main still has none. + console.log((await dev.listIndices()).map((ix) => ix.name)); // branch: two indexes + console.log((await products.listIndices()).map((ix) => ix.name)); // main: [] (untouched) + // --8<-- [end:branch_index] + + expect(await dev.listIndices()).toHaveLength(2); + expect(await products.listIndices()).toHaveLength(0); + await productBranches.delete("index-dev"); }); }); diff --git a/uv.lock b/uv.lock index 0ad41d5..cc6cedd 100644 --- a/uv.lock +++ b/uv.lock @@ -384,7 +384,7 @@ dependencies = [ requires-dist = [ { name = "geneva", specifier = ">=0.13.0b15" }, { name = "lance-namespace", specifier = ">=0.6.1" }, - { name = "lancedb", specifier = ">=0.33.0" }, + { name = "lancedb", specifier = ">=0.34.0" }, { name = "pandas", specifier = ">=3.0.1" }, { name = "pillow", specifier = ">=12.1.1" }, { name = "polars", specifier = ">=1.39.2" }, @@ -717,8 +717,8 @@ wheels = [ [[package]] name = "lancedb" -version = "0.33.1b2" -source = { registry = "https://pypi.fury.io/lancedb/" } +version = "0.34.0" +source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "deprecation" }, { name = "lance-namespace" }, @@ -729,10 +729,10 @@ dependencies = [ { name = "tqdm" }, ] wheels = [ - { url = "https://pypi.fury.io/lancedb/-/ver_1Nlsr0/lancedb-0.33.1b2-cp39-abi3-macosx_11_0_arm64.whl", hash = "sha256:ec87aee4fcb644ec84c78cd3f7029b3bf0d412f98cf12374f682c68f14265fef" }, - { url = "https://pypi.fury.io/lancedb/-/ver_1JjYh/lancedb-0.33.1b2-cp39-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:da5734098ebf16465e850ea661014fe0992f75234aa5a23f04ec95787bd05570" }, - { url = "https://pypi.fury.io/lancedb/-/ver_1a3Ksg/lancedb-0.33.1b2-cp39-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:22cde3613d74d93aa27b7f426453017538d03a36c6a1ddc6c9bf41944953079c" }, - { url = "https://pypi.fury.io/lancedb/-/ver_1FdgZt/lancedb-0.33.1b2-cp39-abi3-win_amd64.whl", hash = "sha256:db1de7ddb72916acf3ab66bfa1bfc8f32f22230716ef60fc7a88df3bf41a0572" }, + { url = "https://files.pythonhosted.org/packages/df/f7/5262b9aa593f790757163c0165ab0da1dda054758901bea7e4f02c9cb633/lancedb-0.34.0-cp39-abi3-macosx_11_0_arm64.whl", hash = "sha256:c462f2e6f933cad659fd0179394eaab578acbc9151fe2ef41bc29b36ecca5058", size = 52654213, upload-time = "2026-07-02T17:13:31.102Z" }, + { url = "https://files.pythonhosted.org/packages/69/99/05ea0d32229ebea695193ff20c15d6ecae25785ad82a9d4723d98832a284/lancedb-0.34.0-cp39-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:48829e88e708947d0520454ab9e4f8efa35f3e3626469eadd3a6e061b89cb223", size = 55434501, upload-time = "2026-07-02T17:13:34.81Z" }, + { url = "https://files.pythonhosted.org/packages/cd/4e/4325c13d5afa93c466428a5a0f168ad4d96f5eb4a77bbe7c5100d39c9897/lancedb-0.34.0-cp39-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:05ba8a5b58e064edfbe5be71b1abf2e411b4eaf295d1a173dcb1a55c5bfb5285", size = 58659359, upload-time = "2026-07-02T17:13:38.424Z" }, + { url = "https://files.pythonhosted.org/packages/d9/5d/8ca165f1386caf6c4d1c515afd52f345b66432264eecfdfb7fd33eefd9af/lancedb-0.34.0-cp39-abi3-win_amd64.whl", hash = "sha256:51cbc11808f9e3332819b9367c975b3a888541447a8e7bea09c57c852a279153", size = 63530726, upload-time = "2026-07-02T17:13:41.612Z" }, ] [[package]]