diff --git a/.github/workflows/python-test.yml b/.github/workflows/python-test.yml index dc5a385..606f51c 100644 --- a/.github/workflows/python-test.yml +++ b/.github/workflows/python-test.yml @@ -6,7 +6,7 @@ on: pull_request: paths: - python/** - - rust/lance-context/** + - crates/** - .github/workflows/python-test.yml concurrency: diff --git a/.github/workflows/rust-test.yml b/.github/workflows/rust-test.yml index 6d6b85a..87d70d8 100644 --- a/.github/workflows/rust-test.yml +++ b/.github/workflows/rust-test.yml @@ -5,7 +5,7 @@ on: - main pull_request: paths: - - rust/** + - crates/** - .github/workflows/rust-test.yml - Cargo.toml - Cargo.lock @@ -51,8 +51,8 @@ jobs: sudo apt update sudo apt install -y protobuf-compiler - name: Build tests - run: cargo test --manifest-path rust/lance-context/Cargo.toml --no-run + run: cargo test -p lance-context-core --no-run - name: Run unit tests - run: cargo test --manifest-path rust/lance-context/Cargo.toml --lib + run: cargo test -p lance-context-core --lib - name: Run doc tests - run: cargo test --manifest-path rust/lance-context/Cargo.toml --doc + run: cargo test -p lance-context-core --doc diff --git a/.github/workflows/style.yml b/.github/workflows/style.yml index 0f20754..8918a76 100644 --- a/.github/workflows/style.yml +++ b/.github/workflows/style.yml @@ -5,10 +5,10 @@ on: - main pull_request: paths: - - rust/** + - crates/** + - Cargo.toml + - Cargo.lock - .github/workflows/style.yml - - rust/lance-context/Cargo.toml - - rust/lance-context/Cargo.lock concurrency: group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} @@ -28,7 +28,7 @@ jobs: with: components: rustfmt - name: Check formatting - run: cargo fmt --manifest-path rust/lance-context/Cargo.toml -- --check + run: cargo fmt --all -- --check clippy: runs-on: ubuntu-24.04 @@ -37,13 +37,13 @@ jobs: - uses: actions/checkout@v4 - uses: Swatinem/rust-cache@v2 with: - workspaces: rust/lance-context + workspaces: . - name: Install dependencies run: | sudo apt update sudo apt install -y protobuf-compiler - name: Clippy - run: cargo clippy --manifest-path rust/lance-context/Cargo.toml --all-targets -- -D warnings + run: cargo clippy --workspace --all-targets -- -D warnings typos: name: Spell Check diff --git a/rust/lance-context/Cargo.lock b/Cargo.lock similarity index 98% rename from rust/lance-context/Cargo.lock rename to Cargo.lock index ace5cb2..8690936 100644 --- a/rust/lance-context/Cargo.lock +++ b/Cargo.lock @@ -3121,6 +3121,15 @@ dependencies = [ "serde_core", ] +[[package]] +name = "indoc" +version = "2.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "79cf5c93f93228cf8efb3ba362535fb11199ac548a09ce117c9b1adc3030d706" +dependencies = [ + "rustversion", +] + [[package]] name = "inout" version = "0.1.4" @@ -3383,6 +3392,13 @@ dependencies = [ [[package]] name = "lance-context" version = "0.1.0" +dependencies = [ + "lance-context-core", +] + +[[package]] +name = "lance-context-core" +version = "0.1.0" dependencies = [ "arrow-array", "arrow-ipc", @@ -3398,6 +3414,16 @@ dependencies = [ "tokio", ] +[[package]] +name = "lance-context-python" +version = "0.1.0" +dependencies = [ + "chrono", + "lance-context", + "pyo3", + "tokio", +] + [[package]] name = "lance-core" version = "1.0.1" @@ -4141,6 +4167,15 @@ dependencies = [ "libc", ] +[[package]] +name = "memoffset" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "488016bfae457b036d996092f6cb448677611ce4449e970ceaf42695203f218a" +dependencies = [ + "autocfg", +] + [[package]] name = "mime" version = "0.3.17" @@ -4431,7 +4466,7 @@ dependencies = [ "serde", "serde_json", "serde_urlencoded", - "thiserror 2.0.17", + "thiserror 2.0.18", "tokio", "tracing", "url", @@ -4923,6 +4958,68 @@ dependencies = [ "cc", ] +[[package]] +name = "pyo3" +version = "0.25.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8970a78afe0628a3e3430376fc5fd76b6b45c4d43360ffd6cdd40bdde72b682a" +dependencies = [ + "indoc", + "libc", + "memoffset", + "once_cell", + "portable-atomic", + "pyo3-build-config", + "pyo3-ffi", + "pyo3-macros", + "unindent", +] + +[[package]] +name = "pyo3-build-config" +version = "0.25.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "458eb0c55e7ece017adeba38f2248ff3ac615e53660d7c71a238d7d2a01c7598" +dependencies = [ + "once_cell", + "target-lexicon", +] + +[[package]] +name = "pyo3-ffi" +version = "0.25.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7114fe5457c61b276ab77c5055f206295b812608083644a5c5b2640c3102565c" +dependencies = [ + "libc", + "pyo3-build-config", +] + +[[package]] +name = "pyo3-macros" +version = "0.25.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a8725c0a622b374d6cb051d11a0983786448f7785336139c3c94f5aa6bef7e50" +dependencies = [ + "proc-macro2", + "pyo3-macros-backend", + "quote", + "syn 2.0.114", +] + +[[package]] +name = "pyo3-macros-backend" +version = "0.25.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4109984c22491085343c05b0dbc54ddc405c3cf7b4374fc533f5c3313a572ccc" +dependencies = [ + "heck", + "proc-macro2", + "pyo3-build-config", + "quote", + "syn 2.0.114", +] + [[package]] name = "quick-xml" version = "0.37.5" @@ -4957,7 +5054,7 @@ dependencies = [ "rustc-hash", "rustls 0.23.36", "socket2 0.6.1", - "thiserror 2.0.17", + "thiserror 2.0.18", "tokio", "tracing", "web-time", @@ -4978,7 +5075,7 @@ dependencies = [ "rustls 0.23.36", "rustls-pki-types", "slab", - "thiserror 2.0.17", + "thiserror 2.0.18", "tinyvec", "tracing", "web-time", @@ -5189,7 +5286,7 @@ checksum = "a4e608c6638b9c18977b00b475ac1f28d14e84b27d8d42f70e0bf1e3dec127ac" dependencies = [ "getrandom 0.2.17", "libredox", - "thiserror 2.0.17", + "thiserror 2.0.18", ] [[package]] @@ -5842,7 +5939,7 @@ checksum = "297f631f50729c8c99b84667867963997ec0b50f32b2a7dbcab828ef0541e8bb" dependencies = [ "num-bigint", "num-traits", - "thiserror 2.0.17", + "thiserror 2.0.18", "time", ] @@ -6129,7 +6226,7 @@ dependencies = [ "tantivy-stacker", "tantivy-tokenizer-api", "tempfile", - "thiserror 2.0.17", + "thiserror 2.0.18", "time", "uuid", "winapi", @@ -6235,6 +6332,12 @@ version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "55937e1799185b12863d447f42597ed69d9928686b8d88a1df17376a097d8369" +[[package]] +name = "target-lexicon" +version = "0.13.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b1dd07eb858a2067e2f3c7155d54e929265c264e6f37efe3ee7a8d1b5a1dd0ba" + [[package]] name = "tempfile" version = "3.24.0" @@ -6259,11 +6362,11 @@ dependencies = [ [[package]] name = "thiserror" -version = "2.0.17" +version = "2.0.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f63587ca0f12b72a0600bcba1d40081f830876000bb46dd2337a3051618f4fc8" +checksum = "4288b5bcbc7920c07a1149a35cf9590a2aa808e0bc1eafaade0b80947865fbc4" dependencies = [ - "thiserror-impl 2.0.17", + "thiserror-impl 2.0.18", ] [[package]] @@ -6279,9 +6382,9 @@ dependencies = [ [[package]] name = "thiserror-impl" -version = "2.0.17" +version = "2.0.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3ff15c8ecd7de3849db632e14d18d2571fa09dfc5ed93479bc4485c7a517c913" +checksum = "ebc4ee7f67670e9b64d05fa4253e753e016c6c95ff35b89b7941d6b856dec1d5" dependencies = [ "proc-macro2", "quote", @@ -6640,6 +6743,12 @@ version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b4ac048d71ede7ee76d585517add45da530660ef4390e49b098733c6e897f254" +[[package]] +name = "unindent" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7264e107f553ccae879d21fbea1d6724ac785e8c3bfc762137959b5802826ef3" + [[package]] name = "untrusted" version = "0.9.0" diff --git a/Cargo.toml b/Cargo.toml new file mode 100644 index 0000000..6c5fe2e --- /dev/null +++ b/Cargo.toml @@ -0,0 +1,14 @@ +[workspace] +members = [ + "crates/lance-context-core", + "crates/lance-context", + "python", +] +resolver = "2" + +[workspace.package] +edition = "2021" +license = "Apache-2.0" + +[workspace.metadata] +# reserved for future use diff --git a/README.md b/README.md index bad65f8..7de4f2e 100644 --- a/README.md +++ b/README.md @@ -23,9 +23,10 @@ Key motivations inspired by the broader Lance roadmap[1](https://github.com ## Project layout ``` -rust/lance-context # Core Rust crate (ContextStore, schema, serialization) -python/ # Python bindings, wheel build, and pytest suite -python/tests/ # High-level integration tests +crates/lance-context-core # Pure Rust context engine (no Python deps) +crates/lance-context # Re-export crate consumed by downstream clients/bindings +python/ # PyO3 bindings, wheel build, and pytest suite +python/tests/ # High-level integration tests ``` ## Getting started @@ -99,7 +100,7 @@ println!("Current version {}", store.version()); ## Testing - `make test` – Python pytest suite (including persistence integration tests). -- `cargo test --manifest-path rust/lance-context/Cargo.toml` – Rust unit tests. +- `cargo test --manifest-path crates/lance-context-core/Cargo.toml` – Rust unit tests. - `python/.venv/bin/ruff check python/` and `python/.venv/bin/pyright` – linting/type checks. ## Roadmap @@ -121,7 +122,7 @@ Contributions are welcome—feel free to comment on the issues above or open you make venv # creates python/.venv using uv make install # installs the package in editable mode with test extras make test # runs pytest (python/tests/) - cargo test --manifest-path rust/lance-context/Cargo.toml + cargo test --manifest-path crates/lance-context-core/Cargo.toml ``` 4. Run linting/type checks: `python/.venv/bin/ruff check python/`, `python/.venv/bin/pyright`, and `~/.cargo/bin/cargo fmt -- --check`. 5. Open a Pull Request with a clear summary of the change. diff --git a/rust/lance-context/Cargo.toml b/crates/lance-context-core/Cargo.toml similarity index 93% rename from rust/lance-context/Cargo.toml rename to crates/lance-context-core/Cargo.toml index 67a5752..0ec3e05 100644 --- a/rust/lance-context/Cargo.toml +++ b/crates/lance-context-core/Cargo.toml @@ -1,11 +1,11 @@ [package] -name = "lance-context" +name = "lance-context-core" version = "0.1.0" edition = "2021" license = "Apache-2.0" authors = ["Lance Devs "] repository = "https://github.com/lancedb/lance-context" -readme = "README.md" +readme = "../../README.md" description = "Multimodal, versioned context storage for agentic workflows" keywords = ["context", "multimodal", "lance", "agents", "storage"] categories = ["database", "data-structures", "science"] diff --git a/crates/lance-context-core/README.md b/crates/lance-context-core/README.md new file mode 100644 index 0000000..2cf1f1a --- /dev/null +++ b/crates/lance-context-core/README.md @@ -0,0 +1,4 @@ +# lance-context-core + +Pure Rust engine for the lance-context project. This crate is free of Python +dependencies and is re-exported by the `lance-context` wrapper crate. diff --git a/rust/lance-context/src/context.rs b/crates/lance-context-core/src/context.rs similarity index 100% rename from rust/lance-context/src/context.rs rename to crates/lance-context-core/src/context.rs diff --git a/rust/lance-context/src/lib.rs b/crates/lance-context-core/src/lib.rs similarity index 100% rename from rust/lance-context/src/lib.rs rename to crates/lance-context-core/src/lib.rs diff --git a/rust/lance-context/src/record.rs b/crates/lance-context-core/src/record.rs similarity index 100% rename from rust/lance-context/src/record.rs rename to crates/lance-context-core/src/record.rs diff --git a/rust/lance-context/src/serde.rs b/crates/lance-context-core/src/serde.rs similarity index 100% rename from rust/lance-context/src/serde.rs rename to crates/lance-context-core/src/serde.rs diff --git a/rust/lance-context/src/store.rs b/crates/lance-context-core/src/store.rs similarity index 100% rename from rust/lance-context/src/store.rs rename to crates/lance-context-core/src/store.rs diff --git a/crates/lance-context/Cargo.toml b/crates/lance-context/Cargo.toml new file mode 100644 index 0000000..5f29746 --- /dev/null +++ b/crates/lance-context/Cargo.toml @@ -0,0 +1,12 @@ +[package] +name = "lance-context" +version = "0.1.0" +edition = "2021" +license = "Apache-2.0" +authors = ["Lance Devs "] +repository = "https://github.com/lancedb/lance-context" +description = "Public re-export crate for lance-context bindings" +readme = "../../README.md" + +[dependencies] +lance-context-core = { path = "../lance-context-core" } diff --git a/crates/lance-context/src/lib.rs b/crates/lance-context/src/lib.rs new file mode 100644 index 0000000..1264030 --- /dev/null +++ b/crates/lance-context/src/lib.rs @@ -0,0 +1 @@ +pub use lance_context_core::*; diff --git a/python/Cargo.toml b/python/Cargo.toml index a785741..642b242 100644 --- a/python/Cargo.toml +++ b/python/Cargo.toml @@ -11,6 +11,6 @@ crate-type = ["cdylib"] [dependencies] chrono = { version = "0.4", default-features = false, features = ["clock"] } -lance-context = { path = "../rust/lance-context" } +lance-context = { path = "../crates/lance-context" } pyo3 = { version = "0.25", features = ["extension-module", "abi3-py39", "py-clone"] } tokio = { version = "1", features = ["rt-multi-thread"] } diff --git a/python/src/lib.rs b/python/src/lib.rs index befdd70..d4e37b9 100644 --- a/python/src/lib.rs +++ b/python/src/lib.rs @@ -184,7 +184,7 @@ fn search_hit_to_py(py: Python<'_>, hit: SearchResult) -> PyResult { state_dict.set_item("custom", metadata.custom)?; state_dict.into_pyobject(py)?.unbind().into() } - None => py.None().into_pyobject(py)?.unbind().into(), + None => py.None().into_pyobject(py)?.unbind(), }; dict.set_item("state_metadata", state_obj)?; dict.set_item("content_type", content_type)?; diff --git a/rust/lance-context/README.md b/rust/lance-context/README.md deleted file mode 100644 index 1e417a3..0000000 --- a/rust/lance-context/README.md +++ /dev/null @@ -1,4 +0,0 @@ -# lance-context (rust) - -Core Rust crate for the lance-context project. -