Skip to content

Commit d5795ef

Browse files
committed
feat: add Python bindings and convert to multi-language workspace
- Add Python bindings using PyO3 for the vectorless Rust library - Convert monorepo structure from single Rust crate to multi-language workspace - Add pyproject.toml, setup configuration for Python package distribution - Include comprehensive Python documentation and examples in README - Add basic integration tests for Python bindings covering core functionality - Update .gitignore to include Python-specific files and directories - Maintain existing Rust functionality while enabling cross-language usage
1 parent 665ce4d commit d5795ef

198 files changed

Lines changed: 1536 additions & 127 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

.gitignore

Lines changed: 29 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -53,4 +53,32 @@ config.toml
5353
.vectorless.toml
5454

5555
# Test fixtures
56-
test_workspace/
56+
test_workspace/
57+
58+
# Python
59+
__pycache__/
60+
*.py[cod]
61+
*$py.class
62+
*.so
63+
.Python
64+
build/
65+
develop-eggs/
66+
dist/
67+
downloads/
68+
eggs/
69+
.eggs/
70+
lib/
71+
lib64/
72+
parts/
73+
sdist/
74+
var/
75+
wheels/
76+
*.egg-info/
77+
.installed.cfg
78+
*.egg
79+
.pytest_cache/
80+
.mypy_cache/
81+
.ruff_cache/
82+
.venv/
83+
venv/
84+
ENV/

Cargo.toml

Lines changed: 3 additions & 125 deletions
Original file line numberDiff line numberDiff line change
@@ -1,125 +1,3 @@
1-
[package]
2-
name = "vectorless"
3-
version = "0.1.19"
4-
edition = "2024"
5-
authors = ["zTgx <beautifularea@gmail.com>"]
6-
description = "Hierarchical, reasoning-native document intelligence engine"
7-
license = "Apache-2.0"
8-
repository = "https://github.com/vectorlessflow/vectorless"
9-
homepage = "https://vectorless.dev"
10-
documentation = "https://docs.rs/vectorless"
11-
keywords = ["rag", "document", "retrieval", "indexing", "llm"]
12-
categories = ["text-processing", "data-structures", "algorithms"]
13-
readme = "README.md"
14-
exclude = ["samples/", "docs/", ".*"]
15-
16-
[dependencies]
17-
# Async runtime
18-
tokio = { version = "1", features = ["full"] }
19-
async-trait = "0.1"
20-
futures = "0.3"
21-
22-
# Serialization
23-
serde = { version = "1.0", features = ["derive"] }
24-
serde_json = "1.0"
25-
toml = "0.8"
26-
27-
# Error handling
28-
thiserror = "2"
29-
anyhow = { version = "1", optional = true }
30-
31-
# OpenAI-compatible API client
32-
async-openai = { version = "0.34", features = ["chat-completion"] }
33-
34-
# UUID
35-
uuid = { version = "1.10", features = ["v4", "serde"] }
36-
37-
# Time
38-
chrono = { version = "0.4", default-features = false, features = ["serde", "clock"] }
39-
40-
# Logging
41-
tracing = "0.1"
42-
43-
# Rate limiting
44-
governor = "0.6"
45-
nonzero_ext = "0.3"
46-
47-
# Token counting
48-
tiktoken-rs = "0.9"
49-
50-
# Text processing
51-
regex = "1.10"
52-
53-
# Markdown parsing
54-
pulldown-cmark = { version = "0.12", default-features = false, features = ["simd"] }
55-
56-
# Tree data structure
57-
indextree = { version = "4.8.0", features = ["deser"] }
58-
59-
# LRU cache
60-
lru = "0.12"
61-
62-
# Checksum
63-
sha2 = "0.10"
64-
65-
# BLAKE2b hashing for fingerprints
66-
blake2 = "0.10"
67-
base64 = "0.22"
68-
69-
# Synchronization primitives (for memo store)
70-
parking_lot = "0.12"
71-
72-
# Compression
73-
flate2 = "1.0"
74-
75-
# File locking (Unix)
76-
[target.'cfg(unix)'.dependencies]
77-
libc = "0.2"
78-
79-
# PDF processing
80-
pdf-extract = "0.10.0"
81-
lopdf = "0.34"
82-
83-
# DOCX processing
84-
zip = "2.2"
85-
roxmltree = "0.20"
86-
87-
# Random number generation (for sampling)
88-
rand = "0.8"
89-
90-
# BM25 scoring
91-
bm25 = { version = "2.3.2", features = ["parallelism"] }
92-
93-
# HTML parsing
94-
scraper = "0.22"
95-
96-
[dev-dependencies]
97-
tempfile = "3.10"
98-
tokio-test = "0.4"
99-
tracing-subscriber = { version = "0.3", features = ["env-filter"] }
100-
101-
[profile.release]
102-
opt-level = 3
103-
lto = "thin"
104-
codegen-units = 1
105-
strip = true
106-
panic = "abort"
107-
108-
[profile.dev]
109-
opt-level = 0
110-
debug = true
111-
112-
[profile.bench]
113-
inherits = "release"
114-
debug = true
115-
116-
[profile.release.package."*"]
117-
opt-level = 3
118-
119-
[lints.rust]
120-
missing_docs = "warn"
121-
unsafe_code = "warn"
122-
123-
[lints.clippy]
124-
all = "warn"
125-
pedantic = "warn"
1+
[workspace]
2+
members = ["rust", "python"]
3+
resolver = "2"

README.md

Lines changed: 25 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@
33
![Vectorless](docs/design/logo-horizontal.svg)
44

55
[![Crates.io](https://img.shields.io/crates/v/vectorless.svg)](https://crates.io/crates/vectorless)
6+
[![PyPI](https://img.shields.io/pypi/v/vectorless.svg)](https://pypi.org/project/vectorless/)
67
[![Downloads](https://img.shields.io/crates/d/vectorless.svg)](https://crates.io/crates/vectorless)
78
[![Documentation](https://docs.rs/vectorless/badge.svg)](https://docs.rs/vectorless)
89
[![License](https://img.shields.io/badge/license-Apache--2.0-blue.svg)](LICENSE)
@@ -90,7 +91,30 @@ Source: Chapter 4 > Section 4.2 > Reset Procedure
9091

9192
## Quick Start
9293

93-
### Installation
94+
### Python
95+
96+
```bash
97+
pip install vectorless
98+
```
99+
100+
```python
101+
from vectorless import Engine, IndexContext
102+
103+
# Create engine (uses OPENAI_API_KEY env var)
104+
engine = Engine(workspace="./data")
105+
106+
# Index a document
107+
ctx = IndexContext.from_file("./report.pdf")
108+
doc_id = engine.index(ctx)
109+
110+
# Query
111+
result = engine.query(doc_id, "What is the total revenue?")
112+
print(f"Answer: {result.content}")
113+
```
114+
115+
See [python/README.md](python/README.md) for full Python documentation.
116+
117+
### Rust
94118

95119
```toml
96120
[dependencies]

0 commit comments

Comments
 (0)