Add namespace-scoped collections with IVF vector indexing for agent database

.github/workflows/ci.yml

@@ -95,7 +95,8 @@ jobs:
             log_level=INFO
           fi
           set -o pipefail
-          uv run pytest tests/integration_tests/ -v --log-cli-level=${log_level} -k "not server and not embedded and not oceanbase" | tee pytest.log
+          uv run pytest tests/integration_tests/ -v --log-cli-level=${log_level} \
+            -k "not server and not embedded and not oceanbase" | tee pytest.log
           tail -n 1 pytest.log | grep '=======' | grep 'passed' |grep -q 'failed' && exit 1 || exit 0
 
   integration-test:
@@ -158,5 +159,6 @@ jobs:
             log_level=INFO
           fi
           set -o pipefail
-          uv run pytest tests/integration_tests/ -v --log-cli-level=${log_level} -k "${{ matrix.test_mode }}" | tee pytest.log
+          uv run pytest tests/integration_tests/ -v --log-cli-level=${log_level} \
+            -k "${{ matrix.test_mode }}" | tee pytest.log
           tail -n 1 pytest.log | grep '=======' | grep 'passed' | grep -q 'failed' && exit 1 || exit 0

docs/guide/dql.md

@@ -101,7 +101,7 @@ results = collection.query(
 - `n_results` (int, required): Number of similar results to return (default: 10)
 - `where` (dict, optional): Metadata filter conditions (see Filter Operators section)
 - `where_document` (dict, optional): Document content filter
-- `include` (List[str], optional): List of fields to include: `["documents", "metadatas", "embeddings"]`
+- `include` (List[str], optional): List of fields to include: `["documents", "metadatas", "embeddings", "distances"]` (singular aliases accepted). Invalid field names raise `ValueError` before the query runs.
 
 **Returns:**
 Dict with keys (chromadb-compatible format):
@@ -288,7 +288,8 @@ results = collection.hybrid_search(
 ### Document Filters (`where_document` parameter)
 - `$contains`: full-text match
 - `$not_contains`: exclude matches
-- `$or` / `$and` combining multiple `$contains` clauses
+- `$regex`: regular expression match (namespace `query` applies via SDK post-filter when OB knn filter cannot enforce it)
+- `$or` / `$and` combining multiple document clauses (including nested combinations)
 
 ## 5.5 Collection Information Methods
 

docs/guide/index.md

@@ -9,6 +9,7 @@ This guide covers installation, connections, data modeling, and common operation
 client-connection
 admin-client
 collection-management
+namespace
 dml
 dql
 embedding-functions

docs/guide/namespace.md

@@ -0,0 +1,99 @@
+# Namespace Collections
+
+Namespace collections partition data inside a single physical collection. Each **namespace** is a logical slice (similar to a tenant or dataset) that shares catalog tables and IVF-backed vector indexes, while keeping documents isolated by `namespace_id`.
+
+**Requirements**
+
+- LakeBase **4.6.1.0** or newer (`OceanBase Database AI` in `SELECT version()`)
+- `use_namespace=True` when creating the collection
+- An explicit **IVF** `Schema` (the default non-namespace path builds **HNSW**, which namespace collections do not support)
+
+## 1. Create a namespace-enabled collection
+
+```python
+import pyseekdb
+from pyseekdb import IVFConfiguration, FulltextIndexConfig, VectorIndexConfig, Schema
+
+client = pyseekdb.Client(host="127.0.0.1", port=2881, database="test")
+
+schema = Schema(
+    vector_index=VectorIndexConfig(
+        ivf=IVFConfiguration(dimension=384, distance="cosine", centroids_fresh_mode="spfresh"),
+        embedding_function=None,  # or pass your embedding function
+    ),
+    fulltext_index=FulltextIndexConfig(analyzer="ik"),
+)
+
+collection = client.create_collection(
+    name="products",
+    schema=schema,
+    use_namespace=True,
+    partition_count=4,  # optional; controls table partitioning
+)
+```
+
+If you omit `schema` with `use_namespace=True`, pyseekdb raises a clear error instead of silently building the default HNSW schema.
+
+**Not supported yet**
+
+- Default / HNSW schemas (`configuration=HNSWConfiguration(...)` without an IVF `Schema`)
+- `SparseVectorIndexConfig` on namespace collections
+
+## 2. Manage namespaces
+
+```python
+ns = collection.create_namespace("electronics")
+# or idempotently:
+ns = collection.get_or_create_namespace("electronics")
+
+collection.list_namespaces()   # ["electronics", ...]
+collection.delete_namespace("electronics")
+```
+
+## 3. Add and query data inside a namespace
+
+Namespace DML uses **record ids** — per-document business keys you pass as `ids` in `namespace.add()`, `get()`, `update()`, and `delete()`. They are stored in `data_content.id` (JSON), not as the table primary key.
+
+```python
+ns.add(
+    ids=["doc_1", "doc_2"],
+    embeddings=[[0.1, 0.2, 0.3], [0.4, 0.5, 0.6]],
+    documents=["phone", "laptop"],
+    metadatas=[{"brand": "A"}, {"brand": "B"}],
+)
+
+rows = ns.get(ids=["doc_1"], include=["documents", "metadatas"])
+hits = ns.query(query_embeddings=[[0.1, 0.2, 0.3]], n_results=5)
+```
+
+**Record id rules (namespace path)**
+
+- Must be non-empty strings
+- Currently limited to `[A-Za-z0-9_]` (letters, digits, underscore)
+- Maximum length 512 characters
+
+Standard (non-namespace) collections accept broader `_id` values (for example UUIDs with hyphens). If you need UUID-style ids on the namespace path, normalize them (for example `doc_a1b2c3d4`) or wait for a future relaxation.
+
+## 4. `get_or_create_collection` compatibility
+
+`get_or_create_collection(..., use_namespace=...)` must match how the collection was originally created:
+
+```python
+# OK — reuses the existing namespace collection
+client.get_or_create_collection("products", schema=schema, use_namespace=True)
+
+# Raises ValueError — name already used by a standard HNSW collection
+client.get_or_create_collection("legacy_coll", use_namespace=True)
+```
+
+Delete the old collection or pick a new name when switching between standard and namespace modes.
+
+## 5. Collection-level hybrid search
+
+Namespace collections also expose `collection.hybrid_search(...)` across namespaces. See integration examples under `tests/integration_tests/test_namespace_*.py` for FTS + vector patterns.
+
+## See also
+
+- [Collection management](collection-management.md) — standard HNSW collections
+- [DML operations](dml.md) — `add` / `update` / `delete` on standard collections
+- `examples/namespace_example.py` — minimal runnable sample

examples/namespace_example.py

@@ -0,0 +1,52 @@
+"""
+Namespace collection example (LakeBase 4.6.1.0+).
+
+Demonstrates:
+1. Creating a namespace-enabled collection with an explicit IVF schema
+2. Creating a namespace and adding documents with record ids
+3. Vector query inside the namespace
+"""
+
+import pyseekdb
+from pyseekdb import FulltextIndexConfig, IVFConfiguration, Schema, VectorIndexConfig
+
+# Connect to LakeBase / OceanBase (adjust host/port/credentials)
+client = pyseekdb.Client(
+    host="127.0.0.1",
+    port=2881,
+    database="test",
+    user="root@test",
+    password="",
+)
+
+schema = Schema(
+    vector_index=VectorIndexConfig(
+        ivf=IVFConfiguration(dimension=3, distance="l2", centroids_fresh_mode="spfresh"),
+        embedding_function=None,
+    ),
+    fulltext_index=FulltextIndexConfig(analyzer="ik"),
+)
+
+collection_name = "demo_namespace_collection"
+collection = client.create_collection(
+    name=collection_name,
+    schema=schema,
+    use_namespace=True,
+    partition_count=4,
+)
+
+ns = collection.get_or_create_namespace("demo")
+
+# Record ids are per-document keys inside this namespace (see docs/guide/namespace.md).
+ns.add(
+    ids=["item_alpha", "item_beta"],
+    embeddings=[[1.0, 0.0, 0.0], [0.0, 1.0, 0.0]],
+    documents=["alpha product", "beta product"],
+    metadatas=[{"tier": "a"}, {"tier": "b"}],
+)
+
+results = ns.query(query_embeddings=[[1.0, 0.0, 0.0]], n_results=2, include=["documents", "metadatas"])
+print("Top match:", results["ids"][0][0], results["documents"][0][0])
+
+collection.delete_namespace("demo")
+client.delete_collection(collection_name)

pyproject.toml

@@ -107,8 +107,9 @@ ignore = [
 ]
 
 [tool.ruff.lint.per-file-ignores]
-"tests/*" = ["S101", "S608"]
-"src/pyseekdb/client/client_base.py" = ["S608"]
+"tests/**" = ["S101", "S608", "C901", "S105", "S110", "TRY300", "B017"]
+"src/pyseekdb/client/client_base.py" = ["S608", "C901"]
+"src/pyseekdb/client/document_query_builder.py" = ["C901"]
 
 [tool.ruff.format]
 preview = true

src/pyseekdb/__init__.py

@@ -86,6 +86,9 @@
     HNSWConfiguration,
     IKMode,
     IKProperties,
+    IVFConfiguration,
+    IVFIndexLib,
+    IVFIndexType,
     K,
     Ngram2Properties,
     NgramProperties,
@@ -103,6 +106,7 @@
     register_sparse_embedding_function,
 )
 from .client.collection import Collection
+from .client.namespace import Namespace
 
 try:
     __version__ = importlib.metadata.version("pyseekdb")
@@ -128,7 +132,11 @@
     "HNSWConfiguration",
     "IKMode",
     "IKProperties",
+    "IVFConfiguration",
+    "IVFIndexLib",
+    "IVFIndexType",
     "K",
+    "Namespace",
     "Ngram2Properties",
     "NgramProperties",
     "RemoteServerClient",

src/pyseekdb/client/__init__.py

@@ -31,6 +31,9 @@
     HNSWConfiguration,
     IKMode,
     IKProperties,
+    IVFConfiguration,
+    IVFIndexLib,
+    IVFIndexType,
     Ngram2Properties,
     NgramProperties,
     SpaceProperties,
@@ -68,6 +71,7 @@ def _resolve_password(password: str) -> str:
 
 def _default_seekdb_path() -> str:
     # Keep existing behavior: default to "seekdb.db" under current working directory.
+    """Return the default on-disk path for the embedded SeekDB store."""
     return os.path.abspath("seekdb.db")
 
 
@@ -167,6 +171,9 @@ def __getattr__(name: str) -> Any:
     "HNSWConfiguration",
     "IKMode",
     "IKProperties",
+    "IVFConfiguration",
+    "IVFIndexLib",
+    "IVFIndexType",
     "K",
     "Ngram2Properties",
     "NgramProperties",

src/pyseekdb/client/admin_client.py

@@ -157,6 +157,7 @@ def fork_database(self, source_name: str, destination_name: str, tenant: str = D
         return self._server.fork_database(source_name=source_name, destination_name=destination_name, tenant=tenant)
 
     def __repr__(self):
+        """Return the developer-readable representation."""
         return f"<AdminClient server={self._server}>"
 
     def __enter__(self):
@@ -194,6 +195,7 @@ def create_collection(
         schema: SchemaParam = None,
         configuration: ConfigurationParam = _NOT_PROVIDED,
         embedding_function: EmbeddingFunctionParam = _NOT_PROVIDED,
+        use_namespace: bool = False,
         **kwargs,
     ) -> "Collection":
         """Proxy to server implementation - collection operations only"""
@@ -202,6 +204,7 @@ def create_collection(
             schema=schema,
             configuration=configuration,
             embedding_function=embedding_function,
+            use_namespace=use_namespace,
             **kwargs,
         )
 
@@ -227,6 +230,7 @@ def get_or_create_collection(
         schema: SchemaParam = None,
         configuration: ConfigurationParam = _NOT_PROVIDED,
         embedding_function: EmbeddingFunctionParam = _NOT_PROVIDED,
+        use_namespace: bool = False,
         **kwargs,
     ) -> "Collection":
         """Proxy to server implementation - collection operations only"""
@@ -235,6 +239,7 @@ def get_or_create_collection(
             schema=schema,
             configuration=configuration,
             embedding_function=embedding_function,
+            use_namespace=use_namespace,
             **kwargs,
         )
 
@@ -243,6 +248,7 @@ def count_collection(self) -> int:
         return self._server.count_collection()
 
     def __repr__(self):
+        """Return the developer-readable representation."""
         return f"<Client server={self._server}>"
 
     def __enter__(self):