Merge branch 'main' into djanicek/incremental-backups

Author: DanielJanicek

.github/workflows/main.yaml

@@ -86,7 +86,7 @@ jobs:
     - name: Run integration tests with pytest
       run: |
         pip install pytest-html
-        pytest test/integration/test_integration.py --html=test-report-${{ matrix.version }}.html --self-contained-html
+        pytest test/integration/test_integration.py test/integration/test_data_integration.py test/integration/test_create_data_return_collection.py --html=test-report-${{ matrix.version }}.html --self-contained-html
   integration-auth-tests:
     needs: [unit-tests, get-latest-weaviate-version]
     env:

requirements-dev.txt

@@ -1,4 +1,4 @@
-weaviate-client>=4.20.3
+weaviate-client>=4.20.4
 click==8.1.7
 twine
 pytest

setup.cfg

@@ -37,7 +37,7 @@ classifiers =
 include_package_data = True
 python_requires = >=3.9
 install_requires =
-    weaviate-client>=4.19.0
+    weaviate-client>=4.20.4
     click==8.1.7
     semver>=3.0.2
     numpy>=1.24.0

test/integration/test_create_data_return_collection.py

@@ -0,0 +1,178 @@
+"""Integration tests verifying that DataManager.create_data returns a usable
+Collection object for both single-tenant and multi-tenant collections.
+
+Regression guard for a bug where parallel multi-tenant ingestion returned the
+base collection (without tenant context), causing ``len(collection)`` and
+``batch.wait_for_vector_indexing()`` to fail with:
+
+    "class X has multi-tenancy enabled, but request was without tenant"
+"""
+
+import pytest
+import weaviate
+from weaviate_cli.managers.collection_manager import CollectionManager
+from weaviate_cli.managers.config_manager import ConfigManager
+from weaviate_cli.managers.data_manager import DataManager
+from weaviate_cli.managers.tenant_manager import TenantManager
+
+NUM_OBJECTS = 20
+NUM_TENANTS = 3
+
+
+@pytest.fixture
+def client() -> weaviate.WeaviateClient:
+    config = ConfigManager()
+    return config.get_client()
+
+
+@pytest.fixture
+def collection_manager(client: weaviate.WeaviateClient) -> CollectionManager:
+    return CollectionManager(client)
+
+
+@pytest.fixture
+def data_manager(client: weaviate.WeaviateClient) -> DataManager:
+    return DataManager(client)
+
+
+@pytest.fixture
+def tenant_manager(client: weaviate.WeaviateClient) -> TenantManager:
+    return TenantManager(client)
+
+
+def test_create_data_returns_usable_collection_single_tenant(
+    client: weaviate.WeaviateClient,
+    collection_manager: CollectionManager,
+    data_manager: DataManager,
+):
+    """For a non-MT collection, the returned collection must support len() and
+    batch.wait_for_vector_indexing() without errors."""
+    collection_name = "TestReturnCollSingleTenant"
+
+    try:
+        if client.collections.exists(collection_name):
+            client.collections.delete(collection_name)
+
+        collection_manager.create_collection(
+            collection=collection_name,
+            vectorizer="none",
+            replication_factor=1,
+            async_enabled=True,
+        )
+
+        returned_col = data_manager.create_data(
+            collection=collection_name,
+            limit=NUM_OBJECTS,
+            consistency_level="one",
+            randomize=True,
+            skip_seed=True,
+            vector_dimensions=128,
+        )
+
+        # The returned collection must be usable for these operations
+        returned_col.batch.wait_for_vector_indexing()
+        assert (
+            len(returned_col) == NUM_OBJECTS
+        ), f"Expected {NUM_OBJECTS} objects via returned collection, got {len(returned_col)}"
+    finally:
+        if client.collections.exists(collection_name):
+            client.collections.delete(collection_name)
+
+
+def test_create_data_returns_usable_collection_multitenant_sequential(
+    client: weaviate.WeaviateClient,
+    collection_manager: CollectionManager,
+    data_manager: DataManager,
+    tenant_manager: TenantManager,
+):
+    """For a multi-tenant collection with sequential ingestion (parallel_workers=1),
+    the returned collection must carry tenant context so len() works."""
+    collection_name = "TestReturnCollMTSeq"
+
+    try:
+        if client.collections.exists(collection_name):
+            client.collections.delete(collection_name)
+
+        collection_manager.create_collection(
+            collection=collection_name,
+            vectorizer="none",
+            replication_factor=1,
+            async_enabled=True,
+            multitenant=True,
+        )
+
+        tenant_manager.create_tenants(
+            collection=collection_name,
+            number_tenants=NUM_TENANTS,
+        )
+
+        returned_col = data_manager.create_data(
+            collection=collection_name,
+            limit=NUM_OBJECTS,
+            consistency_level="one",
+            randomize=True,
+            skip_seed=True,
+            vector_dimensions=128,
+            parallel_workers=1,
+        )
+
+        # Must not raise "multi-tenancy enabled, but request was without tenant"
+        returned_col.batch.wait_for_vector_indexing()
+        count = len(returned_col)
+        assert (
+            count == NUM_OBJECTS
+        ), f"Expected {NUM_OBJECTS} objects via returned collection, got {count}"
+    finally:
+        if client.collections.exists(collection_name):
+            client.collections.delete(collection_name)
+
+
+def test_create_data_returns_usable_collection_multitenant_parallel(
+    client: weaviate.WeaviateClient,
+    collection_manager: CollectionManager,
+    data_manager: DataManager,
+    tenant_manager: TenantManager,
+):
+    """For a multi-tenant collection with parallel ingestion (parallel_workers>1),
+    the returned collection must carry tenant context so len() works.
+
+    This is the exact scenario that was broken: parallel mode never assigned the
+    tenant-scoped collection back to the return variable."""
+    collection_name = "TestReturnCollMTParallel"
+
+    try:
+        if client.collections.exists(collection_name):
+            client.collections.delete(collection_name)
+
+        collection_manager.create_collection(
+            collection=collection_name,
+            vectorizer="none",
+            replication_factor=1,
+            async_enabled=True,
+            multitenant=True,
+        )
+
+        tenant_manager.create_tenants(
+            collection=collection_name,
+            number_tenants=NUM_TENANTS,
+        )
+
+        returned_col = data_manager.create_data(
+            collection=collection_name,
+            limit=NUM_OBJECTS,
+            consistency_level="one",
+            randomize=True,
+            skip_seed=True,
+            vector_dimensions=128,
+            parallel_workers=NUM_TENANTS,
+        )
+
+        # Must not raise "multi-tenancy enabled, but request was without tenant"
+        returned_col.batch.wait_for_vector_indexing()
+        count = len(returned_col)
+        assert (
+            count == NUM_OBJECTS
+        ), f"Expected {NUM_OBJECTS} objects via returned collection, got {count}"
+    finally:
+        if client.collections.exists(collection_name):
+            client.collections.delete(collection_name)

test/integration/test_data_integration.py

@@ -93,7 +93,7 @@ def test_data_creation_with_different_configs(
             assert obj.vector is not None
 
             vector_dimensions = {
-                "transformers": 384,
+                "transformers": 256,
                 "contextionary": 300,
             }
 
@@ -193,11 +193,12 @@ def test_data_creation_with_named_vectors(
             named_vector = obj.vector[named_vector_name]
             assert named_vector is not None
 
-            # Check vector dimensions (should be 768 for transformers)
-            assert len(named_vector) == 384
+            # Check vector dimensions (256 for model2vec transformers, 384 for none/custom)
+            expected_dim = 256 if vectorizer == "transformers" else 384
+            assert len(named_vector) == expected_dim
 
             # Verify vector is not all zeros
-            assert not np.allclose(named_vector, np.zeros(384))
+            assert not np.allclose(named_vector, np.zeros(expected_dim))
 
             # Verify vector has finite values
             assert np.all(np.isfinite(named_vector))