Merge branch 'main' of https://github.com/weaviate/weaviate-python-client into stefan/readthedocs-update

Author: tsmith023

.github/workflows/main.yaml

@@ -19,11 +19,12 @@ concurrency:
 
 env:
   WEAVIATE_124: 1.24.26
-  WEAVIATE_125: stable-v1.25-58e6503
-  WEAVIATE_126: 1.26.17-c1737a0
-  WEAVIATE_127: 1.27.15
-  WEAVIATE_128: 1.28.10
+  WEAVIATE_125: 1.25.34
+  WEAVIATE_126: 1.26.17
+  WEAVIATE_127: 1.27.14
+  WEAVIATE_128: 1.28.8
   WEAVIATE_129: 1.29.1
+  WEAVIATE_130: 1.30.0-rc.0-c1830a7-amd64
 
 jobs:
   lint-and-format:
@@ -64,11 +65,9 @@ jobs:
           python-version: ${{ matrix.version }}
           cache: 'pip' # caching pip dependencies
       - run: pip install -r requirements-devel.txt
-      - name: Run mypy
-        run: mypy --config-file ./pyproject.toml --warn-unused-ignores --python-version ${{matrix.version}} ${{ matrix.folder }}
       - uses: jakebailey/pyright-action@v2
         with:
-          version: 1.1.347
+          version: 1.1.398
 
   unit-tests:
     name: Run Unit Tests
@@ -130,11 +129,11 @@ jobs:
       fail-fast: false
       matrix:
         versions: [
-          { py: "3.9", weaviate: $WEAVIATE_129},
-          { py: "3.10", weaviate: $WEAVIATE_129},
-          { py: "3.11", weaviate: $WEAVIATE_129},
-          { py: "3.12", weaviate: $WEAVIATE_129},
-          { py: "3.13", weaviate: $WEAVIATE_129}
+          { py: "3.9", weaviate: $WEAVIATE_130},
+          { py: "3.10", weaviate: $WEAVIATE_130},
+          { py: "3.11", weaviate: $WEAVIATE_130},
+          { py: "3.12", weaviate: $WEAVIATE_130},
+          { py: "3.13", weaviate: $WEAVIATE_130}
         ]
         optional_dependencies: [false]
     steps:
@@ -168,7 +167,7 @@ jobs:
         run: pytest -n auto --dist loadgroup -v --cov --cov-report=term-missing --cov=weaviate --cov-report xml:coverage-integration.xml integration
       - name: Run integration tests without auth secrets (for forks)
         if: ${{ github.event.pull_request.head.repo.fork }}
-        run: pytest -v --cov --cov-report=term-missing --cov=weaviate --cov-report xml:coverage-integration.xml integration
+        run: pytest -n auto --dist loadgroup -v --cov --cov-report=term-missing --cov=weaviate --cov-report xml:coverage-integration.xml integration
       - name: Archive code coverage results
         if: matrix.versions.py == '3.10' && (github.ref_name != 'main')
         uses: actions/upload-artifact@v4
@@ -279,7 +278,8 @@ jobs:
           $WEAVIATE_126,
           $WEAVIATE_127,
           $WEAVIATE_128,
-          $WEAVIATE_129
+          $WEAVIATE_129,
+          $WEAVIATE_130
         ]
     steps:
       - name: Checkout

.pre-commit-config.yaml

@@ -36,19 +36,6 @@ repos:
       ]
       files: '^weaviate/collections'
 
-- repo: local
-  hooks:
-    - id: mypy
-      name: mypy
-      entry: ./run-mypy.sh
-      language: python
-      language_version: "3.12"
-      # use require_serial so that script
-      # is only called once per commit
-      require_serial: true
-      # Print the number of files as a sanity-check
-      verbose: true
-
 - repo: local
   hooks:
     - id: pyright
@@ -57,4 +44,4 @@ repos:
       language: node
       pass_filenames: false
       types: [python]
-      additional_dependencies: [pyright@1.1.364]
+      additional_dependencies: [pyright@1.1.398]

ci/docker-compose-cluster.yml

@@ -22,6 +22,7 @@ services:
       CONTEXTIONARY_URL: contextionary:9999
       DEFAULT_VECTORIZER_MODULE: text2vec-contextionary
       ENABLE_MODULES: text2vec-contextionary
+      AUTOSCHEMA_ENABLED: 'false'
 
   weaviate-node-2:
     init: true
@@ -54,6 +55,7 @@ services:
       CONTEXTIONARY_URL: contextionary:9999
       DEFAULT_VECTORIZER_MODULE: text2vec-contextionary
       ENABLE_MODULES: text2vec-contextionary
+      AUTOSCHEMA_ENABLED: 'false'
 
   weaviate-node-3:
     init: true
@@ -86,4 +88,5 @@ services:
       CONTEXTIONARY_URL: contextionary:9999
       DEFAULT_VECTORIZER_MODULE: text2vec-contextionary
       ENABLE_MODULES: text2vec-contextionary
+      AUTOSCHEMA_ENABLED: 'false'
 ...

ci/docker-compose-rbac.yml

@@ -27,4 +27,10 @@ services:
       CLUSTER_DATA_BIND_PORT: "7101"
       RAFT_BOOTSTRAP_EXPECT: "1"
       AUTHORIZATION_ENABLE_RBAC: "true"
+      AUTHENTICATION_DB_USERS_ENABLED: "true"
+      AUTHENTICATION_OIDC_ENABLED: 'true'
+      AUTHENTICATION_OIDC_CLIENT_ID: 'wcs'
+      AUTHENTICATION_OIDC_ISSUER: 'https://auth.wcs.api.weaviate.io/auth/realms/SeMI'
+      AUTHENTICATION_OIDC_USERNAME_CLAIM: 'email'
+      AUTHENTICATION_OIDC_GROUPS_CLAIM: 'groups'
 ...

ci/docker-compose-wcs.yml

@@ -24,10 +24,10 @@ services:
       AUTHENTICATION_OIDC_USERNAME_CLAIM: 'email'
       AUTHENTICATION_OIDC_GROUPS_CLAIM: 'groups'
       AUTHORIZATION_ADMINLIST_ENABLED: 'true'
-      AUTHORIZATION_ADMINLIST_USERS: 'ms_2d0e007e7136de11d5f29fce7a53dae219a51458@existiert.net'
+      AUTHORIZATION_ADMINLIST_USERS: 'oidc-test-user@weaviate.io'
       AUTHENTICATION_OIDC_SCOPES: 'openid,email'
       AUTHENTICATION_APIKEY_ENABLED: 'true'
       AUTHENTICATION_APIKEY_ALLOWED_KEYS: 'my-secret-key'
-      AUTHENTICATION_APIKEY_USERS: 'ms_2d0e007e7136de11d5f29fce7a53dae219a51458@existiert.net'
+      AUTHENTICATION_APIKEY_USERS: 'oidc-test-user@weaviate.io'
       DISABLE_TELEMETRY: 'true'
 ...

docs/changelog.rst

@@ -1,6 +1,40 @@
 Changelog
 =========
 
+Version 4.13.2
+--------------
+This patch version includes:
+    - Fixes a parsing bug when providing URLs with ``GenerativeConfig`` for runtime generate queries
+
+Version 4.13.1
+--------------
+This patch version includes:
+   - Fixes ``generative.near_text()`` to not require ``generative_provider``
+
+Version 4.13.0
+--------------
+This minor version provides a complete top-down refactor of the underlying transport for the sync and async clients.
+    - Introduction of a dependency injection system whereby the sync/async connections are injected into each client independently thereby removing their cross-dependency.
+    - Removal of the ``event_loop.py`` file and all logic surrounding its usage by the sync client.
+    - Refactoring of the batching algorithm to use blocking threads with a sync connection, rather than the event loop sidecar thread.
+
+
+Version 4.12.1
+--------------
+This patch version includes:
+    - Renames ``GenerativeConfig.openai_azure`` to ``GenerativeConfig.azure_openai`` to align with the convention of ``Configure.Generative.azure_openai``
+
+Version 4.12.0
+--------------
+This minor version includes:
+    - Support for new 1.30 features:
+        - Dynamic user management of OIDC and native DB users
+        - Improved multi-dimensional/colBERT vectors
+        - Query-time configuration of generative modules
+        - Tenant-based filtering of data permissions in RBAC
+        - The new ``generative-xai`` module
+
+
 Version 4.11.3
 --------------
 This patch version includes:
@@ -35,7 +69,7 @@ Version 4.11.0
 This minor version includes:
     - Support for new 1.29 features:
         - RBAC is ready for production
-        - Multi-dimensional/COLBert vectors  experimental, breaking changes are expected)
+        - Multi-dimensional/COLBert vectors  experimental, breaking changes are expected
         - Aggregations are using GRPC now
     - Improved exception handling for errors returned by Weaviate
     - Add support for NVIDIA modules:

integration/conftest.py

@@ -131,7 +131,7 @@ def _factory(
         reranker_config: Optional[_RerankerProvider] = None,
     ) -> Collection[Any, Any]:
         try:
-            nonlocal client_fixture, name_fixtures, call_counter
+            nonlocal client_fixture, name_fixtures, call_counter  # noqa: F824
             call_counter += 1
             name_fixture = (
                 _sanitize_collection_name(request.node.fspath.basename + "_" + request.node.name)
@@ -268,7 +268,7 @@ async def _factory(
         reranker_config: Optional[_RerankerProvider] = None,
     ) -> CollectionAsync[Any, Any]:
         try:
-            nonlocal client_fixture, name_fixtures
+            nonlocal client_fixture, name_fixtures  # noqa: F824
             name_fixture = _sanitize_collection_name(request.node.name) + name
             name_fixtures.append(name_fixture)
             client_fixture = await async_client_factory(

integration/test_auth.py

@@ -66,14 +66,14 @@ def test_authentication_client_credentials(
 @pytest.mark.parametrize(
     "name,user,env_variable_name,port,scope,warning",
     [
-        # (  # WCS keycloak times out too often
-        #     "WCS",
-        #     "ms_2d0e007e7136de11d5f29fce7a53dae219a51458@existiert.net",
-        #     "WCS_DUMMY_CI_PW",
-        #     WCS_PORT,
-        #     None,
-        #     False,
-        # ),
+        (  # WCS keycloak times out too often
+            "WCS",
+            "oidc-test-user@weaviate.io",
+            "WCS_DUMMY_CI_PW",
+            WCS_PORT,
+            None,
+            False,
+        ),
         (
             "okta",
             "test@test.de",
@@ -137,6 +137,9 @@ def test_client_with_authentication_with_anon_weaviate() -> None:
     with pytest.warns(UserWarning) as recwarn:
         with weaviate.connect_to_local(auth_credentials=auth) as client:
             client.collections.list_all()
+        if len(recwarn) > 1:
+            for rwarning in recwarn.list:
+                print(rwarning.message)
         assert len(recwarn) == 1
         assert str(recwarn.list[0].message).startswith("Auth001")
 
@@ -168,12 +171,12 @@ def _get_access_token(url: str, user: str, pw: str) -> Dict[str, str]:
 @pytest.mark.parametrize(
     "name,user,env_variable_name,port",
     [
-        # (  # WCS keycloak times out too often
-        #     "WCS",
-        #     "ms_2d0e007e7136de11d5f29fce7a53dae219a51458@existiert.net",
-        #     "WCS_DUMMY_CI_PW",
-        #     WCS_PORT,
-        # ),
+        (  # WCS keycloak times out too often
+            "WCS",
+            "oidc-test-user@weaviate.io",
+            "WCS_DUMMY_CI_PW",
+            WCS_PORT,
+        ),
         (
             "okta",
             "test@test.de",
@@ -200,7 +203,7 @@ def test_authentication_with_bearer_token(
         refresh_token=token["refresh_token"],
     )
     with weaviate.connect_to_local(port=port, auth_credentials=auth) as client:
-        client.collections.list_all()
+        client.collections.exists("something")
 
 
 def test_authentication_with_bearer_token_no_refresh() -> None:

integration/test_backup_v4.py

@@ -72,13 +72,13 @@
         "datePublished": datetime.datetime.now(datetime.timezone.utc).isoformat(),
     },
 ]
+CLASSES = ["Paragraph", "Article"]
 
 
 @pytest.fixture(scope="module")
 def client() -> Generator[weaviate.WeaviateClient, None, None]:
     client = weaviate.connect_to_local()
-    client.collections.delete("Paragraph")
-    client.collections.delete("Article")
+    client.collections.delete(CLASSES)
 
     col_para = client.collections.create(
         name="Paragraph",
@@ -123,7 +123,9 @@ def test_create_and_restore_backup_with_waiting(client: weaviate.WeaviateClient)
 
     # create backup
     classes = ["Article", "Paragraph"]
-    resp = client.backup.create(backup_id=backup_id, backend=BACKEND, wait_for_completion=True)
+    resp = client.backup.create(
+        backup_id=backup_id, backend=BACKEND, wait_for_completion=True, include_collections=CLASSES
+    )
     assert resp.status == BackupStatus.SUCCESS
     for cls in classes:
         assert cls in resp.collections
@@ -141,7 +143,9 @@ def test_create_and_restore_backup_with_waiting(client: weaviate.WeaviateClient)
     client.collections.delete("Paragraph")
 
     # restore backup
-    restore = client.backup.restore(backup_id=backup_id, backend=BACKEND, wait_for_completion=True)
+    restore = client.backup.restore(
+        backup_id=backup_id, backend=BACKEND, wait_for_completion=True, include_collections=CLASSES
+    )
     assert restore.status == BackupStatus.SUCCESS
     for cls in classes:
         assert cls in resp.collections
@@ -349,14 +353,14 @@ def test_fail_creating_backup_for_both_include_and_exclude_classes(
 
 
 @pytest.mark.parametrize("dynamic_backup_location", [False, True])
-def test_backup_and_restore_with_collection(
+def test_backup_and_restore_with_dynamic_location(
     client: weaviate.WeaviateClient, dynamic_backup_location: bool, tmp_path: pathlib.Path
 ) -> None:
     backup_id = _create_backup_id()
 
     conf_create: Optional[wvc.backup.BackupConfigCreate] = None
     conf_restore: Optional[wvc.backup.BackupConfigRestore] = None
-    backup_location: Optional[wvc.backup.BackupLocation] = None
+    backup_location: Optional[wvc.backup.BackupLocationType] = None
     if dynamic_backup_location:
         if client._connection._weaviate_version.is_lower_than(1, 27, 2):
             pytest.skip("Cancel backups is only supported from 1.27.2")
@@ -505,15 +509,18 @@ def test_cancel_backup(
     if client._connection._weaviate_version.is_lower_than(1, 24, 25):
         pytest.skip("Cancel backups is only supported from 1.24.25")
 
-    backup_location: Optional[wvc.backup.BackupLocation] = None
+    backup_location: Optional[wvc.backup.BackupLocationType] = None
     if dynamic_backup_location:
         if client._connection._weaviate_version.is_lower_than(1, 27, 2):
             pytest.skip("Cancel backups is only supported from 1.27.2")
 
         backup_location = wvc.backup.BackupLocation.FileSystem(path=str(tmp_path))
 
     resp = client.backup.create(
-        backup_id=backup_id, backend=BACKEND, backup_location=backup_location
+        backup_id=backup_id,
+        backend=BACKEND,
+        backup_location=backup_location,
+        include_collections=CLASSES,
     )
     assert resp.status == BackupStatus.STARTED
 

integration/test_batch_v4.py

@@ -88,7 +88,7 @@ def client_factory(
     def _factory(
         name: str = "", ports: Tuple[int, int] = (8080, 50051), multi_tenant: bool = False
     ) -> Tuple[weaviate.WeaviateClient, str]:
-        nonlocal client_fixture, name_fixtures
+        nonlocal client_fixture, name_fixtures  # noqa: F824
         name_fixture = _sanitize_collection_name(request.node.name) + name
         name_fixtures.append(name_fixture)
         if client_fixture is None: