[2/3] sdks/python: enrich data with Milvus Search [Vector, Keyword, Hybrid] (#35577)

* sdks/python: mark `milvus` as extra dependency

* sdks/python: mark milvus itests with `require_docker_in_docker`

* .github: trigger postcommit python

* .github: trigger postcommit python

* .github: trigger postcommit python

* sdks/python: fix linting issues

* .github: update beam postcommit python

* sdks/python: fix linting issues

* workflows: trigger postcommit python

* sdks/python: update image requirements

* workflows: trigger postcommit python

Author: mohamedawnallah

.github/trigger_files/beam_PostCommit_Python.json

@@ -1,5 +1,5 @@
 {
   "comment": "Modify this file in a trivial way to cause this test suite to run.",
-  "modification": 101
+  "modification": 35
 }
 

sdks/python/apache_beam/ml/rag/enrichment/milvus_search.py

@@ -27,16 +27,16 @@
 from typing import Union
 
 from google.protobuf.json_format import MessageToDict
+
+from apache_beam.ml.rag.types import Chunk
+from apache_beam.ml.rag.types import Embedding
+from apache_beam.transforms.enrichment import EnrichmentSourceHandler
 from pymilvus import AnnSearchRequest
 from pymilvus import Hit
 from pymilvus import Hits
 from pymilvus import MilvusClient
 from pymilvus import SearchResult
 
-from apache_beam.ml.rag.types import Chunk
-from apache_beam.ml.rag.types import Embedding
-from apache_beam.transforms.enrichment import EnrichmentSourceHandler
-
 
 class SearchStrategy(Enum):
   """Search strategies for information retrieval.

sdks/python/apache_beam/ml/rag/enrichment/milvus_search_it_test.py

@@ -34,18 +34,6 @@
 
 import pytest
 import yaml
-from pymilvus import CollectionSchema
-from pymilvus import DataType
-from pymilvus import FieldSchema
-from pymilvus import Function
-from pymilvus import FunctionType
-from pymilvus import MilvusClient
-from pymilvus import RRFRanker
-from pymilvus.milvus_client import IndexParams
-from testcontainers.core.config import MAX_TRIES as TC_MAX_TRIES
-from testcontainers.core.config import testcontainers_config
-from testcontainers.core.generic import DbContainer
-from testcontainers.milvus import MilvusContainer
 
 import apache_beam as beam
 from apache_beam.ml.rag.types import Chunk
@@ -54,7 +42,21 @@
 from apache_beam.testing.test_pipeline import TestPipeline
 from apache_beam.testing.util import assert_that
 
+# pylint: disable=ungrouped-imports
 try:
+  from pymilvus import (
+      CollectionSchema,
+      DataType,
+      FieldSchema,
+      Function,
+      FunctionType,
+      MilvusClient,
+      RRFRanker)
+  from pymilvus.milvus_client import IndexParams
+  from testcontainers.core.config import MAX_TRIES as TC_MAX_TRIES
+  from testcontainers.core.config import testcontainers_config
+  from testcontainers.core.generic import DbContainer
+  from testcontainers.milvus import MilvusContainer
   from apache_beam.transforms.enrichment import Enrichment
   from apache_beam.ml.rag.enrichment.milvus_search import (
       MilvusSearchEnrichmentHandler,
@@ -467,7 +469,7 @@ def create_user_yaml(service_port: int, max_vector_field_num=5):
         os.remove(path)
 
 
-@pytest.mark.uses_testcontainer
+@pytest.mark.require_docker_in_docker
 @unittest.skipUnless(
     platform.system() == "Linux",
     "Test runs only on Linux due to lack of support, as yet, for nested "
@@ -483,22 +485,16 @@ class TestMilvusSearchEnrichment(unittest.TestCase):
 
   @classmethod
   def setUpClass(cls):
-    try:
-      cls._db = MilvusEnrichmentTestHelper.start_db_container(
-          cls._version, vector_client_max_retries=1, tc_max_retries=1)
-      cls._connection_params = MilvusConnectionParameters(
-          uri=cls._db.uri,
-          user=cls._db.user,
-          password=cls._db.password,
-          db_id=cls._db.id,
-          token=cls._db.token)
-      cls._collection_load_params = MilvusCollectionLoadParameters()
-      cls._collection_name = MilvusEnrichmentTestHelper.initialize_db_with_data(
-          cls._connection_params)
-    except Exception as e:
-      pytest.skip(
-          f"Skipping all tests in {cls.__name__} due to DB startup failure: {e}"
-      )
+    cls._db = MilvusEnrichmentTestHelper.start_db_container(cls._version)
+    cls._connection_params = MilvusConnectionParameters(
+        uri=cls._db.uri,
+        user=cls._db.user,
+        password=cls._db.password,
+        db_id=cls._db.id,
+        token=cls._db.token)
+    cls._collection_load_params = MilvusCollectionLoadParameters()
+    cls._collection_name = MilvusEnrichmentTestHelper.initialize_db_with_data(
+        cls._connection_params)
 
   @classmethod
   def tearDownClass(cls):

sdks/python/container/py310/base_image_requirements.txt

@@ -23,25 +23,25 @@
 
 aiofiles==24.1.0
 aiohappyeyeballs==2.6.1
-aiohttp==3.12.14
+aiohttp==3.12.15
 aiosignal==1.4.0
 annotated-types==0.7.0
-anyio==4.9.0
+anyio==4.10.0
 asn1crypto==1.5.1
 async-timeout==5.0.1
 attrs==25.3.0
 backports.tarfile==1.2.0
 beautifulsoup4==4.13.4
 bs4==0.0.2
-build==1.2.2.post1
+build==1.3.0
 cachetools==5.5.2
-certifi==2025.7.14
+certifi==2025.8.3
 cffi==1.17.1
 charset-normalizer==3.4.2
 click==8.2.1
 cloud-sql-python-connector==1.18.3
 crcmod==1.7
-cryptography==45.0.5
+cryptography==45.0.6
 Cython==3.1.2
 dill==0.3.1.1
 dnspython==2.7.0
@@ -50,26 +50,26 @@ docopt==0.6.2
 docstring_parser==0.17.0
 exceptiongroup==1.3.0
 execnet==2.1.1
-fastavro==1.11.1
+fastavro==1.12.0
 fasteners==0.19
-freezegun==1.5.3
+freezegun==1.5.4
 frozenlist==1.7.0
 future==1.0.0
 google-api-core==2.25.1
-google-api-python-client==2.177.0
+google-api-python-client==2.178.0
 google-apitools==0.5.31
 google-auth==2.40.3
 google-auth-httplib2==0.2.0
-google-cloud-aiplatform==1.105.0
+google-cloud-aiplatform==1.108.0
 google-cloud-bigquery==3.35.1
 google-cloud-bigquery-storage==2.32.0
-google-cloud-bigtable==2.31.0
+google-cloud-bigtable==2.32.0
 google-cloud-core==2.4.3
 google-cloud-datastore==2.21.0
 google-cloud-dlp==3.31.0
 google-cloud-language==2.17.2
 google-cloud-profiler==4.1.0
-google-cloud-pubsub==2.31.0
+google-cloud-pubsub==2.31.1
 google-cloud-pubsublite==1.12.0
 google-cloud-recommendations-ai==0.10.18
 google-cloud-resource-manager==1.14.2
@@ -78,10 +78,10 @@ google-cloud-storage==2.19.0
 google-cloud-videointelligence==2.16.2
 google-cloud-vision==3.10.2
 google-crc32c==1.7.1
-google-genai==1.27.0
+google-genai==1.29.0
 google-resumable-media==2.7.2
 googleapis-common-protos==1.70.0
-greenlet==3.2.3
+greenlet==3.2.4
 grpc-google-iam-v1==0.14.2
 grpc-interceptor==0.15.4
 grpcio==1.65.5
@@ -92,7 +92,7 @@ hdfs==2.7.3
 httpcore==1.0.9
 httplib2==0.22.0
 httpx==0.28.1
-hypothesis==6.136.4
+hypothesis==6.137.1
 idna==3.10
 importlib_metadata==8.7.0
 iniconfig==2.1.0
@@ -108,8 +108,8 @@ jsonschema-specifications==2025.4.1
 keyring==25.6.0
 keyrings.google-artifactregistry-auth==1.1.2
 MarkupSafe==3.0.2
-milvus-lite==2.5.1
-mmh3==5.1.0
+milvus==2.3.5
+mmh3==5.2.0
 mock==5.2.0
 more-itertools==10.7.0
 multidict==6.6.3
@@ -118,17 +118,17 @@ nltk==3.9.1
 numpy==2.2.6
 oauth2client==4.1.3
 objsize==0.7.1
-opentelemetry-api==1.35.0
-opentelemetry-sdk==1.35.0
-opentelemetry-semantic-conventions==0.56b0
-oracledb==3.2.0
+opentelemetry-api==1.36.0
+opentelemetry-sdk==1.36.0
+opentelemetry-semantic-conventions==0.57b0
+oracledb==3.3.0
 orjson==3.11.1
 overrides==7.7.0
 packaging==25.0
 pandas==2.2.3
 parameterized==0.9.0
 pg8000==1.31.4
-pip==25.1.1
+pip==25.2
 pluggy==1.6.0
 propcache==0.3.2
 proto-plus==1.26.1
@@ -144,8 +144,7 @@ pydantic_core==2.33.2
 pydot==1.4.2
 PyHamcrest==2.1.0
 PyJWT==2.10.1
-pymilvus==2.5.14
-pymongo==4.13.2
+pymongo==4.14.0
 PyMySQL==1.1.1
 pyparsing==3.2.3
 pyproject_hooks==1.2.0
@@ -159,10 +158,10 @@ pytz==2025.2
 PyYAML==6.0.2
 redis==5.3.1
 referencing==0.36.2
-regex==2024.11.6
+regex==2025.7.34
 requests==2.32.4
 requests-mock==1.12.1
-rpds-py==0.26.0
+rpds-py==0.27.0
 rsa==4.9.1
 scikit-learn==1.7.1
 scipy==1.15.3
@@ -174,7 +173,7 @@ six==1.17.0
 sniffio==1.3.1
 sortedcontainers==2.4.0
 soupsieve==2.7
-SQLAlchemy==2.0.41
+SQLAlchemy==2.0.42
 sqlalchemy_pytds==1.0.2
 sqlparse==0.5.3
 tenacity==8.5.0
@@ -185,7 +184,6 @@ tqdm==4.67.1
 typing-inspection==0.4.1
 typing_extensions==4.14.1
 tzdata==2025.2
-ujson==5.10.0
 uritemplate==4.2.0
 urllib3==2.5.0
 virtualenv-clone==0.5.7