Skip to content

Commit 5447d64

Browse files
moawnallahmoawnallah
authored
[bugFix] sdks/python: properly add Milvus as extra dependency (#35920)
* sdks/python: properly make milvus as extra dependency * sdks/python: update image requirements * .github: trigger postcommit python * sdks/python: fix linting issues * sdks/python: fix formatting issues * .github: trigger beam postcommit python * sdks/python: revert milvus version in itests * sdks/python: update image requirements * trigger_files: trigger postcommit python * workflows: capture DinD tests in PreCommit Py Coverage workflow * workflows: temporarily removing `ubuntu-latest` till resolving deps * workflows: add `matrix.os` label to `beam_PreCommit_Python_Coverage`
1 parent 02a532b commit 5447d64

14 files changed

Lines changed: 147 additions & 129 deletions

‎.github/trigger_files/beam_PostCommit_Python.json‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
{
22
"comment": "Modify this file in a trivial way to cause this test suite to run.",
3-
"modification": 33
3+
"modification": 27
44
}
55

‎.github/workflows/beam_PreCommit_Python_Coverage.yml‎

Lines changed: 27 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -58,35 +58,45 @@ env:
5858

5959
jobs:
6060
beam_PreCommit_Python_Coverage:
61-
name: ${{ matrix.job_name }} (${{ matrix.job_phrase }})
62-
runs-on: [self-hosted, ubuntu-20.04, highmem]
61+
name: ${{ matrix.job_name }} (${{ matrix.job_phrase }} ${{ matrix.python_version }}) (${{ join(matrix.os, ', ') }})
62+
runs-on: ${{ matrix.os }}
6363
strategy:
64+
fail-fast: false
6465
matrix:
6566
job_name: [beam_PreCommit_Python_Coverage]
6667
job_phrase: [Run Python_Coverage PreCommit]
68+
python_version: ['3.9']
69+
# Run on both self-hosted and GitHub-hosted runners.
70+
# Some tests (marked require_docker_in_docker) can't run on Beam's
71+
# self-hosted runners due to Docker-in-Docker environment constraint.
72+
# These tests will only execute on ubuntu-latest (GitHub-hosted).
73+
# Context: https://github.com/apache/beam/pull/35585
74+
# Temporary removed the ubuntu-latest env till resolving deps issues.
75+
os: [[self-hosted, ubuntu-20.04, highmem]]
6776
timeout-minutes: 180
6877
if: |
6978
github.event_name == 'push' ||
7079
github.event_name == 'pull_request_target' ||
7180
(github.event_name == 'schedule' && github.repository == 'apache/beam') ||
7281
github.event_name == 'workflow_dispatch' ||
73-
github.event.comment.body == 'Run Python_Coverage PreCommit'
82+
startswith(github.event.comment.body, 'Run Python_Coverage PreCommit 3.')
7483
steps:
7584
- uses: actions/checkout@v4
7685
- name: Setup repository
7786
uses: ./.github/actions/setup-action
7887
with:
79-
comment_phrase: ${{ matrix.job_phrase }}
88+
comment_phrase: ${{ matrix.job_phrase }} ${{ matrix.python_version }}
8089
github_token: ${{ secrets.GITHUB_TOKEN }}
81-
github_job: ${{ matrix.job_name }} (${{ matrix.job_phrase }})
90+
github_job: ${{ matrix.job_name }} (${{ matrix.job_phrase }} ${{ matrix.python_version }}) (${{ join(matrix.os, ', ') }})
8291
- name: Setup environment
8392
uses: ./.github/actions/setup-environment-action
8493
with:
8594
java-version: default
86-
python-version: default
95+
python-version: ${{ matrix.python_version }}
8796
- name: Start DinD
8897
uses: ./.github/actions/dind-up-action
8998
id: dind
99+
if: contains(matrix.os, 'self-hosted')
90100
with:
91101
# Enable all the new features
92102
cleanup-dind-on-start: "true"
@@ -97,9 +107,9 @@ jobs:
97107
export-gh-env: "true"
98108
- name: Run preCommitPyCoverage
99109
env:
100-
DOCKER_HOST: ${{ steps.dind.outputs.docker-host }}
110+
DOCKER_HOST: ${{ contains(matrix.os, 'self-hosted') && steps.dind.outputs.docker-host || '' }}
101111
TOX_TESTENV_PASSENV: "DOCKER_*,TESTCONTAINERS_*,TC_*,BEAM_*,GRPC_*,OMP_*,OPENBLAS_*,PYTHONHASHSEED,PYTEST_*"
102-
TESTCONTAINERS_HOST_OVERRIDE: ${{ env.DIND_IP }}
112+
TESTCONTAINERS_HOST_OVERRIDE: ${{ contains(matrix.os, 'self-hosted') && env.DIND_IP || '' }}
103113
TESTCONTAINERS_DOCKER_SOCKET_OVERRIDE: "/var/run/docker.sock"
104114
TESTCONTAINERS_RYUK_DISABLED: "false"
105115
TESTCONTAINERS_RYUK_CONTAINER_PRIVILEGED: "true"
@@ -110,6 +120,12 @@ jobs:
110120
uses: ./.github/actions/gradle-command-self-hosted-action
111121
with:
112122
gradle-command: :sdks:python:test-suites:tox:py39:preCommitPyCoverage
123+
arguments: |
124+
-Pposargs="${{
125+
contains(matrix.os, 'self-hosted') &&
126+
'-m (not require_docker_in_docker)' ||
127+
'-m require_docker_in_docker'
128+
}}"
113129
- uses: codecov/codecov-action@v3
114130
with:
115131
flags: python
@@ -118,7 +134,7 @@ jobs:
118134
uses: actions/upload-artifact@v4
119135
if: failure()
120136
with:
121-
name: Python Test Results
137+
name: Python ${{ matrix.python_version }} Test Results (${{ join(matrix.os, ', ') }})
122138
path: '**/pytest*.xml'
123139
- name: Publish Python Test Results
124140
env:
@@ -129,4 +145,5 @@ jobs:
129145
commit: '${{ env.prsha || env.GITHUB_SHA }}'
130146
comment_mode: ${{ github.event_name == 'issue_comment' && 'always' || 'off' }}
131147
files: '**/pytest*.xml'
132-
large_files: true
148+
large_files: true
149+
check_name: "Python ${{ matrix.python_version }} Test Results (${{ join(matrix.os, ', ') }})"

‎sdks/python/apache_beam/ml/rag/enrichment/milvus_search.py‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -27,16 +27,16 @@
2727
from typing import Union
2828

2929
from google.protobuf.json_format import MessageToDict
30-
31-
from apache_beam.ml.rag.types import Chunk
32-
from apache_beam.ml.rag.types import Embedding
33-
from apache_beam.transforms.enrichment import EnrichmentSourceHandler
3430
from pymilvus import AnnSearchRequest
3531
from pymilvus import Hit
3632
from pymilvus import Hits
3733
from pymilvus import MilvusClient
3834
from pymilvus import SearchResult
3935

36+
from apache_beam.ml.rag.types import Chunk
37+
from apache_beam.ml.rag.types import Embedding
38+
from apache_beam.transforms.enrichment import EnrichmentSourceHandler
39+
4040

4141
class SearchStrategy(Enum):
4242
"""Search strategies for information retrieval.

‎sdks/python/apache_beam/ml/rag/enrichment/milvus_search_it_test.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -481,7 +481,7 @@ class TestMilvusSearchEnrichment(unittest.TestCase):
481481
"""Tests for search functionality across all search strategies"""
482482

483483
_db: MilvusDBContainerInfo
484-
_version = "milvusdb/milvus:v2.3.9"
484+
_version = "milvusdb/milvus:v2.5.10"
485485

486486
@classmethod
487487
def setUpClass(cls):

‎sdks/python/container/ml/py310/base_image_requirements.txt‎

Lines changed: 14 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -67,8 +67,8 @@ google-api-python-client==2.179.0
6767
google-apitools==0.5.31
6868
google-auth==2.40.3
6969
google-auth-httplib2==0.2.0
70-
google-cloud-aiplatform==1.109.0
71-
google-cloud-bigquery==3.35.1
70+
google-cloud-aiplatform==1.110.0
71+
google-cloud-bigquery==3.36.0
7272
google-cloud-bigquery-storage==2.32.0
7373
google-cloud-bigtable==2.32.0
7474
google-cloud-core==2.4.3
@@ -85,7 +85,7 @@ google-cloud-storage==2.19.0
8585
google-cloud-videointelligence==2.16.2
8686
google-cloud-vision==3.10.2
8787
google-crc32c==1.7.1
88-
google-genai==1.30.0
88+
google-genai==1.31.0
8989
google-pasta==0.2.0
9090
google-resumable-media==2.7.2
9191
googleapis-common-protos==1.70.0
@@ -98,35 +98,33 @@ guppy3==3.1.5
9898
h11==0.16.0
9999
h5py==3.14.0
100100
hdfs==2.7.3
101-
hf-xet==1.1.7
101+
hf-xet==1.1.8
102102
httpcore==1.0.9
103103
httplib2==0.22.0
104104
httpx==0.28.1
105105
huggingface-hub==0.34.4
106-
hypothesis==6.138.2
106+
hypothesis==6.138.3
107107
idna==3.10
108108
importlib_metadata==8.7.0
109109
iniconfig==2.1.0
110110
jaraco.classes==3.4.0
111111
jaraco.context==6.0.1
112-
jaraco.functools==4.2.1
112+
jaraco.functools==4.3.0
113113
jeepney==0.9.0
114114
Jinja2==3.1.6
115115
joblib==1.5.1
116116
jsonpickle==3.4.2
117-
jsonschema==4.25.0
117+
jsonschema==4.25.1
118118
jsonschema-specifications==2025.4.1
119-
keras==3.11.2
119+
keras==3.11.3
120120
keyring==25.6.0
121121
keyrings.google-artifactregistry-auth==1.1.2
122122
libclang==18.1.1
123123
Markdown==3.8.2
124124
markdown-it-py==4.0.0
125125
MarkupSafe==3.0.2
126126
mdurl==0.1.2
127-
ujson==5.8.0
128127
milvus-lite==2.5.1
129-
pymilvus==2.5.10
130128
ml-dtypes==0.3.2
131129
mmh3==5.2.0
132130
mock==5.2.0
@@ -182,8 +180,9 @@ pydot==1.4.2
182180
Pygments==2.19.2
183181
PyHamcrest==2.1.0
184182
PyJWT==2.10.1
185-
pymongo==4.14.0
186-
PyMySQL==1.1.1
183+
pymilvus==2.5.15
184+
pymongo==4.14.1
185+
PyMySQL==1.1.2
187186
pyparsing==3.2.3
188187
pyproject_hooks==1.2.0
189188
pytest==7.4.4
@@ -197,7 +196,7 @@ PyYAML==6.0.2
197196
redis==5.3.1
198197
referencing==0.36.2
199198
regex==2025.7.34
200-
requests==2.32.4
199+
requests==2.32.5
201200
requests-mock==1.12.1
202201
rich==14.1.0
203202
rpds-py==0.27.0
@@ -230,11 +229,12 @@ tokenizers==0.21.4
230229
tomli==2.2.1
231230
torch==2.7.1
232231
tqdm==4.67.1
233-
transformers==4.55.2
232+
transformers==4.55.4
234233
triton==3.3.1
235234
typing-inspection==0.4.1
236235
typing_extensions==4.14.1
237236
tzdata==2025.2
237+
ujson==5.11.0
238238
uritemplate==4.2.0
239239
urllib3==2.5.0
240240
virtualenv-clone==0.5.7

‎sdks/python/container/ml/py311/base_image_requirements.txt‎

Lines changed: 14 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -65,8 +65,8 @@ google-api-python-client==2.179.0
6565
google-apitools==0.5.31
6666
google-auth==2.40.3
6767
google-auth-httplib2==0.2.0
68-
google-cloud-aiplatform==1.109.0
69-
google-cloud-bigquery==3.35.1
68+
google-cloud-aiplatform==1.110.0
69+
google-cloud-bigquery==3.36.0
7070
google-cloud-bigquery-storage==2.32.0
7171
google-cloud-bigtable==2.32.0
7272
google-cloud-core==2.4.3
@@ -83,7 +83,7 @@ google-cloud-storage==2.19.0
8383
google-cloud-videointelligence==2.16.2
8484
google-cloud-vision==3.10.2
8585
google-crc32c==1.7.1
86-
google-genai==1.30.0
86+
google-genai==1.31.0
8787
google-pasta==0.2.0
8888
google-resumable-media==2.7.2
8989
googleapis-common-protos==1.70.0
@@ -96,35 +96,33 @@ guppy3==3.1.5
9696
h11==0.16.0
9797
h5py==3.14.0
9898
hdfs==2.7.3
99-
hf-xet==1.1.7
99+
hf-xet==1.1.8
100100
httpcore==1.0.9
101101
httplib2==0.22.0
102102
httpx==0.28.1
103103
huggingface-hub==0.34.4
104-
hypothesis==6.138.2
104+
hypothesis==6.138.3
105105
idna==3.10
106106
importlib_metadata==8.7.0
107107
iniconfig==2.1.0
108108
jaraco.classes==3.4.0
109109
jaraco.context==6.0.1
110-
jaraco.functools==4.2.1
110+
jaraco.functools==4.3.0
111111
jeepney==0.9.0
112112
Jinja2==3.1.6
113113
joblib==1.5.1
114114
jsonpickle==3.4.2
115-
jsonschema==4.25.0
115+
jsonschema==4.25.1
116116
jsonschema-specifications==2025.4.1
117-
keras==3.11.2
117+
keras==3.11.3
118118
keyring==25.6.0
119119
keyrings.google-artifactregistry-auth==1.1.2
120120
libclang==18.1.1
121121
Markdown==3.8.2
122122
markdown-it-py==4.0.0
123123
MarkupSafe==3.0.2
124124
mdurl==0.1.2
125-
ujson==5.8.0
126125
milvus-lite==2.5.1
127-
pymilvus==2.5.10
128126
ml-dtypes==0.3.2
129127
mmh3==5.2.0
130128
mock==5.2.0
@@ -180,8 +178,9 @@ pydot==1.4.2
180178
Pygments==2.19.2
181179
PyHamcrest==2.1.0
182180
PyJWT==2.10.1
183-
pymongo==4.14.0
184-
PyMySQL==1.1.1
181+
pymilvus==2.5.15
182+
pymongo==4.14.1
183+
PyMySQL==1.1.2
185184
pyparsing==3.2.3
186185
pyproject_hooks==1.2.0
187186
pytest==7.4.4
@@ -195,7 +194,7 @@ PyYAML==6.0.2
195194
redis==5.3.1
196195
referencing==0.36.2
197196
regex==2025.7.34
198-
requests==2.32.4
197+
requests==2.32.5
199198
requests-mock==1.12.1
200199
rich==14.1.0
201200
rpds-py==0.27.0
@@ -227,11 +226,12 @@ threadpoolctl==3.6.0
227226
tokenizers==0.21.4
228227
torch==2.7.1
229228
tqdm==4.67.1
230-
transformers==4.55.2
229+
transformers==4.55.4
231230
triton==3.3.1
232231
typing-inspection==0.4.1
233232
typing_extensions==4.14.1
234233
tzdata==2025.2
234+
ujson==5.11.0
235235
uritemplate==4.2.0
236236
urllib3==2.5.0
237237
virtualenv-clone==0.5.7

0 commit comments

Comments
 (0)