From 3f7f2645aa178c25e4ac54a514b4eb687f8355d3 Mon Sep 17 00:00:00 2001 From: jrmccluskey Date: Tue, 28 Jul 2026 15:10:42 +0000 Subject: [PATCH 1/3] Bump lower and upper bounds for pyarrow, remove unnecessary CVE hotfix --- sdks/python/setup.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/sdks/python/setup.py b/sdks/python/setup.py index 65c247fa5057..dbdbe82eb2ad 100644 --- a/sdks/python/setup.py +++ b/sdks/python/setup.py @@ -150,11 +150,7 @@ def cythonize(*args, **kwargs): pyarrow_dependency = [''] else: pyarrow_dependency = [ - 'pyarrow>=6.0.1,<24.0.0', - # NOTE(https://github.com/apache/beam/issues/29392): We can remove this - # once Beam increases the pyarrow lower bound to a version that fixes CVE. - # (lower bound >= 14.0.1) - 'pyarrow-hotfix<1' + 'pyarrow>=14.0.1,<26.0.0', ] # Exclude pandas<=1.4.2 since it doesn't work with numpy 1.24.x. From ae6fe15df5421f17081300f899d9fc18dba08962 Mon Sep 17 00:00:00 2001 From: jrmccluskey Date: Tue, 28 Jul 2026 15:18:25 +0000 Subject: [PATCH 2/3] tighten bounds further, match with our testing environment --- sdks/python/setup.py | 5 ++++- sdks/python/tox.ini | 11 +++-------- 2 files changed, 7 insertions(+), 9 deletions(-) diff --git a/sdks/python/setup.py b/sdks/python/setup.py index dbdbe82eb2ad..3dbf9e86f985 100644 --- a/sdks/python/setup.py +++ b/sdks/python/setup.py @@ -150,7 +150,10 @@ def cythonize(*args, **kwargs): pyarrow_dependency = [''] else: pyarrow_dependency = [ - 'pyarrow>=14.0.1,<26.0.0', + # Generally try to cover versions released in the last two years. + # Update python/sdks/tox.ini to cover the same pyarrow versions + # when updating the bounds here. + 'pyarrow>=17.0.0,<26.0.0', ] # Exclude pandas<=1.4.2 since it doesn't work with numpy 1.24.x. diff --git a/sdks/python/tox.ini b/sdks/python/tox.ini index 6dab85083a02..245511e952d5 100644 --- a/sdks/python/tox.ini +++ b/sdks/python/tox.ini @@ -335,16 +335,9 @@ extras = test commands = bash {toxinidir}/scripts/pytest_validates_runner.sh {envname} {toxinidir}/apache_beam/runners/portability/prism_runner_test.py {posargs} -[testenv:py{310,311}-pyarrow-{6,15,16,17,18,19,20,21,22,23}] +[testenv:py{310,311}-pyarrow-{17,18,19,20,21,22,23,24,25}] deps = - # As a courtesy to users, test against the oldest allowed version of Pyarrow. - # We'd have to increase the pyarrow lower bound when Python 3.9 is deprecated. - # Since Pandas 2 requires pyarrow>=7, downgrade pandas for this test. - 6: pyarrow>=6,<7 - 6: pandas<2 # Test against versions of pyarrow released in last ~2 years. - 15: pyarrow>=15,<16 - 16: pyarrow>=16,<17 17: pyarrow>=17,<18 18: pyarrow>=18,<19 19: pyarrow>=19,<20 @@ -352,6 +345,8 @@ deps = 21: pyarrow>=21,<22 22: pyarrow>=22,<23 23: pyarrow>=23,<24 + 24: pyarrow>=24,<25 + 25: pyarrow>=25,<26 numpy==1.26.4 commands = # Log pyarrow and numpy version for debugging From 2360e0157a000c8dd46f94ad0d9d6ecc26c7e036 Mon Sep 17 00:00:00 2001 From: jrmccluskey Date: Tue, 28 Jul 2026 16:02:50 +0000 Subject: [PATCH 3/3] transitive deps --- sdks/python/setup.py | 18 ++++++------------ 1 file changed, 6 insertions(+), 12 deletions(-) diff --git a/sdks/python/setup.py b/sdks/python/setup.py index 3dbf9e86f985..86a628cfb5a2 100644 --- a/sdks/python/setup.py +++ b/sdks/python/setup.py @@ -588,17 +588,14 @@ def get_portability_package_data(): # tensorflow-transform requires dill, but doesn't set dill as a # hard requirement in setup.py. 'dill', # match tft extra. - 'tensorflow_transform>=1.14.0,<1.15.0', - # TFT->TFX-BSL require pandas 1.x, which is not compatible - # with numpy 2.x - 'numpy<2', + 'tensorflow_transform>=1.21.0,<1.22.0', # Comment out xgboost as it is breaking presubmit python ml # tests due to tag check introduced since pip 24.2 # https://github.com/apache/beam/issues/31285 # 'xgboost<2.0', # https://github.com/apache/beam/issues/31252 # tft needs protobuf<5; tf2onnx 1.17+ allows protobuf 5 on the # ADK-only path. - 'tf2onnx>=1.16.1,<1.17', + 'tf2onnx>=1.17.0,<1.18', ] + ml_base_core, 'p310_ml_test': [ 'datatable', @@ -608,7 +605,7 @@ def get_portability_package_data(): ] + ml_base + qdrant_dependency, # maintainer: milvus tests only run with this extension. Make sure it # is covered by docker-in-docker test when changing py version - 'p313_ml_test': ml_base + milvus_dependency + qdrant_dependency, + 'p313_ml_test': ml_base_core + milvus_dependency + qdrant_dependency, 'aws': ['boto3>=1.9,<2'], 'azure': [ 'azure-storage-blob>=12.3.2,<13', @@ -641,7 +638,7 @@ def get_portability_package_data(): # https://docs.google.com/document/d/1c84Gc-cZRCfrU8f7kWGsNR2o8oSRjCM-dGHO9KvPWPw/edit?usp=sharing 'torch': ['torch>=1.9.0,<2.8.0'], 'tensorflow': [ - 'tensorflow>=2.12rc1,<2.21', # tensorflow transitive dep + 'tensorflow>=2.12rc1,<2.22', # tensorflow transitive dep 'absl-py>=0.12.0' ], 'transformers': [ @@ -658,10 +655,7 @@ def get_portability_package_data(): ], 'redis': ['redis>=5.0.0,<6'], 'tft': [ - 'tensorflow_transform>=1.14.0,<1.15.0', - # TFT->TFX-BSL require pandas 1.x, which is not compatible - # with numpy 2.x - 'numpy<2', + 'tensorflow_transform>=1.21.0,<1.22.0', # tensorflow-transform requires dill, but doesn't set dill as a # hard requirement in setup.py. 'dill' @@ -671,7 +665,7 @@ def get_portability_package_data(): 'onnxruntime==1.13.1', 'torch==1.13.1', 'tensorflow==2.11.0', - 'tf2onnx==1.13.0', + 'tf2onnx==1.17.0', 'skl2onnx==1.13', 'transformers==4.25.1', # tensorflow transient dep 'absl-py>=0.12.0'