Bump lower and upper bounds for pyarrow + related dependencies, remove unnecessary CVE hotfix (#39530)

* Bump lower and upper bounds for pyarrow, remove unnecessary CVE hotfix

* tighten bounds further, match with our testing environment

* transitive deps

* Update sdks/python/setup.py

Co-authored-by: Danny McCormick <dannymccormick@google.com>

---------

Co-authored-by: Danny McCormick <dannymccormick@google.com>

Author: jrmccluskey

sdks/python/setup.py

@@ -150,11 +150,10 @@ def cythonize(*args, **kwargs):
   pyarrow_dependency = ['']
 else:
   pyarrow_dependency = [
-      'pyarrow>=6.0.1,<24.0.0',
-      # NOTE(https://github.com/apache/beam/issues/29392): We can remove this
-      # once Beam increases the pyarrow lower bound to a version that fixes CVE.
-      # (lower bound >= 14.0.1)
-      'pyarrow-hotfix<1'
+    # Generally try to cover versions released in the last two years.
+    # Update python/sdks/tox.ini to cover the same pyarrow versions
+    # when updating the bounds here.
+    'pyarrow>=14.0.1,<26.0.0',
   ]
 
 # Exclude pandas<=1.4.2 since it doesn't work with numpy 1.24.x.
@@ -589,17 +588,14 @@ def get_portability_package_data():
               # tensorflow-transform requires dill, but doesn't set dill as a
               # hard requirement in setup.py.
               'dill',  # match tft extra.
-              'tensorflow_transform>=1.14.0,<1.15.0',
-              # TFT->TFX-BSL require pandas 1.x, which is not compatible
-              # with numpy 2.x
-              'numpy<2',
+              'tensorflow_transform>=1.21.0,<1.22.0',
               # Comment out xgboost as it is breaking presubmit python ml
               # tests due to tag check introduced since pip 24.2
               # https://github.com/apache/beam/issues/31285
               # 'xgboost<2.0',  # https://github.com/apache/beam/issues/31252
               # tft needs protobuf<5; tf2onnx 1.17+ allows protobuf 5 on the
               # ADK-only path.
-              'tf2onnx>=1.16.1,<1.17',
+              'tf2onnx>=1.17.0,<1.18',
           ] + ml_base_core,
           'p310_ml_test': [
             'datatable',
@@ -609,7 +605,7 @@ def get_portability_package_data():
           ] + ml_base + qdrant_dependency,
           # maintainer: milvus tests only run with this extension. Make sure it
           # is covered by docker-in-docker test when changing py version
-          'p313_ml_test': ml_base + milvus_dependency + qdrant_dependency,
+          'p313_ml_test': ml_base_core + milvus_dependency + qdrant_dependency,
           'aws': ['boto3>=1.9,<2'],
           'azure': [
               'azure-storage-blob>=12.3.2,<13',
@@ -642,7 +638,7 @@ def get_portability_package_data():
           # https://docs.google.com/document/d/1c84Gc-cZRCfrU8f7kWGsNR2o8oSRjCM-dGHO9KvPWPw/edit?usp=sharing
           'torch': ['torch>=1.9.0,<2.8.0'],
           'tensorflow': [
-              'tensorflow>=2.12rc1,<2.21',  # tensorflow transitive dep
+              'tensorflow>=2.12rc1,<2.22',  # tensorflow transitive dep
               'absl-py>=0.12.0'
           ],
           'transformers': [
@@ -659,10 +655,7 @@ def get_portability_package_data():
           ],
           'redis': ['redis>=5.0.0,<6'],
           'tft': [
-              'tensorflow_transform>=1.14.0,<1.15.0',
-              # TFT->TFX-BSL require pandas 1.x, which is not compatible
-              # with numpy 2.x
-              'numpy<2',
+              'tensorflow_transform>=1.21.0,<1.22.0',
               # tensorflow-transform requires dill, but doesn't set dill as a
               # hard requirement in setup.py.
               'dill'
@@ -672,7 +665,7 @@ def get_portability_package_data():
               'onnxruntime==1.13.1',
               'torch==1.13.1',
               'tensorflow==2.11.0',
-              'tf2onnx==1.13.0',
+              'tf2onnx==1.17.0',
               'skl2onnx==1.13',
               'transformers==4.25.1',  # tensorflow transient dep
               'absl-py>=0.12.0'

sdks/python/tox.ini

@@ -335,23 +335,18 @@ extras = test
 commands =
   bash {toxinidir}/scripts/pytest_validates_runner.sh {envname} {toxinidir}/apache_beam/runners/portability/prism_runner_test.py {posargs}
 
-[testenv:py{310,311}-pyarrow-{6,15,16,17,18,19,20,21,22,23}]
+[testenv:py{310,311}-pyarrow-{17,18,19,20,21,22,23,24,25}]
 deps =
-  # As a courtesy to users, test against the oldest allowed version of Pyarrow.
-  # We'd have to increase the pyarrow lower bound when Python 3.9 is deprecated.
-  # Since Pandas 2 requires pyarrow>=7, downgrade pandas for this test.
-  6: pyarrow>=6,<7
-  6: pandas<2
   # Test against versions of pyarrow released in last ~2 years.
-  15: pyarrow>=15,<16
-  16: pyarrow>=16,<17
   17: pyarrow>=17,<18
   18: pyarrow>=18,<19
   19: pyarrow>=19,<20
   20: pyarrow>=20,<21
   21: pyarrow>=21,<22
   22: pyarrow>=22,<23
   23: pyarrow>=23,<24
+  24: pyarrow>=24,<25
+  25: pyarrow>=25,<26
   numpy==1.26.4
 commands =
   # Log pyarrow and numpy version for debugging