Merge pull request #39163 from apache/resolution-too-deep

Upgrade TFX stack to 1.21.0 and remove numpy/pandas bounds

Author: Amar3tto

.github/workflows/beam_Inference_Python_Benchmarks_Dataflow.yml

@@ -226,7 +226,7 @@ jobs:
             -Prunner=DataflowRunner \
             -PpythonVersion=3.10 \
             -PloadTest.requirementsTxtFile=apache_beam/examples/ml_transform/mltransform_generate_vocab_requirements.txt \
-            '-PloadTest.args=${{ env.beam_Inference_Python_Benchmarks_Dataflow_test_arguments_11 }} --job_name=benchmark-tests-mltransform-generate-vocab-batch-${{env.NOW_UTC}}'
+            '-PloadTest.args=${{ env.beam_Inference_Python_Benchmarks_Dataflow_test_arguments_11 }} --job_name=benchmark-tests-mltransform-generate-vocab-batch-${{env.NOW_UTC}} --artifact_location=gs://temp-storage-for-perf-tests/mltransform/vocab_artifacts_${{env.NOW_UTC}} --output_vocab=gs://temp-storage-for-perf-tests/mltransform/vocab_outputs/mltransform_generate_vocab_batch_${{env.NOW_UTC}}'
       - name: run MLTransform One-Hot Encoding Batch
         uses: ./.github/actions/gradle-command-self-hosted-action
         timeout-minutes: 180
@@ -236,6 +236,5 @@ jobs:
             -PloadTest.mainClass=apache_beam.testing.benchmarks.inference.mltransform_one_hot_encoding_benchmark \
             -Prunner=DataflowRunner \
             -PpythonVersion=3.10 \
-            -PbeamPythonExtra=ml_test \
-            -PloadTest.requirementsTxtFile=apache_beam/ml/transforms/mltransform_tests_requirements.txt \
-            '-PloadTest.args=${{ env.beam_Inference_Python_Benchmarks_Dataflow_test_arguments_12 }} --autoscaling_algorithm=NONE --metrics_table=mltransform_one_hot_encoding_batch --influx_measurement=mltransform_one_hot_encoding_batch --job_name=benchmark-tests-mltransform-one-hot-encoding-batch-${{env.NOW_UTC}} --output_file=gs://temp-storage-for-end-to-end-tests/mltransform/one_hot_output_${{env.NOW_UTC}} --artifact_location=gs://temp-storage-for-end-to-end-tests/mltransform/artifacts_${{env.NOW_UTC}}'
+            -PloadTest.requirementsTxtFile=apache_beam/examples/ml_transform/mltransform_one_hot_encoding_requirements.txt \
+            '-PloadTest.args=${{ env.beam_Inference_Python_Benchmarks_Dataflow_test_arguments_12 }} --autoscaling_algorithm=NONE --metrics_table=mltransform_one_hot_encoding_batch --influx_measurement=mltransform_one_hot_encoding_batch --job_name=benchmark-tests-mltransform-one-hot-encoding-batch-${{env.NOW_UTC}} --output_file=gs://temp-storage-for-end-to-end-tests/mltransform/one_hot_output_${{env.NOW_UTC}} --artifact_location=gs://temp-storage-for-end-to-end-tests/mltransform/artifacts_${{env.NOW_UTC}}'

.github/workflows/load-tests-pipeline-options/beam_Inference_Python_Benchmarks_Dataflow_MLTransform_One_Hot_Encoding_Batch.txt

@@ -22,7 +22,7 @@
 --staging_location=gs://temp-storage-for-perf-tests/loadtests
 --temp_location=gs://temp-storage-for-perf-tests/loadtests
 --sdk_location=container
---requirements_file=apache_beam/ml/transforms/mltransform_tests_requirements.txt
+--requirements_file=apache_beam/examples/ml_transform/mltransform_one_hot_encoding_requirements.txt
 --publish_to_big_query=true
 --metrics_dataset=beam_run_inference
 --metrics_table=mltransform_one_hot_encoding_batch

sdks/python/apache_beam/examples/ml_transform/mltransform_generate_vocab_requirements.txt

@@ -18,9 +18,9 @@
 # MLTransform TFT operations need a consistent TensorFlow Transform stack;
 # otherwise workers can crash-loop with pandas/numpy ABI mismatches.
 google-cloud-monitoring>=2.27.0
-tensorflow_transform>=1.14.0,<1.15.0
-tensorflow-metadata>=1.14.0,<1.15.0
-tfx-bsl>=1.14.0,<1.15.0
-numpy<2
-pandas<2
+tensorflow_transform==1.21.0
+tensorflow-metadata==1.21.0
+tfx-bsl==1.21.0
 dill
+numpy
+pandas

…forms/mltransform_tests_requirements.txt …nsform_one_hot_encoding_requirements.txtsdks/python/apache_beam/ml/transforms/mltransform_tests_requirements.txt renamed to sdks/python/apache_beam/examples/ml_transform/mltransform_one_hot_encoding_requirements.txt sdks/python/apache_beam/ml/transforms/mltransform_tests_requirements.txt renamed to sdks/python/apache_beam/examples/ml_transform/mltransform_one_hot_encoding_requirements.txt

@@ -18,12 +18,12 @@
 # Requirements for MLTransform tests on Dataflow workers.
 # Keep this aligned with CloudML benchmark stack to avoid worker import errors.
 dill==0.4.1
-tfx_bsl==1.16.1
-tensorflow-transform==1.16.0
-tensorflow>=2.16,<2.17
-numpy>=1.22.0,<2.0
-tensorflow-metadata>=1.16.1,<1.17.0
-pyarrow>=10,<11
-tensorflow-serving-api>=2.16.1,<2.20
-tf-keras>=2.16.0,<2.17
+tfx_bsl==1.21.0
+tensorflow-transform==1.21.0
+tensorflow-metadata==1.21.0
+tensorflow
+numpy
+pyarrow
+tensorflow-serving-api
+tf-keras
 google-cloud-monitoring>=2.27.0