Fix pip ResolutionTooDeep (#39017)

* Fix pip ResolutionTooDeep

* added constraints file

* added dill

* Add usage comment

Author: aIbrahiim

.github/trigger_files/beam_CloudML_Benchmarks_Dataflow.json

@@ -1,4 +1,4 @@
 {
     "comment": "Modify this file in a trivial way to cause this test suite to run.",
-    "modification": 3
+    "modification": 5
 }

sdks/python/apache_beam/testing/benchmarks/cloudml/constraints.txt

@@ -0,0 +1,42 @@
+#
+#    Licensed to the Apache Software Foundation (ASF) under one or more
+#    contributor license agreements.  See the NOTICE file distributed with
+#    this work for additional information regarding copyright ownership.
+#    The ASF licenses this file to You under the Apache License, Version 2.0
+#    (the "License"); you may not use this file except in compliance with
+#    the License.  You may obtain a copy of the License at
+#
+#       http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+# Constraints file to pin versions and avoid pip ResolutionTooDeep.
+# This file is used with: pip install -c constraints.txt -r requirements.txt
+
+# Core dependencies
+tfx_bsl==1.15.1
+tensorflow-transform==1.15.0
+
+# TensorFlow ecosystem
+tensorflow==2.15.1
+tensorflow-metadata==1.15.0
+tf-keras==2.15.1
+
+# NumPy and data handling
+numpy==1.26.4
+pyarrow==10.0.1
+
+# Google Cloud (pin to avoid transitive resolution)
+google-cloud-aiplatform==1.60.0
+google-api-core==2.19.1
+
+# Note: google-auth is NOT constrained - let pip resolve it to satisfy
+# apache-beam's google-genai requirement (>=2.48.1)
+
+# Note: tensorflow-serving-api is NOT constrained - let pip resolve it within
+# the range specified in requirements.txt (>=2.15,<2.16)

sdks/python/apache_beam/testing/benchmarks/cloudml/requirements.txt

@@ -15,12 +15,15 @@
 #    limitations under the License.
 #
 
-dill==0.4.1
-tfx_bsl==1.16.1
-tensorflow-transform==1.16.0
-tensorflow>=2.16,<2.17
+# Core TFT dependencies with version bounds.
+# Note: To avoid pip ResolutionTooDeep errors, always install using the constraints file:
+# pip install -c constraints.txt -r requirements.txt
+dill>=0.3,<0.5
+tfx_bsl>=1.15,<1.17
+tensorflow-transform>=1.15,<1.17
+tensorflow>=2.15,<2.16
 numpy>=1.22.0,<2.0
-tensorflow-metadata>=1.16.1,<1.17.0
+tensorflow-metadata>=1.15,<1.16
 pyarrow>=10,<11
-tensorflow-serving-api>=2.16.1,<2.20
-tf-keras>=2.16.0,<2.17
+tensorflow-serving-api>=2.15,<2.16
+tf-keras>=2.15,<2.16

sdks/python/test-suites/dataflow/common.gradle

@@ -573,13 +573,9 @@ task installTFTRequirements {
     exec {
       workingDir "$rootProject.projectDir/sdks/python/apache_beam/testing/benchmarks/cloudml/"
       executable 'sh'
-      // installGcpTest already installed apache-beam[gcp]. tensorflow-transform also
-      // lists that dependency, so a plain pip install -r can re-resolve the GCP extra
-      // and hit ResolutionTooDeep. Install TFT with --no-deps instead.
-      args '-c', ". ${envdir}/bin/activate && " +
-          "grep -v '^tensorflow-transform' requirements.txt > /tmp/cloudml_tft_base_requirements.txt && " +
-          "pip install -r /tmp/cloudml_tft_base_requirements.txt && " +
-          "pip install --no-deps tensorflow-transform==1.16.0"
+      // Use constraints file to pin versions while allowing pip to
+      // resolve compatible versions within the specified ranges in requirements.txt
+      args '-c', ". ${envdir}/bin/activate && pip install -c constraints.txt -r requirements.txt"
     }
   }
 }