revert integration test changes

Author: derrickaw

.github/workflows/beam_PostCommit_Yaml_Xlang_Direct.yml

@@ -80,7 +80,7 @@ jobs:
       - name: run PostCommit Yaml Xlang Direct script
         uses: ./.github/actions/gradle-command-self-hosted-action
         with:
-          gradle-command: :sdks:python:postCommitYamlIntegrationTests -PyamlTestSet=${{ matrix.test_set }}
+          gradle-command: :sdks:python:postCommitYamlIntegrationTests -PyamlTestSet=${{ matrix.test_set }} -PbeamPythonExtra=ml_test,yaml,transformers
       - name: Archive Python Test Results
         uses: actions/upload-artifact@v7
         if: failure()

.github/workflows/beam_PreCommit_Yaml_Xlang_Direct.yml

@@ -91,7 +91,7 @@ jobs:
       - name: run PreCommit Yaml Xlang Direct script
         uses: ./.github/actions/gradle-command-self-hosted-action
         with:
-          gradle-command: :sdks:python:yamlIntegrationTests
+          gradle-command: :sdks:python:yamlIntegrationTests -PbeamPythonExtra=ml_test,yaml,transformers
       - name: Archive Python Test Results
         uses: actions/upload-artifact@v7
         if: failure()

sdks/python/build.gradle

@@ -124,35 +124,10 @@ tasks.register("generateYamlDocs") {
   outputs.file "${buildDir}/yaml-examples.html"
 }
 
-tasks.register("installYamlIntegrationTestDeps") {
-  dependsOn installGcpTest
-  doLast {
-    exec {
-      executable 'sh'
-      args '-c', ". ${envdir}/bin/activate && " +
-              "py_ver=\$(python -c 'import sys; print(f\"{sys.version_info.major}{sys.version_info.minor}\")') && " +
-              "ml_extra=\"p\${py_ver}_ml_test\" && " +
-              "constraint_file=\"\" && " +
-              "if [ -f \"container/ml/py\${py_ver}/base_image_requirements.txt\" ]; then " +
-              "  constraint_file=\"container/ml/py\${py_ver}/base_image_requirements.txt\"; " +
-              "elif [ -f \"container/py\${py_ver}/base_image_requirements.txt\" ]; then " +
-              "  constraint_file=\"container/py\${py_ver}/base_image_requirements.txt\"; " +
-              "fi && " +
-              "if [ -n \"\$constraint_file\" ]; then " +
-              "  echo \"Installing with constraint file: \$constraint_file\" && " +
-              "  pip install --pre --retries 10 -c \"\$constraint_file\" ${buildDir}/apache-beam.tar.gz[\$ml_extra,yaml,transformers] --extra-index-url https://download.pytorch.org/whl/cpu; " +
-              "else " +
-              "  echo \"No constraint file found, installing without constraints\" && " +
-              "  pip install --pre --retries 10 ${buildDir}/apache-beam.tar.gz[\$ml_extra,yaml,transformers] --extra-index-url https://download.pytorch.org/whl/cpu; " +
-              "fi"
-    }
-  }
-}
-
 tasks.register("yamlIntegrationTests") {
   description "Runs precommit integration tests for yaml pipelines."
 
-  dependsOn installYamlIntegrationTestDeps
+  dependsOn installGcpTest
   // Need to build all expansion services referenced in apache_beam/yaml/*.*
   // grep -oh 'sdk.*Jar' sdks/python/apache_beam/yaml/*.yaml | sort | uniq
   dependsOn ":sdks:java:extensions:schemaio-expansion-service:shadowJar"
@@ -171,7 +146,7 @@ tasks.register("yamlIntegrationTests") {
 tasks.register("postCommitYamlIntegrationTests") {
   description "Runs postcommit integration tests for yaml pipelines - parameterized by yamlTestSet."
 
-  dependsOn installYamlIntegrationTestDeps
+  dependsOn installGcpTest
   // Need to build all expansion services referenced in apache_beam/yaml/*.*
   // grep -oh 'sdk.*Jar' sdks/python/apache_beam/yaml/*.yaml | sort | uniq
   dependsOn ":sdks:java:extensions:schemaio-expansion-service:shadowJar"

sdks/python/setup.py

@@ -181,7 +181,6 @@ def cythonize(*args, **kwargs):
     'sentence-transformers>=2.2.2',
     'skl2onnx',
     'pyod>=0.7.6',  # 0.7.5 crashes setuptools
-    'contourpy<1.3.0; python_version < "3.11"',
     'tensorflow',
     # tensorflow transitive dep, lower versions not compatible with Python3.10+
     'absl-py>=0.12.0',
@@ -208,15 +207,13 @@ def cythonize(*args, **kwargs):
 ]
 
 ml_adk_dependency = [
-    # google-adk is excluded here because it requires google-genai<2.0.0,
-    # which conflicts with google-genai>=2.0.0 (e.g. 2.6.0) pinned in containers.
-    # 'google-adk==1.28.1',
+    'google-adk==1.28.1',
     # proto-plus<1.24 caps protobuf<5; opentelemetry-proto (via ADK) needs
     # protobuf>=5. Scoped here so the main dependency list stays broader.
     'proto-plus>=1.26.1,<2',
-    'opentelemetry-api>=1.37.0,<2',
-    'opentelemetry-sdk>=1.37.0,<2',
-    'opentelemetry-exporter-otlp-proto-http>=1.37.0,<2',
+    'opentelemetry-api==1.37.0',
+    'opentelemetry-sdk==1.37.0',
+    'opentelemetry-exporter-otlp-proto-http==1.37.0',
     # protobuf>=5 (ADK/OTel); tf2onnx 1.16.x pins protobuf~=3.20 only.
     'tf2onnx>=1.17.0,<1.18',
 ]
@@ -420,7 +417,6 @@ def get_portability_package_data():
       ext_modules=extensions,
       install_requires=[
           'cryptography>=39.0.0,<48.0.0',
-          'aiohttp<4.0.0',
           'envoy-data-plane>=1.0.3,<2; python_version >= "3.11"',
           # Newer version only work on Python 3.11. Versions 0.3 <= ver < 1.x
           # conflict with other GCP dependencies.
@@ -612,11 +608,10 @@ def get_portability_package_data():
           ] + ml_base_core,
           'p310_ml_test': [
               'datatable',
-              'dill',
-              'tensorflow_transform>=1.14.0,<1.15.0',
           ] + ml_base,
-          'p311_ml_test': ml_base,
-          'p312_ml_test': ml_base,
+          'p312_ml_test': [
+              'datatable',
+          ] + ml_base,
           # maintainer: milvus tests only run with this extension. Make sure it
           # is covered by docker-in-docker test when changing py version
           'p313_ml_test': ml_base + milvus_dependency,
@@ -690,8 +685,7 @@ def get_portability_package_data():
           'xgboost': ['xgboost>=1.6.0,<2.1.3', 'datatable==1.0.0'],
           'tensorflow-hub': ['tensorflow-hub>=0.14.0,<0.16.0'],
           'milvus': milvus_dependency,
-          'vllm': ['openai==1.107.1', 'vllm==0.10.1.1', 'triton==3.3.1'],
-          'adk': ['google-adk==1.28.1']
+          'vllm': ['openai==1.107.1', 'vllm==0.10.1.1', 'triton==3.3.1']
       },
       zip_safe=False,
       # PyPI package information.