From 2ea22b640d444c05389894f51f1571129a0b9903 Mon Sep 17 00:00:00 2001 From: nblomerus Date: Mon, 18 May 2026 06:08:05 +0000 Subject: [PATCH] Add support for Python 3.11 Closes #7743. == Packaging == - Bump requires-python upper bound from <3.11 to <3.12 in pyproject.toml, package_build/tfx/pyproject.toml, and package_build/ml-pipelines-sdk/pyproject.toml. - Add Programming Language :: Python :: 3.11 classifier in each. == CI matrix == - Add '3.11' to ci-test.yml matrix and 'cp311' to wheels.yml matrix. - Disable strategy.fail-fast so individual job failures don't cancel the whole matrix (useful regardless of this change). == Dependency pins (test_constraints.txt + nightly_test_constraints.txt) == TFX 1.17 cp311 wheels require newer versions of several deps than the cp39/cp310 wheels. Use PEP 508 environment markers to pin per-Python-version: - apache-beam: 2.50.0 on <3.11, 2.53.0 on >=3.11 (tfx-bsl/tfdv cp311 wheels require beam >=2.53) - protobuf: 4.21.12 on <3.11, 4.25.5 on >=3.11 (tfx-bsl/tfdv/ml-metadata cp311 wheels require protobuf >=4.25.2; beam 2.53 requires !=4.21.*) - pyarrow: pin to 10.0.1 on >=3.11 (10.0.0 has no cp311 wheel and fails to build from sdist under setuptools 70) - tensorflow-io-gcs-filesystem: 0.24.0 on <3.11, 0.37.1 on >=3.11 (0.24.0 has no cp311 wheel) - grpcio-status: 1.62.3 (was 1.48.2); google-api-core requires grpcio-status>=1.49.1 on >=3.11, while grpcio-status 1.63+ would require protobuf 5 which conflicts with TFX's protobuf<5 == Test fixes == Two test files had patterns that broke on Python 3.11: - tfx/components/infra_validator/request_builder_test.py: mock.patch.object(..., wraps=Cls) returns _SentinelObject for .return_value before the wrapped class is called. Switch to autospec=True. autospec is strict about kwargs so pass examples= by keyword. - tfx/components/trainer/rewriting/tflite_rewriter_test.py: mock.patch('tensorflow.lite.TFLiteConverter.from_saved_model') fails on 3.11 because mock's string-path resolution doesn't trigger TF's lazy tf.lite module wrapper. Switch to mock.patch.object( tf.lite.TFLiteConverter, ...) for direct object reference. All 12 CI jobs (3 Python versions x not-e2e/e2e x DEFAULT/NIGHTLY) pass on this branch. --- .github/workflows/ci-test.yml | 3 ++- .github/workflows/wheels.yml | 2 +- nightly_test_constraints.txt | 12 ++++++++---- package_build/ml-pipelines-sdk/pyproject.toml | 3 ++- package_build/tfx/pyproject.toml | 3 ++- pyproject.toml | 3 ++- test_constraints.txt | 12 ++++++++---- .../infra_validator/request_builder_test.py | 6 +++--- .../trainer/rewriting/tflite_rewriter_test.py | 4 ++-- 9 files changed, 30 insertions(+), 18 deletions(-) diff --git a/.github/workflows/ci-test.yml b/.github/workflows/ci-test.yml index 0d988491ae..af82f49327 100644 --- a/.github/workflows/ci-test.yml +++ b/.github/workflows/ci-test.yml @@ -21,8 +21,9 @@ jobs: runs-on: ubuntu-latest strategy: + fail-fast: false matrix: - python-version: ['3.9', '3.10'] + python-version: ['3.9', '3.10', '3.11'] which-tests: ["not e2e", "e2e"] dependency-selector: ["NIGHTLY", "DEFAULT"] diff --git a/.github/workflows/wheels.yml b/.github/workflows/wheels.yml index 2ea84f7c68..395aec73f8 100644 --- a/.github/workflows/wheels.yml +++ b/.github/workflows/wheels.yml @@ -48,7 +48,7 @@ jobs: fail-fast: false matrix: os: [ubuntu] - python-version: ['cp39', 'cp310'] + python-version: ['cp39', 'cp310', 'cp311'] runs-on: ${{ format('{0}-latest', matrix.os) }} steps: diff --git a/nightly_test_constraints.txt b/nightly_test_constraints.txt index 9e1714c3b1..266236e0fd 100644 --- a/nightly_test_constraints.txt +++ b/nightly_test_constraints.txt @@ -22,7 +22,8 @@ alembic==1.13.3 annotated-types==0.7.0 anyio==4.6.0 apache-airflow==2.10.3 -apache-beam==2.50.0 +apache-beam==2.50.0; python_version < "3.11" +apache-beam==2.53.0; python_version >= "3.11" apispec==6.6.1 argcomplete==3.5.1 argon2-cffi==23.1.0 @@ -125,7 +126,7 @@ greenlet==3.1.1 grpc-google-iam-v1==0.13.1 grpc-interceptor==0.15.4 grpcio==1.66.2 -grpcio-status==1.48.2 +grpcio-status==1.62.3 gunicorn==23.0.0 h11==0.14.0 h5py==3.12.1 @@ -246,9 +247,11 @@ promise==2.3 prompt_toolkit==3.0.48 propcache==0.2.0 proto-plus==1.24.0 -protobuf==4.21.12 +protobuf==4.21.12; python_version < "3.11" +protobuf==4.25.5; python_version >= "3.11" psutil==6.0.0 ptyprocess==0.7.0 +pyarrow==10.0.1; python_version >= "3.11" pyarrow-hotfix==0.6 pyasn1==0.6.1 pyasn1_modules==0.4.1 @@ -315,7 +318,8 @@ tensorflow-decision-forests==1.10.1 tensorflow-estimator==2.15.0 tensorflow-hub==0.15.0 tensorflow-io==0.24.0 -tensorflow-io-gcs-filesystem==0.24.0 +tensorflow-io-gcs-filesystem==0.24.0; python_version < "3.11" +tensorflow-io-gcs-filesystem==0.37.1; python_version >= "3.11" tensorflow-metadata>=1.17.1 # tensorflow-ranking==0.5.5 tensorflow-serving-api==2.17.1 diff --git a/package_build/ml-pipelines-sdk/pyproject.toml b/package_build/ml-pipelines-sdk/pyproject.toml index 72852b4608..e654bfefbf 100644 --- a/package_build/ml-pipelines-sdk/pyproject.toml +++ b/package_build/ml-pipelines-sdk/pyproject.toml @@ -22,6 +22,7 @@ classifiers = [ "Programming Language :: Python :: 3", "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", "Programming Language :: Python :: 3 :: Only", "Topic :: Scientific/Engineering", "Topic :: Scientific/Engineering :: Artificial Intelligence", @@ -31,7 +32,7 @@ classifiers = [ "Topic :: Software Development :: Libraries :: Python Modules" ] keywords = ["tensorflow", "tfx"] -requires-python = ">=3.9,<3.11" +requires-python = ">=3.9,<3.12" [project.urls] Homepage = "https://www.tensorflow.org/tfx" Repository = "https://github.com/tensorflow/tfx" diff --git a/package_build/tfx/pyproject.toml b/package_build/tfx/pyproject.toml index f4d10a35fc..d9590b1a5c 100644 --- a/package_build/tfx/pyproject.toml +++ b/package_build/tfx/pyproject.toml @@ -22,6 +22,7 @@ classifiers = [ "Programming Language :: Python :: 3", "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", "Programming Language :: Python :: 3 :: Only", "Topic :: Scientific/Engineering", "Topic :: Scientific/Engineering :: Artificial Intelligence", @@ -31,7 +32,7 @@ classifiers = [ "Topic :: Software Development :: Libraries :: Python Modules" ] keywords = ["tensorflow", "tfx"] -requires-python = ">=3.9,<3.11" +requires-python = ">=3.9,<3.12" [project.urls] Homepage = "https://www.tensorflow.org/tfx" Repository = "https://github.com/tensorflow/tfx" diff --git a/pyproject.toml b/pyproject.toml index 70bbd6934e..d564d41f90 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -22,6 +22,7 @@ classifiers = [ "Programming Language :: Python :: 3", "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", "Programming Language :: Python :: 3 :: Only", "Topic :: Scientific/Engineering", "Topic :: Scientific/Engineering :: Artificial Intelligence", @@ -31,7 +32,7 @@ classifiers = [ "Topic :: Software Development :: Libraries :: Python Modules" ] keywords = ["tensorflow", "tfx"] -requires-python = ">=3.9,<3.11" +requires-python = ">=3.9,<3.12" [project.urls] Homepage = "https://www.tensorflow.org/tfx" Repository = "https://github.com/tensorflow/tfx" diff --git a/test_constraints.txt b/test_constraints.txt index de61db74fb..2255d496f7 100644 --- a/test_constraints.txt +++ b/test_constraints.txt @@ -22,7 +22,8 @@ alembic==1.13.3 annotated-types==0.7.0 anyio==4.6.0 apache-airflow==2.10.3 -apache-beam==2.50.0 +apache-beam==2.50.0; python_version < "3.11" +apache-beam==2.53.0; python_version >= "3.11" apispec==6.6.1 argcomplete==3.5.1 argon2-cffi==23.1.0 @@ -125,7 +126,7 @@ greenlet==3.1.1 grpc-google-iam-v1==0.13.1 grpc-interceptor==0.15.4 grpcio==1.66.2 -grpcio-status==1.48.2 +grpcio-status==1.62.3 gunicorn==23.0.0 h11==0.14.0 h5py==3.12.1 @@ -246,9 +247,11 @@ promise==2.3 prompt_toolkit==3.0.48 propcache==0.2.0 proto-plus==1.24.0 -protobuf==4.21.12 +protobuf==4.21.12; python_version < "3.11" +protobuf==4.25.5; python_version >= "3.11" psutil==6.0.0 ptyprocess==0.7.0 +pyarrow==10.0.1; python_version >= "3.11" pyarrow-hotfix==0.6 pyasn1==0.6.1 pyasn1_modules==0.4.1 @@ -315,7 +318,8 @@ tensorflow-decision-forests==1.10.1 tensorflow-estimator==2.15.0 tensorflow-hub==0.15.0 tensorflow-io==0.24.0 -tensorflow-io-gcs-filesystem==0.24.0 +tensorflow-io-gcs-filesystem==0.24.0; python_version < "3.11" +tensorflow-io-gcs-filesystem==0.37.1; python_version >= "3.11" tensorflow-metadata>=1.16.1 # tensorflow-ranking==0.5.5 tensorflow-serving-api==2.17.1 diff --git a/tfx/components/infra_validator/request_builder_test.py b/tfx/components/infra_validator/request_builder_test.py index 5e46a2db59..b48cc55415 100644 --- a/tfx/components/infra_validator/request_builder_test.py +++ b/tfx/components/infra_validator/request_builder_test.py @@ -440,7 +440,7 @@ def setUp(self): def _PrepareTFServingRequestBuilder(self): patcher = mock.patch.object( request_builder, '_TFServingRpcRequestBuilder', - wraps=request_builder._TFServingRpcRequestBuilder) + autospec=True) builder_cls = patcher.start() self.addCleanup(patcher.stop) return builder_cls @@ -466,7 +466,7 @@ def testBuildRequests_TFServing(self): model_name='foo', signatures={'serving_default': mock.ANY}) builder.ReadExamplesArtifact.assert_called_with( - self._examples, + examples=self._examples, split_name='eval', num_examples=1) builder.BuildRequests.assert_called() @@ -512,6 +512,6 @@ def testBuildRequests_DefaultArgument(self): ) builder.ReadExamplesArtifact.assert_called_with( - self._examples, + examples=self._examples, split_name=None, # Without split_name (will choose any split). num_examples=1) # Default num_examples = 1. diff --git a/tfx/components/trainer/rewriting/tflite_rewriter_test.py b/tfx/components/trainer/rewriting/tflite_rewriter_test.py index d353f41bf1..fd9cff725d 100644 --- a/tfx/components/trainer/rewriting/tflite_rewriter_test.py +++ b/tfx/components/trainer/rewriting/tflite_rewriter_test.py @@ -181,7 +181,7 @@ def testInvokeTFLiteRewriterQuantizationFloat16Succeeds(self, converter): @mock.patch('tfx.components.trainer.rewriting.' 'tflite_rewriter._create_tflite_compatible_saved_model') - @mock.patch('tensorflow.lite.TFLiteConverter.from_saved_model') + @mock.patch.object(tf.lite.TFLiteConverter, 'from_saved_model') def testInvokeTFLiteRewriterQuantizationFullIntegerFailsNoData( self, converter, model): @@ -231,7 +231,7 @@ def representative_dataset(): with fileio.open(expected_model, 'rb') as f: self.assertEqual(f.read(), b'model') - @mock.patch('tensorflow.lite.TFLiteConverter.from_saved_model') + @mock.patch.object(tf.lite.TFLiteConverter, 'from_saved_model') def testInvokeTFLiteRewriterWithSignatureKey(self, converter): m = self.ConverterMock() converter.return_value = m