From d8fb39c524ab3aa8eb7469a006168c1a7d8dc78d Mon Sep 17 00:00:00 2001 From: jrmccluskey Date: Tue, 16 Sep 2025 09:37:58 -0400 Subject: [PATCH 1/8] Move Vertex LLM Inference IT Test to new Endpoint --- .../apache_beam/ml/inference/vertex_ai_inference_it_test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sdks/python/apache_beam/ml/inference/vertex_ai_inference_it_test.py b/sdks/python/apache_beam/ml/inference/vertex_ai_inference_it_test.py index 7c96dbe8b847..a64fb0445930 100644 --- a/sdks/python/apache_beam/ml/inference/vertex_ai_inference_it_test.py +++ b/sdks/python/apache_beam/ml/inference/vertex_ai_inference_it_test.py @@ -37,7 +37,7 @@ _INPUT = "gs://apache-beam-ml/testing/inputs/vertex_images/*/*.jpg" _OUTPUT_DIR = "gs://apache-beam-ml/testing/outputs/vertex_images" _FLOWER_ENDPOINT_ID = "5384055553544683520" -_LLM_ENDPOINT_ID = "9157860935048626176" +_LLM_ENDPOINT_ID = "1273985431406379008" _ENDPOINT_PROJECT = "apache-beam-testing" _ENDPOINT_REGION = "us-central1" _ENDPOINT_NETWORK = "projects/844138762903/global/networks/beam-test-vpc" From f8b3ff89a12c774ee30746b3c41c43f3bac26fc7 Mon Sep 17 00:00:00 2001 From: jrmccluskey Date: Tue, 16 Sep 2025 13:24:08 -0400 Subject: [PATCH 2/8] trigger python postcommit --- .github/trigger_files/beam_PostCommit_Python.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/trigger_files/beam_PostCommit_Python.json b/.github/trigger_files/beam_PostCommit_Python.json index 8675e9535061..815b511b8988 100644 --- a/.github/trigger_files/beam_PostCommit_Python.json +++ b/.github/trigger_files/beam_PostCommit_Python.json @@ -1,5 +1,5 @@ { "comment": "Modify this file in a trivial way to cause this test suite to run.", - "modification": 28 + "modification": 30 } From f33bcdac96606c532533ded6b65272b6c4964059 Mon Sep 17 00:00:00 2001 From: jrmccluskey Date: Thu, 18 Sep 2025 10:22:17 -0400 Subject: [PATCH 3/8] Remove LLM IT test, add note to example --- .../vertex_ai_llm_text_classification.py | 8 ++++++++ .../ml/inference/vertex_ai_inference_it_test.py | 15 --------------- 2 files changed, 8 insertions(+), 15 deletions(-) diff --git a/sdks/python/apache_beam/examples/inference/vertex_ai_llm_text_classification.py b/sdks/python/apache_beam/examples/inference/vertex_ai_llm_text_classification.py index e587ba87b91b..3c47d4704813 100644 --- a/sdks/python/apache_beam/examples/inference/vertex_ai_llm_text_classification.py +++ b/sdks/python/apache_beam/examples/inference/vertex_ai_llm_text_classification.py @@ -21,6 +21,14 @@ model can be generated by fine tuning the text-bison model or another similar model (see https://cloud.google.com/vertex-ai/docs/generative-ai/models/tune-models#supervised-fine-tuning) + +**NOTE** +Google has deprecated non-Gemini LLMs and no longer supports querying them on +Vertex AI endpoints. Separately, the use of the Vertex AI Predict API is not +supported for Gemini models in favor of use of the google-genai API. As a +result, this example no longer works. To perform inference with remote LLMs +deployed on Google infrastructure, please see the GeminiModelHandler and +the gemini_text_classification.py example. """ import argparse diff --git a/sdks/python/apache_beam/ml/inference/vertex_ai_inference_it_test.py b/sdks/python/apache_beam/ml/inference/vertex_ai_inference_it_test.py index a64fb0445930..feaee56e88e7 100644 --- a/sdks/python/apache_beam/ml/inference/vertex_ai_inference_it_test.py +++ b/sdks/python/apache_beam/ml/inference/vertex_ai_inference_it_test.py @@ -65,21 +65,6 @@ def test_vertex_ai_run_flower_image_classification(self): test_pipeline.get_full_options_as_args(**extra_opts)) self.assertEqual(FileSystems().exists(output_file), True) - @pytest.mark.vertex_ai_postcommit - def test_vertex_ai_run_llm_text_classification(self): - output_file = '/'.join([_OUTPUT_DIR, str(uuid.uuid4()), 'output.txt']) - - test_pipeline = TestPipeline(is_integration_test=True) - extra_opts = { - 'output': output_file, - 'endpoint_id': _LLM_ENDPOINT_ID, - 'endpoint_project': _ENDPOINT_PROJECT, - 'endpoint_region': _ENDPOINT_REGION - } - vertex_ai_llm_text_classification.run( - test_pipeline.get_full_options_as_args(**extra_opts)) - self.assertEqual(FileSystems().exists(output_file), True) - if __name__ == '__main__': logging.getLogger().setLevel(logging.DEBUG) From 09a9e0ced68cae15e28024167b6c7b7329651bfd Mon Sep 17 00:00:00 2001 From: jrmccluskey Date: Thu, 18 Sep 2025 10:29:40 -0400 Subject: [PATCH 4/8] remove unused var --- .../apache_beam/ml/inference/vertex_ai_inference_it_test.py | 1 - 1 file changed, 1 deletion(-) diff --git a/sdks/python/apache_beam/ml/inference/vertex_ai_inference_it_test.py b/sdks/python/apache_beam/ml/inference/vertex_ai_inference_it_test.py index feaee56e88e7..3acce46f8510 100644 --- a/sdks/python/apache_beam/ml/inference/vertex_ai_inference_it_test.py +++ b/sdks/python/apache_beam/ml/inference/vertex_ai_inference_it_test.py @@ -37,7 +37,6 @@ _INPUT = "gs://apache-beam-ml/testing/inputs/vertex_images/*/*.jpg" _OUTPUT_DIR = "gs://apache-beam-ml/testing/outputs/vertex_images" _FLOWER_ENDPOINT_ID = "5384055553544683520" -_LLM_ENDPOINT_ID = "1273985431406379008" _ENDPOINT_PROJECT = "apache-beam-testing" _ENDPOINT_REGION = "us-central1" _ENDPOINT_NETWORK = "projects/844138762903/global/networks/beam-test-vpc" From 071cf78e7c31b88cd37bb6040a8dfbcf5a530b26 Mon Sep 17 00:00:00 2001 From: jrmccluskey Date: Thu, 18 Sep 2025 10:29:57 -0400 Subject: [PATCH 5/8] Remove unused import --- .../apache_beam/ml/inference/vertex_ai_inference_it_test.py | 1 - 1 file changed, 1 deletion(-) diff --git a/sdks/python/apache_beam/ml/inference/vertex_ai_inference_it_test.py b/sdks/python/apache_beam/ml/inference/vertex_ai_inference_it_test.py index 3acce46f8510..c6d62eb3e3e1 100644 --- a/sdks/python/apache_beam/ml/inference/vertex_ai_inference_it_test.py +++ b/sdks/python/apache_beam/ml/inference/vertex_ai_inference_it_test.py @@ -29,7 +29,6 @@ # pylint: disable=ungrouped-imports try: from apache_beam.examples.inference import vertex_ai_image_classification - from apache_beam.examples.inference import vertex_ai_llm_text_classification except ImportError as e: raise unittest.SkipTest( "Vertex AI model handler dependencies are not installed") From 4c462f5ca18053031373768b2c22e33e39ffa1cd Mon Sep 17 00:00:00 2001 From: Jack McCluskey <34928439+jrmccluskey@users.noreply.github.com> Date: Thu, 18 Sep 2025 10:30:21 -0400 Subject: [PATCH 6/8] Update sdks/python/apache_beam/examples/inference/vertex_ai_llm_text_classification.py Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com> --- .../examples/inference/vertex_ai_llm_text_classification.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sdks/python/apache_beam/examples/inference/vertex_ai_llm_text_classification.py b/sdks/python/apache_beam/examples/inference/vertex_ai_llm_text_classification.py index 3c47d4704813..1d8d0b1a1efc 100644 --- a/sdks/python/apache_beam/examples/inference/vertex_ai_llm_text_classification.py +++ b/sdks/python/apache_beam/examples/inference/vertex_ai_llm_text_classification.py @@ -27,8 +27,8 @@ Vertex AI endpoints. Separately, the use of the Vertex AI Predict API is not supported for Gemini models in favor of use of the google-genai API. As a result, this example no longer works. To perform inference with remote LLMs -deployed on Google infrastructure, please see the GeminiModelHandler and -the gemini_text_classification.py example. +deployed on Google infrastructure, please see the `GeminiModelHandler` (in `apache_beam.ml.inference.gemini_inference`) and +the `gemini_text_classification.py` example. """ import argparse From f27eec717b9b7e78954f97cae7bda8680382bd91 Mon Sep 17 00:00:00 2001 From: jrmccluskey Date: Thu, 18 Sep 2025 11:01:06 -0400 Subject: [PATCH 7/8] linting --- .../examples/inference/vertex_ai_llm_text_classification.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/sdks/python/apache_beam/examples/inference/vertex_ai_llm_text_classification.py b/sdks/python/apache_beam/examples/inference/vertex_ai_llm_text_classification.py index 1d8d0b1a1efc..944f806f4187 100644 --- a/sdks/python/apache_beam/examples/inference/vertex_ai_llm_text_classification.py +++ b/sdks/python/apache_beam/examples/inference/vertex_ai_llm_text_classification.py @@ -27,8 +27,9 @@ Vertex AI endpoints. Separately, the use of the Vertex AI Predict API is not supported for Gemini models in favor of use of the google-genai API. As a result, this example no longer works. To perform inference with remote LLMs -deployed on Google infrastructure, please see the `GeminiModelHandler` (in `apache_beam.ml.inference.gemini_inference`) and -the `gemini_text_classification.py` example. +deployed on Google infrastructure, please see the `GeminiModelHandler` +(in `apache_beam.ml.inference.gemini_inference`) and the +`gemini_text_classification.py` example. """ import argparse From ec86fec2e9fbed7298abcc3a516608c81dbba1b4 Mon Sep 17 00:00:00 2001 From: jrmccluskey Date: Thu, 18 Sep 2025 15:56:25 -0400 Subject: [PATCH 8/8] Soften disclaimer language a bit, add call-out to README --- sdks/python/apache_beam/examples/inference/README.md | 6 ++++++ .../inference/vertex_ai_llm_text_classification.py | 11 ++++++----- 2 files changed, 12 insertions(+), 5 deletions(-) diff --git a/sdks/python/apache_beam/examples/inference/README.md b/sdks/python/apache_beam/examples/inference/README.md index f9c5af436965..e0367ea69384 100644 --- a/sdks/python/apache_beam/examples/inference/README.md +++ b/sdks/python/apache_beam/examples/inference/README.md @@ -856,6 +856,12 @@ Each line represents a prediction of the flower type along with the confidence i ## Text classifcation with a Vertex AI LLM +**NOTE** +Google has deprecated PaLM LLMs like text-bison and no longer supports querying them on Vertex AI endpoints. Separately, the use of the Vertex AI Predict API is +not supported for Gemini models in favor of use of the google-genai API. As a result, this example no longer works as-written. To perform inference with +Gemini models deployed on Google infrastructure, please see the `GeminiModelHandler` (in `apache_beam.ml.inference.gemini_inference`) and the +[`gemini_text_classification.py` example](./gemini_text_classification.py). For custom LLMs, you may still follow this design pattern. + [`vertex_ai_llm_text_classification.py`](./vertex_ai_llm_text_classification.py) contains an implementation for a RunInference pipeline that performs image classification using a model hosted on Vertex AI (based on https://cloud.google.com/vertex-ai/docs/tutorials/image-recognition-custom). The pipeline reads image urls, performs basic preprocessing to convert them into a List of floats, passes the masked sentence to the Vertex AI implementation of RunInference, and then writes the predictions to a text file. diff --git a/sdks/python/apache_beam/examples/inference/vertex_ai_llm_text_classification.py b/sdks/python/apache_beam/examples/inference/vertex_ai_llm_text_classification.py index 944f806f4187..75f021c37128 100644 --- a/sdks/python/apache_beam/examples/inference/vertex_ai_llm_text_classification.py +++ b/sdks/python/apache_beam/examples/inference/vertex_ai_llm_text_classification.py @@ -23,13 +23,14 @@ https://cloud.google.com/vertex-ai/docs/generative-ai/models/tune-models#supervised-fine-tuning) **NOTE** -Google has deprecated non-Gemini LLMs and no longer supports querying them on +Google has deprecated PaLM LLMs and no longer supports querying them on Vertex AI endpoints. Separately, the use of the Vertex AI Predict API is not supported for Gemini models in favor of use of the google-genai API. As a -result, this example no longer works. To perform inference with remote LLMs -deployed on Google infrastructure, please see the `GeminiModelHandler` -(in `apache_beam.ml.inference.gemini_inference`) and the -`gemini_text_classification.py` example. +result, this example no longer works as-written. To perform inference with +Gemini models deployed on Google infrastructure, please see the +`GeminiModelHandler` (in `apache_beam.ml.inference.gemini_inference`) and the +`gemini_text_classification.py` example. For custom LLMs, you may still follow +this design pattern. """ import argparse