Skip to content

Commit 2049473

Browse files
aIbrahiimaIbrahiim
authored
Add MLTransform embedding benchmarks (#38917)
* Add MLTransform embedding benchmarks * resolved code review comments * Refactor DataflowCostBenchmark worker interval and throughput helpers
1 parent c6e13b6 commit 2049473

19 files changed

Lines changed: 1424 additions & 72 deletions

‎.github/workflows/beam_Inference_Python_Benchmarks_Dataflow.yml‎

Lines changed: 36 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -96,6 +96,9 @@ jobs:
9696
${{ github.workspace }}/.github/workflows/load-tests-pipeline-options/beam_Inference_Python_Benchmarks_Dataflow_Table_Row_Inference_Stream.txt
9797
${{ github.workspace }}/.github/workflows/load-tests-pipeline-options/beam_Inference_Python_Benchmarks_Dataflow_MLTransform_Generate_Vocab_Batch.txt
9898
${{ github.workspace }}/.github/workflows/load-tests-pipeline-options/beam_Inference_Python_Benchmarks_Dataflow_MLTransform_One_Hot_Encoding_Batch.txt
99+
${{ github.workspace }}/.github/workflows/load-tests-pipeline-options/beam_Inference_Python_Benchmarks_Dataflow_MLTransform_Image_Embedding_GPU_Batch.txt
100+
${{ github.workspace }}/.github/workflows/load-tests-pipeline-options/beam_Inference_Python_Benchmarks_Dataflow_MLTransform_Text_Embedding_Batch.txt
101+
${{ github.workspace }}/.github/workflows/load-tests-pipeline-options/beam_Inference_Python_Benchmarks_Dataflow_MLTransform_Image_Embedding_CPU_Batch.txt
99102
# The env variables are created and populated in the test-arguments-action as "<github.job>_test_arguments_<argument_file_paths_index>"
100103
- name: get current time
101104
run: echo "NOW_UTC=$(date '+%m%d%H%M%S' --utc)" >> $GITHUB_ENV
@@ -238,3 +241,36 @@ jobs:
238241
-PpythonVersion=3.10 \
239242
-PloadTest.requirementsTxtFile=apache_beam/examples/ml_transform/mltransform_one_hot_encoding_requirements.txt \
240243
'-PloadTest.args=${{ env.beam_Inference_Python_Benchmarks_Dataflow_test_arguments_12 }} --autoscaling_algorithm=NONE --metrics_table=mltransform_one_hot_encoding_batch --influx_measurement=mltransform_one_hot_encoding_batch --job_name=benchmark-tests-mltransform-one-hot-encoding-batch-${{env.NOW_UTC}} --output_file=gs://temp-storage-for-end-to-end-tests/mltransform/one_hot_output_${{env.NOW_UTC}} --artifact_location=gs://temp-storage-for-end-to-end-tests/mltransform/artifacts_${{env.NOW_UTC}}'
244+
- name: run MLTransform Image Embedding GPU Batch
245+
uses: ./.github/actions/gradle-command-self-hosted-action
246+
timeout-minutes: 180
247+
with:
248+
gradle-command: :sdks:python:apache_beam:testing:load_tests:run
249+
arguments: |
250+
-PloadTest.mainClass=apache_beam.testing.benchmarks.inference.mltransform_image_embedding_benchmark \
251+
-Prunner=DataflowRunner \
252+
-PpythonVersion=3.10 \
253+
-PloadTest.requirementsTxtFile=apache_beam/ml/transforms/mltransform_embedding_tests_requirements.txt \
254+
'-PloadTest.args=${{ env.beam_Inference_Python_Benchmarks_Dataflow_test_arguments_13 }} --artifact_location=gs://temp-storage-for-perf-tests/loadtests/mltransform_image_embedding_gpu/artifacts-${{env.NOW_UTC}} --job_name=benchmark-tests-mltransform-image-embedding-gpu-batch-${{env.NOW_UTC}}'
255+
- name: run MLTransform Text Embedding Batch
256+
uses: ./.github/actions/gradle-command-self-hosted-action
257+
timeout-minutes: 180
258+
with:
259+
gradle-command: :sdks:python:apache_beam:testing:load_tests:run
260+
arguments: |
261+
-PloadTest.mainClass=apache_beam.testing.benchmarks.inference.mltransform_text_embedding_benchmark \
262+
-Prunner=DataflowRunner \
263+
-PpythonVersion=3.10 \
264+
-PloadTest.requirementsTxtFile=apache_beam/ml/transforms/mltransform_embedding_tests_requirements.txt \
265+
'-PloadTest.args=${{ env.beam_Inference_Python_Benchmarks_Dataflow_test_arguments_14 }} --output=gs://temp-storage-for-perf-tests/loadtests/mltransform_text_embedding/results-${{env.NOW_UTC}} --artifact_location=gs://temp-storage-for-perf-tests/loadtests/mltransform_text_embedding/artifacts-${{env.NOW_UTC}} --job_name=benchmark-tests-mltransform-text-embedding-batch-${{env.NOW_UTC}}'
266+
- name: run MLTransform Image Embedding CPU Batch
267+
uses: ./.github/actions/gradle-command-self-hosted-action
268+
timeout-minutes: 180
269+
with:
270+
gradle-command: :sdks:python:apache_beam:testing:load_tests:run
271+
arguments: |
272+
-PloadTest.mainClass=apache_beam.testing.benchmarks.inference.mltransform_image_embedding_benchmark \
273+
-Prunner=DataflowRunner \
274+
-PpythonVersion=3.10 \
275+
-PloadTest.requirementsTxtFile=apache_beam/ml/transforms/mltransform_embedding_tests_requirements.txt \
276+
'-PloadTest.args=${{ env.beam_Inference_Python_Benchmarks_Dataflow_test_arguments_15 }} --artifact_location=gs://temp-storage-for-perf-tests/loadtests/mltransform_image_embedding_cpu/artifacts-${{env.NOW_UTC}} --job_name=benchmark-tests-mltransform-image-embedding-cpu-batch-${{env.NOW_UTC}}'

‎.github/workflows/load-tests-pipeline-options/beam_Inference_Python_Benchmarks_Dataflow_MLTransform_Image_Embedding_CPU_Batch.txt‎

Lines changed: 42 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,42 @@
1+
# Licensed to the Apache Software Foundation (ASF) under one or more
2+
# contributor license agreements. See the NOTICE file distributed with
3+
# this work for additional information regarding copyright ownership.
4+
# The ASF licenses this file to You under the Apache License, Version 2.0
5+
# (the "License"); you may not use this file except in compliance with
6+
# the License. You may obtain a copy of the License at
7+
#
8+
# http://www.apache.org/licenses/LICENSE-2.0
9+
#
10+
# Unless required by applicable law or agreed to in writing, software
11+
# distributed under the License is distributed on an "AS IS" BASIS,
12+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
# See the License for the specific language governing permissions and
14+
# limitations under the License.
15+
16+
--runner=DataflowRunner
17+
--project=apache-beam-testing
18+
--region=us-central1
19+
--num_workers=5
20+
--max_num_workers=20
21+
--disk_size_gb=100
22+
--autoscaling_algorithm=THROUGHPUT_BASED
23+
--staging_location=gs://temp-storage-for-perf-tests/loadtests
24+
--temp_location=gs://temp-storage-for-perf-tests/loadtests
25+
--requirements_file=apache_beam/ml/transforms/mltransform_embedding_tests_requirements.txt
26+
--publish_to_big_query=true
27+
--metrics_dataset=beam_run_inference
28+
--metrics_table=mltransform_image_embedding_cpu_batch
29+
--input_options={}
30+
--influx_measurement=mltransform_image_embedding_cpu_batch
31+
--mode=batch
32+
--input=gs://apache-beam-ml/testing/inputs/openimage_50k_benchmark.txt
33+
--artifact_location=gs://temp-storage-for-perf-tests/loadtests/mltransform_image_embedding_cpu/artifacts
34+
--output_table=apache-beam-testing:beam_run_inference.result_mltransform_image_embedding_cpu_batch
35+
--pretrained_model_name=clip-ViT-B-32
36+
--device=CPU
37+
--min_batch_size=8
38+
--max_batch_size=64
39+
--embedding_min_ram=16GB
40+
--dataflow_service_options=enable_prime
41+
--experiments=use_runner_v2
42+
--timeout_ms=3600000

‎.github/workflows/load-tests-pipeline-options/beam_Inference_Python_Benchmarks_Dataflow_MLTransform_Image_Embedding_GPU_Batch.txt‎

Lines changed: 44 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,44 @@
1+
# Licensed to the Apache Software Foundation (ASF) under one or more
2+
# contributor license agreements. See the NOTICE file distributed with
3+
# this work for additional information regarding copyright ownership.
4+
# The ASF licenses this file to You under the Apache License, Version 2.0
5+
# (the "License"); you may not use this file except in compliance with
6+
# the License. You may obtain a copy of the License at
7+
#
8+
# http://www.apache.org/licenses/LICENSE-2.0
9+
#
10+
# Unless required by applicable law or agreed to in writing, software
11+
# distributed under the License is distributed on an "AS IS" BASIS,
12+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
# See the License for the specific language governing permissions and
14+
# limitations under the License.
15+
16+
--runner=DataflowRunner
17+
--project=apache-beam-testing
18+
--region=us-central1
19+
--num_workers=5
20+
--max_num_workers=20
21+
--disk_size_gb=100
22+
--autoscaling_algorithm=THROUGHPUT_BASED
23+
--staging_location=gs://temp-storage-for-perf-tests/loadtests
24+
--temp_location=gs://temp-storage-for-perf-tests/loadtests
25+
--requirements_file=apache_beam/ml/transforms/mltransform_embedding_tests_requirements.txt
26+
--publish_to_big_query=true
27+
--metrics_dataset=beam_run_inference
28+
--metrics_table=mltransform_image_embedding_gpu_batch
29+
--input_options={}
30+
--influx_measurement=mltransform_image_embedding_gpu_batch
31+
--mode=batch
32+
--input=gs://apache-beam-ml/testing/inputs/openimage_50k_benchmark.txt
33+
--artifact_location=gs://temp-storage-for-perf-tests/loadtests/mltransform_image_embedding_gpu/artifacts
34+
--output_table=apache-beam-testing:beam_run_inference.result_mltransform_image_embedding_gpu_batch
35+
--pretrained_model_name=clip-ViT-B-32
36+
--device=GPU
37+
--min_batch_size=8
38+
--max_batch_size=64
39+
--embedding_accelerator=type:nvidia-tesla-t4;count:1;install-nvidia-driver
40+
--sdk_location=container
41+
--sdk_container_image=us.gcr.io/apache-beam-testing/python-postcommit-it/tensor_rt:latest
42+
--dataflow_service_options=enable_prime
43+
--experiments=use_runner_v2
44+
--timeout_ms=3600000

‎.github/workflows/load-tests-pipeline-options/beam_Inference_Python_Benchmarks_Dataflow_MLTransform_Text_Embedding_Batch.txt‎

Lines changed: 42 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,42 @@
1+
# Licensed to the Apache Software Foundation (ASF) under one or more
2+
# contributor license agreements. See the NOTICE file distributed with
3+
# this work for additional information regarding copyright ownership.
4+
# The ASF licenses this file to You under the Apache License, Version 2.0
5+
# (the "License"); you may not use this file except in compliance with
6+
# the License. You may obtain a copy of the License at
7+
#
8+
# http://www.apache.org/licenses/LICENSE-2.0
9+
#
10+
# Unless required by applicable law or agreed to in writing, software
11+
# distributed under the License is distributed on an "AS IS" BASIS,
12+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
# See the License for the specific language governing permissions and
14+
# limitations under the License.
15+
16+
--runner=DataflowRunner
17+
--project=apache-beam-testing
18+
--region=us-central1
19+
--machine_type=n1-standard-4
20+
--num_workers=10
21+
--max_num_workers=50
22+
--disk_size_gb=100
23+
--autoscaling_algorithm=THROUGHPUT_BASED
24+
--staging_location=gs://temp-storage-for-perf-tests/loadtests
25+
--temp_location=gs://temp-storage-for-perf-tests/loadtests
26+
--requirements_file=apache_beam/ml/transforms/mltransform_embedding_tests_requirements.txt
27+
--publish_to_big_query=true
28+
--metrics_dataset=beam_run_inference
29+
--metrics_table=mltransform_text_embedding_batch
30+
--input_options={}
31+
--influx_measurement=mltransform_text_embedding_batch
32+
--input_file=gs://apache-beam-ml/testing/inputs/sentences_50k.txt
33+
--output=gs://temp-storage-for-perf-tests/loadtests/mltransform_text_embedding/results
34+
--artifact_location=gs://temp-storage-for-perf-tests/loadtests/mltransform_text_embedding/artifacts
35+
--model_name=sentence-transformers/all-MiniLM-L6-v2
36+
--min_batch_size=16
37+
--max_batch_size=128
38+
--model_batch_size=32
39+
--device=CPU
40+
--sdk_location=container
41+
--sdk_container_image=us.gcr.io/apache-beam-testing/python-postcommit-it/tensor_rt:latest
42+
--experiments=use_runner_v2

‎.test-infra/tools/refresh_looker_metrics.py‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -46,7 +46,10 @@
4646
("96", ["270", "304", "305", "353", "354"]), # Table Row Inference Sklearn Batch
4747
("106", ["355", "356", "357", "358", "359"]), # Table Row Inference Sklearn Streaming
4848
("107", ["360", "361", "362", "363", "364"]), # MLTransform Generate Vocab Batch
49-
("108", ["365", "366", "367", "368", "369"]) # MLTransform One-Hot Encoding Batch
49+
("108", ["365", "366", "367", "368", "369"]), # MLTransform One-Hot Encoding Batch
50+
("109", ["375", "376", "377", "378", "379"]), # MLTransform Text Embedding Batch
51+
("110", ["380", "381", "382", "383", "385"]), # MLTransform Image Embedding GPU Batch
52+
("111", ["370", "371", "372", "373", "374"]), # MLTransform Image Embedding CPU Batch
5053
]
5154

5255
def get_look(id: str) -> models.Look:

0 commit comments

Comments
 (0)