Skip to content

Commit b513d66

Browse files
Amar3ttoAmar3tto
committed
fix
1 parent b0e4997 commit b513d66

3 files changed

Lines changed: 63 additions & 65 deletions

File tree

‎.github/workflows/beam_Inference_Python_Benchmarks_Dataflow.yml‎

Lines changed: 62 additions & 63 deletions
Original file line numberDiff line numberDiff line change
@@ -108,135 +108,134 @@ jobs:
108108
image_tag: ${{ github.sha }}
109109
- name: Run VLLM Gemma Batch Test
110110
uses: ./.github/actions/gradle-command-self-hosted-action
111-
timeout-minutes: 270
111+
timeout-minutes: 180
112112
with:
113113
gradle-command: :sdks:python:apache_beam:testing:load_tests:run
114114
arguments: |
115-
-PloadTest.mainClass=apache_beam.testing.benchmarks.inference.vllm_gemma_benchmarks
116-
-Prunner=DataflowRunner
117-
-PsdkLocationOverride=false
118-
-PpythonVersion=3.10
119-
-PloadTest.requirementsTxtFile=apache_beam/ml/inference/vllm_tests_requirements.txt
120-
-PloadTest.args="${{ env.beam_Inference_Python_Benchmarks_Dataflow_test_arguments_8 }} --mode=batch --job_name=benchmark-tests-vllm-with-gemma-2b-it-batch-${{env.NOW_UTC}} --sdk_container_image=${{ steps.build_vllm_image.outputs.image_url }}"
115+
-PloadTest.mainClass=apache_beam.testing.benchmarks.inference.vllm_gemma_benchmarks \
116+
-Prunner=DataflowRunner \
117+
-PsdkLocationOverride=false \
118+
-PpythonVersion=3.10 \
119+
-PloadTest.requirementsTxtFile=apache_beam/ml/inference/vllm_tests_requirements.txt '-PloadTest.args=${{ env.beam_Inference_Python_Benchmarks_Dataflow_test_arguments_8 }} --mode=batch --job_name=benchmark-tests-vllm-with-gemma-2b-it-batch-${{env.NOW_UTC}} --sdk_container_image=${{ steps.build_vllm_image.outputs.image_url }}'
121120
- name: run Pytorch Sentiment Streaming using Hugging Face distilbert-base-uncased model
122121
uses: ./.github/actions/gradle-command-self-hosted-action
123122
timeout-minutes: 180
124123
with:
125124
gradle-command: :sdks:python:apache_beam:testing:load_tests:run
126125
arguments: |
127-
-PloadTest.mainClass=apache_beam.testing.benchmarks.inference.pytorch_sentiment_benchmarks
128-
-Prunner=DataflowRunner
129-
-PpythonVersion=3.10
130-
-PloadTest.requirementsTxtFile=apache_beam/ml/inference/torch_tests_requirements.txt
131-
-PloadTest.args="${{ env.beam_Inference_Python_Benchmarks_Dataflow_test_arguments_6 }} --mode=streaming --job_name=benchmark-tests-pytorch-sentiment-distilbert-base-uncased-streaming-${{env.NOW_UTC}} --output_table=apache-beam-testing.beam_run_inference.result_sentiment_distilbert_base_uncased"
126+
-PloadTest.mainClass=apache_beam.testing.benchmarks.inference.pytorch_sentiment_benchmarks \
127+
-Prunner=DataflowRunner \
128+
-PpythonVersion=3.10 \
129+
-PloadTest.requirementsTxtFile=apache_beam/ml/inference/torch_tests_requirements.txt \
130+
'-PloadTest.args=${{ env.beam_Inference_Python_Benchmarks_Dataflow_test_arguments_6 }} --mode=streaming --job_name=benchmark-tests-pytorch-sentiment-distilbert-base-uncased-streaming-${{env.NOW_UTC}} --output_table=apache-beam-testing.beam_run_inference.result_sentiment_distilbert_base_uncased' \
132131
- name: run Pytorch Sentiment Batch using Hugging Face distilbert-base-uncased model
133132
uses: ./.github/actions/gradle-command-self-hosted-action
134133
timeout-minutes: 180
135134
with:
136135
gradle-command: :sdks:python:apache_beam:testing:load_tests:run
137136
arguments: |
138-
-PloadTest.mainClass=apache_beam.testing.benchmarks.inference.pytorch_sentiment_benchmarks
139-
-Prunner=DataflowRunner
140-
-PpythonVersion=3.10
141-
-PloadTest.requirementsTxtFile=apache_beam/ml/inference/torch_tests_requirements.txt
142-
-PloadTest.args="${{ env.beam_Inference_Python_Benchmarks_Dataflow_test_arguments_7 }} --mode=batch --job_name=benchmark-tests-pytorch-sentiment-distilbert-base-uncased-batch-${{env.NOW_UTC}} --output_table=apache-beam-testing.beam_run_inference.result_sentiment_distilbert_base_uncased"
137+
-PloadTest.mainClass=apache_beam.testing.benchmarks.inference.pytorch_sentiment_benchmarks \
138+
-Prunner=DataflowRunner \
139+
-PpythonVersion=3.10 \
140+
-PloadTest.requirementsTxtFile=apache_beam/ml/inference/torch_tests_requirements.txt \
141+
'-PloadTest.args=${{ env.beam_Inference_Python_Benchmarks_Dataflow_test_arguments_7 }} --mode=batch --job_name=benchmark-tests-pytorch-sentiment-distilbert-base-uncased-batch-${{env.NOW_UTC}} --output_table=apache-beam-testing.beam_run_inference.result_sentiment_distilbert_base_uncased' \
143142
- name: run Pytorch Vision Classification with Resnet 101
144143
uses: ./.github/actions/gradle-command-self-hosted-action
145144
timeout-minutes: 180
146145
with:
147146
gradle-command: :sdks:python:apache_beam:testing:load_tests:run
148147
arguments: |
149-
-PloadTest.mainClass=apache_beam.testing.benchmarks.inference.pytorch_image_classification_benchmarks
150-
-Prunner=DataflowRunner
151-
-PpythonVersion=3.10
152-
-PloadTest.requirementsTxtFile=apache_beam/ml/inference/torch_tests_requirements.txt
153-
-PloadTest.args="${{ env.beam_Inference_Python_Benchmarks_Dataflow_test_arguments_1 }} --job_name=benchmark-tests-pytorch-imagenet-python-101-${{env.NOW_UTC}} --output=gs://temp-storage-for-end-to-end-tests/torch/result_resnet101-${{env.NOW_UTC}}.txt"
148+
-PloadTest.mainClass=apache_beam.testing.benchmarks.inference.pytorch_image_classification_benchmarks \
149+
-Prunner=DataflowRunner \
150+
-PpythonVersion=3.10 \
151+
-PloadTest.requirementsTxtFile=apache_beam/ml/inference/torch_tests_requirements.txt \
152+
'-PloadTest.args=${{ env.beam_Inference_Python_Benchmarks_Dataflow_test_arguments_1 }} --job_name=benchmark-tests-pytorch-imagenet-python-101-${{env.NOW_UTC}} --output=gs://temp-storage-for-end-to-end-tests/torch/result_resnet101-${{env.NOW_UTC}}.txt' \
154153
- name: run Pytorch Imagenet Classification with Resnet 152
155154
uses: ./.github/actions/gradle-command-self-hosted-action
156155
timeout-minutes: 180
157156
with:
158157
gradle-command: :sdks:python:apache_beam:testing:load_tests:run
159158
arguments: |
160-
-PloadTest.mainClass=apache_beam.testing.benchmarks.inference.pytorch_image_classification_benchmarks
161-
-Prunner=DataflowRunner
162-
-PpythonVersion=3.10
163-
-PloadTest.requirementsTxtFile=apache_beam/ml/inference/torch_tests_requirements.txt
164-
-PloadTest.args="${{ env.beam_Inference_Python_Benchmarks_Dataflow_test_arguments_2 }} --job_name=benchmark-tests-pytorch-imagenet-python-152-${{env.NOW_UTC}} --output=gs://temp-storage-for-end-to-end-tests/torch/result_resnet152-${{env.NOW_UTC}}.txt"
159+
-PloadTest.mainClass=apache_beam.testing.benchmarks.inference.pytorch_image_classification_benchmarks \
160+
-Prunner=DataflowRunner \
161+
-PpythonVersion=3.10 \
162+
-PloadTest.requirementsTxtFile=apache_beam/ml/inference/torch_tests_requirements.txt \
163+
'-PloadTest.args=${{ env.beam_Inference_Python_Benchmarks_Dataflow_test_arguments_2 }} --job_name=benchmark-tests-pytorch-imagenet-python-152-${{env.NOW_UTC}} --output=gs://temp-storage-for-end-to-end-tests/torch/result_resnet152-${{env.NOW_UTC}}.txt' \
165164
- name: run Pytorch Language Modeling using Hugging Face bert-base-uncased model
166165
uses: ./.github/actions/gradle-command-self-hosted-action
167166
timeout-minutes: 180
168167
with:
169168
gradle-command: :sdks:python:apache_beam:testing:load_tests:run
170169
arguments: |
171-
-PloadTest.mainClass=apache_beam.testing.benchmarks.inference.pytorch_language_modeling_benchmarks
172-
-Prunner=DataflowRunner
173-
-PpythonVersion=3.10
174-
-PloadTest.requirementsTxtFile=apache_beam/ml/inference/torch_tests_requirements.txt
175-
-PloadTest.args="${{ env.beam_Inference_Python_Benchmarks_Dataflow_test_arguments_3 }} --job_name=benchmark-tests-pytorch-language-modeling-bert-base-uncased-${{env.NOW_UTC}} --output=gs://temp-storage-for-end-to-end-tests/torch/result_bert_base_uncased-${{env.NOW_UTC}}.txt"
170+
-PloadTest.mainClass=apache_beam.testing.benchmarks.inference.pytorch_language_modeling_benchmarks \
171+
-Prunner=DataflowRunner \
172+
-PpythonVersion=3.10 \
173+
-PloadTest.requirementsTxtFile=apache_beam/ml/inference/torch_tests_requirements.txt \
174+
'-PloadTest.args=${{ env.beam_Inference_Python_Benchmarks_Dataflow_test_arguments_3 }} --job_name=benchmark-tests-pytorch-language-modeling-bert-base-uncased-${{env.NOW_UTC}} --output=gs://temp-storage-for-end-to-end-tests/torch/result_bert_base_uncased-${{env.NOW_UTC}}.txt' \
176175
- name: run Pytorch Langauge Modeling using Hugging Face bert-large-uncased model
177176
uses: ./.github/actions/gradle-command-self-hosted-action
178177
timeout-minutes: 180
179178
with:
180179
gradle-command: :sdks:python:apache_beam:testing:load_tests:run
181180
arguments: |
182-
-PloadTest.mainClass=apache_beam.testing.benchmarks.inference.pytorch_language_modeling_benchmarks
183-
-Prunner=DataflowRunner
184-
-PpythonVersion=3.10
185-
-PloadTest.requirementsTxtFile=apache_beam/ml/inference/torch_tests_requirements.txt
186-
-PloadTest.args="${{ env.beam_Inference_Python_Benchmarks_Dataflow_test_arguments_4 }} --job_name=benchmark-tests-pytorch-language-modeling-bert-large-uncased-${{env.NOW_UTC}} --output=gs://temp-storage-for-end-to-end-tests/torch/result_bert_large_uncased-${{env.NOW_UTC}}.txt"
181+
-PloadTest.mainClass=apache_beam.testing.benchmarks.inference.pytorch_language_modeling_benchmarks \
182+
-Prunner=DataflowRunner \
183+
-PpythonVersion=3.10 \
184+
-PloadTest.requirementsTxtFile=apache_beam/ml/inference/torch_tests_requirements.txt \
185+
'-PloadTest.args=${{ env.beam_Inference_Python_Benchmarks_Dataflow_test_arguments_4 }} --job_name=benchmark-tests-pytorch-language-modeling-bert-large-uncased-${{env.NOW_UTC}} --output=gs://temp-storage-for-end-to-end-tests/torch/result_bert_large_uncased-${{env.NOW_UTC}}.txt' \
187186
- name: run Pytorch Imagenet Classification with Resnet 152 with Tesla T4 GPU
188187
uses: ./.github/actions/gradle-command-self-hosted-action
189188
timeout-minutes: 180
190189
with:
191190
gradle-command: :sdks:python:apache_beam:testing:load_tests:run
192191
arguments: |
193-
-PloadTest.mainClass=apache_beam.testing.benchmarks.inference.pytorch_image_classification_benchmarks
194-
-Prunner=DataflowRunner
195-
-PpythonVersion=3.10
196-
-PloadTest.requirementsTxtFile=apache_beam/ml/inference/torch_tests_requirements.txt
197-
-PloadTest.args="${{ env.beam_Inference_Python_Benchmarks_Dataflow_test_arguments_5 }} --job_name=benchmark-tests-pytorch-imagenet-python-gpu-${{env.NOW_UTC}} --output=gs://temp-storage-for-end-to-end-tests/torch/result_resnet152_gpu-${{env.NOW_UTC}}.txt"
192+
-PloadTest.mainClass=apache_beam.testing.benchmarks.inference.pytorch_image_classification_benchmarks \
193+
-Prunner=DataflowRunner \
194+
-PpythonVersion=3.10 \
195+
-PloadTest.requirementsTxtFile=apache_beam/ml/inference/torch_tests_requirements.txt \
196+
'-PloadTest.args=${{ env.beam_Inference_Python_Benchmarks_Dataflow_test_arguments_5 }} --job_name=benchmark-tests-pytorch-imagenet-python-gpu-${{env.NOW_UTC}} --output=gs://temp-storage-for-end-to-end-tests/torch/result_resnet152_gpu-${{env.NOW_UTC}}.txt'
198197
- name: run Table Row Inference Sklearn Batch
199198
uses: ./.github/actions/gradle-command-self-hosted-action
200199
timeout-minutes: 180
201200
with:
202201
gradle-command: :sdks:python:apache_beam:testing:load_tests:run
203202
arguments: |
204-
-PloadTest.mainClass=apache_beam.testing.benchmarks.inference.table_row_inference_benchmark
205-
-Prunner=DataflowRunner
206-
-PpythonVersion=3.10
207-
-PloadTest.requirementsTxtFile=apache_beam/ml/inference/table_row_inference_requirements.txt
208-
-PloadTest.args="${{ env.beam_Inference_Python_Benchmarks_Dataflow_test_arguments_9 }} --autoscaling_algorithm=NONE --metrics_table=result_table_row_inference_batch --influx_measurement=result_table_row_inference_batch --mode=batch --input_file=gs://apache-beam-ml/testing/inputs/table_rows_100k_benchmark.jsonl --input_expand_factor=100 --output_table=apache-beam-testing:beam_run_inference.result_table_row_inference_batch_outputs --job_name=benchmark-tests-table-row-inference-batch-${{env.NOW_UTC}}"
203+
-PloadTest.mainClass=apache_beam.testing.benchmarks.inference.table_row_inference_benchmark \
204+
-Prunner=DataflowRunner \
205+
-PpythonVersion=3.10 \
206+
-PloadTest.requirementsTxtFile=apache_beam/ml/inference/table_row_inference_requirements.txt \
207+
'-PloadTest.args=${{ env.beam_Inference_Python_Benchmarks_Dataflow_test_arguments_9 }} --autoscaling_algorithm=NONE --metrics_table=result_table_row_inference_batch --influx_measurement=result_table_row_inference_batch --mode=batch --input_file=gs://apache-beam-ml/testing/inputs/table_rows_100k_benchmark.jsonl --input_expand_factor=100 --output_table=apache-beam-testing:beam_run_inference.result_table_row_inference_batch_outputs --job_name=benchmark-tests-table-row-inference-batch-${{env.NOW_UTC}}'
209208
- name: run Table Row Inference Sklearn Stream
210209
uses: ./.github/actions/gradle-command-self-hosted-action
211210
timeout-minutes: 180
212211
with:
213212
gradle-command: :sdks:python:apache_beam:testing:load_tests:run
214213
arguments: |
215-
-PloadTest.mainClass=apache_beam.testing.benchmarks.inference.table_row_inference_benchmark
216-
-Prunner=DataflowRunner
217-
-PpythonVersion=3.10
218-
-PloadTest.requirementsTxtFile=apache_beam/ml/inference/table_row_inference_requirements.txt
219-
-PloadTest.args="${{ env.beam_Inference_Python_Benchmarks_Dataflow_test_arguments_10 }} --autoscaling_algorithm=THROUGHPUT_BASED --max_num_workers=20 --metrics_table=result_table_row_inference_stream --influx_measurement=result_table_row_inference_stream --mode=streaming --input_subscription=projects/apache-beam-testing/subscriptions/table_row_inference_benchmark --window_size_sec=60 --trigger_interval_sec=30 --timeout_ms=900000 --output_table=apache-beam-testing:beam_run_inference.result_table_row_inference_stream_outputs --job_name=benchmark-tests-table-row-inference-stream-${{env.NOW_UTC}}"
214+
-PloadTest.mainClass=apache_beam.testing.benchmarks.inference.table_row_inference_benchmark \
215+
-Prunner=DataflowRunner \
216+
-PpythonVersion=3.10 \
217+
-PloadTest.requirementsTxtFile=apache_beam/ml/inference/table_row_inference_requirements.txt \
218+
'-PloadTest.args=${{ env.beam_Inference_Python_Benchmarks_Dataflow_test_arguments_10 }} --autoscaling_algorithm=THROUGHPUT_BASED --max_num_workers=20 --metrics_table=result_table_row_inference_stream --influx_measurement=result_table_row_inference_stream --mode=streaming --input_subscription=projects/apache-beam-testing/subscriptions/table_row_inference_benchmark --window_size_sec=60 --trigger_interval_sec=30 --timeout_ms=900000 --output_table=apache-beam-testing:beam_run_inference.result_table_row_inference_stream_outputs --job_name=benchmark-tests-table-row-inference-stream-${{env.NOW_UTC}}'
220219
- name: run MLTransform Generate Vocab Batch
221220
uses: ./.github/actions/gradle-command-self-hosted-action
222221
timeout-minutes: 180
223222
with:
224223
gradle-command: :sdks:python:apache_beam:testing:load_tests:run
225224
arguments: |
226-
-PloadTest.mainClass=apache_beam.testing.benchmarks.inference.mltransform_generate_vocab_benchmark
227-
-Prunner=DataflowRunner
228-
-PpythonVersion=3.10
229-
-PloadTest.requirementsTxtFile=apache_beam/examples/ml_transform/mltransform_generate_vocab_requirements.txt
230-
-PloadTest.args="${{ env.beam_Inference_Python_Benchmarks_Dataflow_test_arguments_11 }} --job_name=benchmark-tests-mltransform-generate-vocab-batch-${{env.NOW_UTC}}"
225+
-PloadTest.mainClass=apache_beam.testing.benchmarks.inference.mltransform_generate_vocab_benchmark \
226+
-Prunner=DataflowRunner \
227+
-PpythonVersion=3.10 \
228+
-PloadTest.requirementsTxtFile=apache_beam/examples/ml_transform/mltransform_generate_vocab_requirements.txt \
229+
'-PloadTest.args=${{ env.beam_Inference_Python_Benchmarks_Dataflow_test_arguments_11 }} --job_name=benchmark-tests-mltransform-generate-vocab-batch-${{env.NOW_UTC}}'
231230
- name: run MLTransform One-Hot Encoding Batch
232231
uses: ./.github/actions/gradle-command-self-hosted-action
233232
timeout-minutes: 180
234233
with:
235234
gradle-command: :sdks:python:apache_beam:testing:load_tests:run
236235
arguments: |
237-
-PloadTest.mainClass=apache_beam.testing.benchmarks.inference.mltransform_one_hot_encoding_benchmark
238-
-Prunner=DataflowRunner
239-
-PpythonVersion=3.10
240-
-PbeamPythonExtra=ml_test
241-
-PloadTest.requirementsTxtFile=apache_beam/ml/transforms/mltransform_tests_requirements.txt
242-
-PloadTest.args="${{ env.beam_Inference_Python_Benchmarks_Dataflow_test_arguments_12 }} --autoscaling_algorithm=NONE --metrics_table=mltransform_one_hot_encoding_batch --influx_measurement=mltransform_one_hot_encoding_batch --job_name=benchmark-tests-mltransform-one-hot-encoding-batch-${{env.NOW_UTC}} --output_file=gs://temp-storage-for-end-to-end-tests/mltransform/one_hot_output_${{env.NOW_UTC}} --artifact_location=gs://temp-storage-for-end-to-end-tests/mltransform/artifacts_${{env.NOW_UTC}}"
236+
-PloadTest.mainClass=apache_beam.testing.benchmarks.inference.mltransform_one_hot_encoding_benchmark \
237+
-Prunner=DataflowRunner \
238+
-PpythonVersion=3.10 \
239+
-PbeamPythonExtra=ml_test \
240+
-PloadTest.requirementsTxtFile=apache_beam/ml/transforms/mltransform_tests_requirements.txt \
241+
'-PloadTest.args=${{ env.beam_Inference_Python_Benchmarks_Dataflow_test_arguments_12 }} --autoscaling_algorithm=NONE --metrics_table=mltransform_one_hot_encoding_batch --influx_measurement=mltransform_one_hot_encoding_batch --job_name=benchmark-tests-mltransform-one-hot-encoding-batch-${{env.NOW_UTC}} --output_file=gs://temp-storage-for-end-to-end-tests/mltransform/one_hot_output_${{env.NOW_UTC}} --artifact_location=gs://temp-storage-for-end-to-end-tests/mltransform/artifacts_${{env.NOW_UTC}}'

‎.github/workflows/load-tests-pipeline-options/beam_Inference_Python_Benchmarks_Dataflow_VLLM_Gemma_Batch.txt‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -35,4 +35,3 @@
3535
--requirements_file=apache_beam/ml/inference/vllm_tests_requirements.txt
3636
--dataflow_service_options=worker_accelerator=type:nvidia-tesla-t4;count:1;install-nvidia-driver
3737
--experiments=use_runner_v2
38-
--timeout_ms=16200000

‎sdks/python/apache_beam/testing/load_tests/build.gradle‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -55,7 +55,7 @@ task run(type: Exec, dependsOn: installGcpTest) {
5555
exec {
5656
setWorkingDir "${project.rootDir}/sdks/python"
5757
executable 'sh'
58-
args '-c', ". ${project.ext.envdir}/bin/activate && pip install -r ${requirementsTxtFileArg}"
58+
args '-c', "${project.ext.envdir}/bin/python -m pip install -r ${requirementsTxtFileArg}"
5959
}
6060
}
6161
}

0 commit comments

Comments
 (0)