Skip to content

Commit 942e3fa

Browse files
chamikaramjchamikaramj
authored
Adds multi-language Python and Java quickstart release validations (#25910)
* Adds multi-language Python and Java quickstart release validations * Addressing reviewer comments
1 parent b2bd2a8 commit 942e3fa

3 files changed

Lines changed: 262 additions & 11 deletions

File tree

‎release/src/main/scripts/run_rc_validation.sh‎

Lines changed: 248 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -34,6 +34,10 @@ function clean_up(){
3434
echo "====================Final Steps===================="
3535
echo "-----------------Stopping Pubsub Java Injector-----------------"
3636
echo "Please stop java injector manually."
37+
echo "-----------------Stopping multi-language quickstart services-----------------"
38+
echo "Please stop the Java expansion service manually."
39+
echo "Please stop the Python expansion service manually."
40+
echo "Please stop the Python portable runner Job server manually."
3741
echo "-----------------Signing up Spreadsheet-----------------"
3842
echo "Please open this spreadsheet: https://s.apache.org/beam-release-validation"
3943
echo "Please sign up your name in the tests you have ran."
@@ -216,6 +220,19 @@ if [[ -z `which kubectl` ]]; then
216220
fi
217221
kubectl version
218222

223+
if [[ ("$python_xlang_quickstart" = true) \
224+
|| ("$java_xlang_quickstart" = true) ]]; then
225+
echo "[Confirmation Required] Multi-language quickstart tests require generating
226+
final Docker tags for several candidate Docker images. This should be safe to do
227+
so since these tags will be overridden during the Docker image finalization step
228+
of the Beam release.
229+
Continue with the release validation ? [y|N]"
230+
read confirmation
231+
if [[ $confirmation != "y" ]]; then
232+
echo "Cannot continue with the validation. Exiting."
233+
exit
234+
fi
235+
fi
219236

220237
echo ""
221238
echo "====================Starting Python Quickstart and MobileGame==================="
@@ -484,8 +501,236 @@ else
484501
echo "* Skipping Python Leaderboard & GameStates Validations"
485502
fi
486503

504+
# Setting up Docker images for multi-language quickstart tests.
505+
if [[ ("$python_xlang_quickstart" = true) \
506+
|| ("$java_xlang_quickstart" = true) ]]; then
507+
echo ""
508+
echo "====================Generating Docker tags for multi-language quickstart tests==============="
509+
510+
RC_DOCKER_TAG=${RELEASE_VER}rc${RC_NUM}
511+
FINAL_DOCKER_TAG=${RELEASE_VER}
512+
513+
for py_version in "${PYTHON_VERSIONS_TO_VALIDATE[@]}"
514+
do
515+
PYTHON_DOCKER_IMAGE_REPO=apache/beam_${py_version}_sdk
516+
PYTHON_RC_DOCKER_IMAGE=${PYTHON_DOCKER_IMAGE_REPO}:${RC_DOCKER_TAG}
517+
PYTHON_FINAL_DOCKER_IMAGE=${PYTHON_DOCKER_IMAGE_REPO}:${FINAL_DOCKER_TAG}
518+
docker pull ${PYTHON_DOCKER_IMAGE_REPO}:${RC_DOCKER_TAG}
519+
echo "Creating Docker tag ${FINAL_DOCKER_TAG} from image ${PYTHON_RC_DOCKER_IMAGE}"
520+
docker tag ${PYTHON_RC_DOCKER_IMAGE} ${PYTHON_FINAL_DOCKER_IMAGE}
521+
done
522+
523+
JAVA_DOCKER_IMAGE_REPO=apache/beam_java11_sdk # Using the default Java version.
524+
JAVA_RC_DOCKER_IMAGE=${JAVA_DOCKER_IMAGE_REPO}:${RC_DOCKER_TAG}
525+
JAVA_FINAL_DOCKER_IMAGE=${JAVA_DOCKER_IMAGE_REPO}:${FINAL_DOCKER_TAG}
526+
docker pull ${JAVA_DOCKER_IMAGE_REPO}:${RC_DOCKER_TAG}
527+
echo "Creating Docker tag ${FINAL_DOCKER_TAG} from image ${JAVA_RC_DOCKER_IMAGE}"
528+
docker tag ${JAVA_RC_DOCKER_IMAGE} ${JAVA_FINAL_DOCKER_IMAGE}
529+
fi
530+
531+
echo ""
532+
echo "====================Starting Python Multi-language Quickstart Validations==============="
533+
if [[ ("$python_xlang_quickstart" = true) \
534+
&& ! -z `which gnome-terminal` && ! -z `which kubectl` ]]; then
535+
cd ${LOCAL_BEAM_DIR}
536+
537+
echo "---------------------Downloading Python Staging RC----------------------------"
538+
wget ${PYTHON_RC_DOWNLOAD_URL}/${RELEASE_VER}/python/apache-beam-${RELEASE_VER}.zip
539+
wget ${PYTHON_RC_DOWNLOAD_URL}/${RELEASE_VER}/python/apache-beam-${RELEASE_VER}.zip.sha512
540+
if [[ ! -f apache-beam-${RELEASE_VER}.zip ]]; then
541+
{ echo "Failed to download Python Staging RC files." ;exit 1; }
542+
fi
543+
544+
echo "--------------------------Verifying Hashes------------------------------------"
545+
sha512sum -c apache-beam-${RELEASE_VER}.zip.sha512
546+
547+
`which pip` install --upgrade pip
548+
`which pip` install --upgrade setuptools
549+
550+
echo "-----------------------Setting up Shell Env Vars------------------------------"
551+
set_bashrc
552+
553+
# Run Python Multi-language pipelines under multiple versions of Python using DirectRunner
554+
cd ${LOCAL_BEAM_DIR}
555+
for py_version in "${PYTHON_VERSIONS_TO_VALIDATE[@]}"
556+
do
557+
rm -rf ./beam_env_${py_version}
558+
echo "--------------Setting up virtualenv with $py_version interpreter----------------"
559+
$py_version -m venv beam_env_${py_version}
560+
. ./beam_env_${py_version}/bin/activate
561+
pip install --upgrade pip setuptools wheel
562+
ln -s ${LOCAL_BEAM_DIR}/sdks beam_env_${py_version}/lib/sdks
563+
564+
echo "--------------------------Installing Python SDK-------------------------------"
565+
pip install apache-beam-${RELEASE_VER}.zip
566+
567+
echo '************************************************************';
568+
echo '* Running Python Multi-language Quickstart with DirectRunner';
569+
echo '************************************************************';
570+
571+
PYTHON_MULTILANG_QUICKSTART_FILE_PREFIX=python_multilang_quickstart
572+
PYTHON_MULTILANG_QUICKSTART_INPUT_FILE_NAME=${PYTHON_MULTILANG_QUICKSTART_FILE_PREFIX}_input
573+
PYTHON_MULTILANG_QUICKSTART_OUTPUT_FILE_NAME=${PYTHON_MULTILANG_QUICKSTART_FILE_PREFIX}_output
574+
PYTHON_MULTILANG_QUICKSTART_EXPECTED_OUTPUT_FILE_NAME=${PYTHON_MULTILANG_QUICKSTART_FILE_PREFIX}_expected_output
575+
PYTHON_MULTILANG_QUICKSTART_SORTED_OUTPUT_FILE_NAME=${PYTHON_MULTILANG_QUICKSTART_FILE_PREFIX}_sorted_output
576+
577+
# Cleaning up data from any previous runs.
578+
rm ${PYTHON_MULTILANG_QUICKSTART_FILE_PREFIX}*
579+
rm ./beam-examples-multi-language-${RELEASE_VER}.jar
580+
581+
# Generating an input file.
582+
input_data=( aaa bbb ccc ddd eee)
583+
584+
touch $PYTHON_MULTILANG_QUICKSTART_INPUT_FILE_NAME
585+
touch $PYTHON_MULTILANG_QUICKSTART_EXPECTED_OUTPUT_FILE_NAME
586+
587+
for item in ${input_data[*]}
588+
do
589+
echo $item >> $PYTHON_MULTILANG_QUICKSTART_INPUT_FILE_NAME
590+
echo python:java:$item >> $PYTHON_MULTILANG_QUICKSTART_EXPECTED_OUTPUT_FILE_NAME
591+
done
592+
593+
# Downloading the expansion service jar.
594+
wget ${REPO_URL}/org/apache/beam/beam-examples-multi-language/${RELEASE_VER}/beam-examples-multi-language-${RELEASE_VER}.jar
595+
JAVA_EXPANSION_SERVICE_PORT=33333
596+
597+
# Starting up the expansion service in a seperate shell.
598+
echo "A new terminal will pop up and start a java expansion service."
599+
gnome-terminal -x sh -c \
600+
"echo '******************************************************';
601+
echo '* Running Java expansion service in port ${JAVA_EXPANSION_SERVICE_PORT}';
602+
echo '******************************************************';
603+
java -jar ./beam-examples-multi-language-${RELEASE_VER}.jar ${JAVA_EXPANSION_SERVICE_PORT};
604+
exec bash"
605+
606+
echo "Sleeping 10 seconds for the expansion service to start up."
607+
sleep 10s
608+
609+
# Running the pipeline
610+
python ${LOCAL_BEAM_DIR}/examples/multi-language/python/addprefix.py \
611+
--runner DirectRunner \
612+
--environment_type=DOCKER \
613+
--input $PYTHON_MULTILANG_QUICKSTART_INPUT_FILE_NAME \
614+
--output $PYTHON_MULTILANG_QUICKSTART_OUTPUT_FILE_NAME \
615+
--expansion_service_port $JAVA_EXPANSION_SERVICE_PORT
616+
617+
# Validating output
618+
cat ${PYTHON_MULTILANG_QUICKSTART_OUTPUT_FILE_NAME}* | sort >> ${PYTHON_MULTILANG_QUICKSTART_SORTED_OUTPUT_FILE_NAME}
619+
620+
if cmp --silent -- $PYTHON_MULTILANG_QUICKSTART_EXPECTED_OUTPUT_FILE_NAME $PYTHON_MULTILANG_QUICKSTART_SORTED_OUTPUT_FILE_NAME; then
621+
echo "Successfully validated Python multi-language quickstart example. No additional manual validation needed."
622+
else
623+
echo "Python multi-language quickstart output validation failed. Since the output of the pipeline did not match the expected output"
624+
echo "Expected output:\n"
625+
cat $PYTHON_MULTILANG_QUICKSTART_EXPECTED_OUTPUT_FILE_NAME
626+
echo "\n"
627+
echo "Pipeline output:\n"
628+
cat $PYTHON_MULTILANG_QUICKSTART_SORTED_OUTPUT_FILE_NAME
629+
echo "\n"
630+
exit 1
631+
fi
632+
done # Loop over Python versions.
633+
else
634+
echo "* Skipping Python Multi-language Quickstart Validations"
635+
fi
636+
637+
echo ""
638+
echo "====================Starting Java Multi-language Quickstart Validations==============="
639+
if [[ ("$java_xlang_quickstart" = true) \
640+
&& ! -z `which gnome-terminal` && ! -z `which kubectl` ]]; then
641+
cd ${LOCAL_BEAM_DIR}
642+
643+
echo "---------------------Downloading Python Staging RC----------------------------"
644+
wget ${PYTHON_RC_DOWNLOAD_URL}/${RELEASE_VER}/python/apache-beam-${RELEASE_VER}.zip
645+
wget ${PYTHON_RC_DOWNLOAD_URL}/${RELEASE_VER}/python/apache-beam-${RELEASE_VER}.zip.sha512
646+
if [[ ! -f apache-beam-${RELEASE_VER}.zip ]]; then
647+
{ echo "Failed to download Python Staging RC files." ;exit 1; }
648+
fi
649+
650+
echo "--------------------------Verifying Hashes------------------------------------"
651+
sha512sum -c apache-beam-${RELEASE_VER}.zip.sha512
652+
653+
`which pip` install --upgrade pip
654+
`which pip` install --upgrade setuptools
655+
656+
echo "-----------------------Setting up Shell Env Vars------------------------------"
657+
set_bashrc
658+
659+
# Run Java Multi-language pipelines under multiple versions of Python using DirectRunner
660+
cd ${LOCAL_BEAM_DIR}
661+
for py_version in "${PYTHON_VERSIONS_TO_VALIDATE[@]}"
662+
do
663+
rm -rf ./beam_env_${py_version}
664+
echo "--------------Setting up virtualenv with $py_version interpreter----------------"
665+
$py_version -m venv beam_env_${py_version}
666+
. ./beam_env_${py_version}/bin/activate
667+
pip install --upgrade pip setuptools wheel
668+
ln -s ${LOCAL_BEAM_DIR}/sdks beam_env_${py_version}/lib/sdks
669+
670+
echo "--------------------------Installing Python SDK-------------------------------"
671+
pip install apache-beam-${RELEASE_VER}.zip[dataframe]
672+
673+
# Deacrivating in the main shell. We will reactivate the virtual environment new shells
674+
# for the expansion service and the job server.
675+
deactivate
676+
677+
PYTHON_PORTABLE_RUNNER_JOB_SERVER_PORT=44443
678+
PYTHON_EXPANSION_SERVICE_PORT=44444
679+
680+
# Starting up the Job Server
681+
echo "A new terminal will pop up and start a Python PortableRunner job server."
682+
gnome-terminal -x sh -c \
683+
"echo '******************************************************';
684+
echo '* Running Python PortableRunner in port ${PYTHON_PORTABLE_RUNNER_JOB_SERVER_PORT}';
685+
echo '******************************************************';
686+
. ./beam_env_${py_version}/bin/activate;
687+
python -m apache_beam.runners.portability.local_job_service_main -p ${PYTHON_PORTABLE_RUNNER_JOB_SERVER_PORT};
688+
exec bash"
689+
690+
# Starting up the Python expansion service
691+
echo "A new terminal will pop up and start a Python expansion service."
692+
gnome-terminal -x sh -c \
693+
"echo '******************************************************';
694+
echo '* Running Python Portabexpansion service in port ${PYTHON_EXPANSION_SERVICE_PORT}';
695+
echo '******************************************************';
696+
. ./beam_env_${py_version}/bin/activate;
697+
python -m apache_beam.runners.portability.expansion_service_main --port=${PYTHON_EXPANSION_SERVICE_PORT} \
698+
--fully_qualified_name_glob=* \
699+
--pickle_library=cloudpickle;
700+
exec bash"
701+
702+
echo "Sleeping 10 seconds for the job server and the expansion service to start up."
703+
sleep 10s
704+
705+
echo '************************************************************';
706+
echo '* Running Java Multi-language Quickstart with DirectRunner';
707+
echo '************************************************************';
708+
709+
JAVA_MULTILANG_QUICKSTART_FILE_PREFIX=java_multilang_quickstart
710+
JAVA_MULTILANG_QUICKSTART_OUTPUT_FILE_NAME=${JAVA_MULTILANG_QUICKSTART_FILE_PREFIX}_output
711+
712+
./gradlew :examples:multi-language:pythonDataframeWordCount -Pver=${RELEASE_VER} -Prepourl=${REPO_URL} --args=" \
713+
--runner=PortableRunner \
714+
--jobEndpoint=localhost:${PYTHON_PORTABLE_RUNNER_JOB_SERVER_PORT} \
715+
--expansionService=localhost:${PYTHON_EXPANSION_SERVICE_PORT} \
716+
--output=${JAVA_MULTILANG_QUICKSTART_OUTPUT_FILE_NAME}"
717+
718+
# We cannot validate local output since
719+
# TODO: Write output to GCS and validate when Python portable runner can forward credentials to GCS appropriately.
720+
721+
java_xlang_quickstart_status=$?
722+
if [[ $java_xlang_quickstart_status -eq 0 ]]; then
723+
echo "Successfully completed Java multi-language quickstart example. No manual validation needed."
724+
else
725+
{ echo "Java multi-language quickstart failed since the pipeline execution failed." ;exit 1; }
726+
fi
727+
done # Loop over Python versions.
728+
else
729+
echo "* Skipping Java Multi-language Quickstart Validations"
730+
fi
731+
487732
echo ""
488-
echo "====================Starting Python Cross-language Validations==============="
733+
echo "====================Starting Python Multi-language Validations with DataflowRunner==============="
489734
if [[ ("$python_xlang_kafka_taxi_dataflow" = true
490735
|| "$python_xlang_sql_taxi_dataflow" = true) \
491736
&& ! -z `which gnome-terminal` && ! -z `which kubectl` ]]; then
@@ -519,7 +764,7 @@ if [[ ("$python_xlang_kafka_taxi_dataflow" = true
519764
echo "* Skipping Kafka cluster setup"
520765
fi
521766

522-
# Run Python XLang pipelines under multiple versions of Python
767+
# Run Python multi-language pipelines under multiple versions of Python using Dataflow Runner
523768
cd ${LOCAL_BEAM_DIR}
524769
for py_version in "${PYTHON_VERSIONS_TO_VALIDATE[@]}"
525770
do
@@ -630,7 +875,7 @@ if [[ ("$python_xlang_kafka_taxi_dataflow" = true
630875
fi
631876
done # Loop over Python versions.
632877
else
633-
echo "* Skipping Python Cross-language Validations"
878+
echo "* Skipping Python Multi-language Dataflow Validations"
634879
fi
635880
echo "*************************************************************"
636881
echo " NOTE: Streaming pipelines are not automatically canceled. "

‎release/src/main/scripts/script.config‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -52,6 +52,8 @@ RC_VALIDATE_CONFIGS=(
5252
python_leaderboard_dataflow
5353
python_gamestats_direct
5454
python_gamestats_dataflow
55+
python_xlang_quickstart
56+
java_xlang_quickstart
5557
python_xlang_kafka_taxi_dataflow
5658
python_xlang_sql_taxi_dataflow
5759
)
@@ -158,5 +160,7 @@ python_leaderboard_direct=true
158160
python_leaderboard_dataflow=true
159161
python_gamestats_direct=true
160162
python_gamestats_dataflow=true
163+
python_xlang_quickstart=true
164+
java_xlang_quickstart=true
161165
python_xlang_kafka_taxi_dataflow=true
162166
python_xlang_sql_taxi_dataflow=true

‎website/www/site/content/en/contribute/release-guide.md‎

Lines changed: 10 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -846,18 +846,20 @@ versions to run all of the tests. See Python installation tips in [Developer Wik
846846
```
847847
848848
* **Tasks included**
849-
1. Create a PR to trigger python validation job, including
849+
1. Create a PR to trigger Python validation job, including
850850
* Python quickstart in batch and streaming mode with direct runner and Dataflow runner.
851851
* Python Mobile Games(UserScore, HourlyTeamScore) with direct runner and Dataflow runner.
852852
1. Run Python Streaming MobileGames, includes
853853
* Start a new terminal to run Java Pubsub injector.
854-
* Start a new terminal to run python LeaderBoard with Direct Runner.
855-
* Start a new terminal to run python LeaderBoard with Dataflow Runner.
856-
* Start a new terminal to run python GameStats with Direct Runner.
857-
* Start a new terminal to run python GameStats with Dataflow Runner.
858-
1. Xlang validation, includes
859-
* Start a new terminal to run python xlang Java kafka validation with Dataflow Runner.
860-
* Start a new terminal to run python xlang Java sql validation with Dataflow Runner.
854+
* Start a new terminal to run Python LeaderBoard with Direct Runner.
855+
* Start a new terminal to run Python LeaderBoard with Dataflow Runner.
856+
* Start a new terminal to run Python GameStats with Direct Runner.
857+
* Start a new terminal to run Python GameStats with Dataflow Runner.
858+
1. Multi-language pipelines validation, includes
859+
* Running the Python quickstart example using Python portable DirectRunner. This will start a new terminal for the Java expansion service.
860+
* Running the Java quickstart example using Python portable DirectRunner. This will start new terminals for the Python expansion service and the job server.
861+
* Start a new terminal to run Python multi-language Java kafka validation with Dataflow Runner.
862+
* Start a new terminal to run Python multi-language Java sql validation with Dataflow Runner.
861863
862864
* **Tasks you need to do manually**
863865
1. Check whether validations succeed by following console output instructions.

0 commit comments

Comments
 (0)