From 023286a8e6cafed83200c50bad0e4cb1b4d52149 Mon Sep 17 00:00:00 2001 From: Zita Dombi Date: Wed, 1 Apr 2026 13:33:44 +0200 Subject: [PATCH 1/5] HDDS-14897. Add multiple S3 gateways to the rolling-upgrade suite --- hadoop-ozone/dist/src/main/compose/testlib.sh | 2 +- .../upgrade/compose/ha/docker-compose.yaml | 51 +++++++++++++++++-- .../main/compose/upgrade/compose/ha/load.sh | 2 +- .../dist/src/main/compose/upgrade/test.sh | 4 +- .../upgrades/rolling-upgrade/driver.sh | 20 +++----- 5 files changed, 58 insertions(+), 21 deletions(-) diff --git a/hadoop-ozone/dist/src/main/compose/testlib.sh b/hadoop-ozone/dist/src/main/compose/testlib.sh index 040d9c1db1c6..f21330de0215 100755 --- a/hadoop-ozone/dist/src/main/compose/testlib.sh +++ b/hadoop-ozone/dist/src/main/compose/testlib.sh @@ -278,7 +278,7 @@ reorder_om_nodes() { "if [[ -f /etc/hadoop/ozone-site.xml ]]; then \ sed -i -e 's/om1,om2,om3/${new_order}/' /etc/hadoop/ozone-site.xml; \ echo 'Replaced OM order with ${new_order} in ${c}'; \ - fi" + fi" || true done fi } diff --git a/hadoop-ozone/dist/src/main/compose/upgrade/compose/ha/docker-compose.yaml b/hadoop-ozone/dist/src/main/compose/upgrade/compose/ha/docker-compose.yaml index 20c93493a14d..e389943572a4 100644 --- a/hadoop-ozone/dist/src/main/compose/upgrade/compose/ha/docker-compose.yaml +++ b/hadoop-ozone/dist/src/main/compose/upgrade/compose/ha/docker-compose.yaml @@ -72,6 +72,16 @@ x-volumes: - &ozone-dir ../../../..:${OZONE_DIR} - &transformation ../../../../libexec/transformation.py:/opt/hadoop/libexec/transformation.py +x-s3-worker: + &s3-worker + command: ["ozone","s3g"] + <<: *common-config + environment: + <<: *environment + ports: + - 9878 + - 19878 + services: kdc: command: ["/opt/hadoop/compose/common/init-kdc.sh"] @@ -265,18 +275,49 @@ services: - *ozone-dir - *transformation s3g: - command: ["ozone","s3g"] - <<: *common-config - environment: - <<: *environment + image: haproxy:lts-alpine hostname: s3g networks: net: ipv4_address: 10.9.0.23 ports: - 9878:9878 + - 19878:19878 + volumes: + - ../../../common/s3-haproxy.cfg:/usr/local/etc/haproxy/haproxy.cfg + command: ["haproxy", "-f", "/usr/local/etc/haproxy/haproxy.cfg"] + s3g1: + <<: *s3-worker + hostname: s3g1 + networks: + net: + ipv4_address: 10.9.0.24 + volumes: + - ${OZONE_VOLUME}/s3g1:/data + - *keytabs + - *krb5conf + - *ozone-dir + - *transformation + s3g2: + <<: *s3-worker + hostname: s3g2 + networks: + net: + ipv4_address: 10.9.0.25 + volumes: + - ${OZONE_VOLUME}/s3g2:/data + - *keytabs + - *krb5conf + - *ozone-dir + - *transformation + s3g3: + <<: *s3-worker + hostname: s3g3 + networks: + net: + ipv4_address: 10.9.0.26 volumes: - - ${OZONE_VOLUME}/s3g:/data + - ${OZONE_VOLUME}/s3g3:/data - *keytabs - *krb5conf - *ozone-dir diff --git a/hadoop-ozone/dist/src/main/compose/upgrade/compose/ha/load.sh b/hadoop-ozone/dist/src/main/compose/upgrade/compose/ha/load.sh index b1063db05440..e275ee59b40b 100755 --- a/hadoop-ozone/dist/src/main/compose/upgrade/compose/ha/load.sh +++ b/hadoop-ozone/dist/src/main/compose/upgrade/compose/ha/load.sh @@ -27,6 +27,6 @@ export COMPOSE_FILE="$TEST_DIR/compose/ha/docker-compose.yaml" export OM_SERVICE_ID=omservice export SECURITY_ENABLED="true" -create_data_dirs dn{1..5} kms om{1..3} recon s3g scm{1..3} +create_data_dirs dn{1..5} kms om{1..3} recon s3g s3g{1..3} scm{1..3} echo "Using docker cluster defined in $COMPOSE_FILE" diff --git a/hadoop-ozone/dist/src/main/compose/upgrade/test.sh b/hadoop-ozone/dist/src/main/compose/upgrade/test.sh index 014d1ba81e21..51c61d2e80fa 100755 --- a/hadoop-ozone/dist/src/main/compose/upgrade/test.sh +++ b/hadoop-ozone/dist/src/main/compose/upgrade/test.sh @@ -33,7 +33,7 @@ RESULT_DIR="$ALL_RESULT_DIR" create_results_dir # This is the version of Ozone that should use the runner image to run the # code that was built. Other versions will pull images from docker hub. -run_test ha non-rolling-upgrade 2.1.0 "$OZONE_CURRENT_VERSION" +# run_test ha non-rolling-upgrade 2.1.0 "$OZONE_CURRENT_VERSION" # run_test ha non-rolling-upgrade 2.0.0 "$OZONE_CURRENT_VERSION" #run_test non-ha non-rolling-upgrade 1.4.1 "$OZONE_CURRENT_VERSION" #run_test ha non-rolling-upgrade 1.4.1 "$OZONE_CURRENT_VERSION" @@ -43,7 +43,7 @@ run_test ha non-rolling-upgrade 2.1.0 "$OZONE_CURRENT_VERSION" # run_test om-ha non-rolling-upgrade 1.1.0 "$OZONE_CURRENT_VERSION" # Rolling upgrade test, commented out for now -# run_test ha rolling-upgrade "$OZONE_CURRENT_VERSION" "$OZONE_CURRENT_VERSION" +run_test ha rolling-upgrade "$OZONE_CURRENT_VERSION" "$OZONE_CURRENT_VERSION" generate_report "upgrade" "$ALL_RESULT_DIR" diff --git a/hadoop-ozone/dist/src/main/compose/upgrade/upgrades/rolling-upgrade/driver.sh b/hadoop-ozone/dist/src/main/compose/upgrade/upgrades/rolling-upgrade/driver.sh index 73663dc53614..0f5fadfb29e2 100755 --- a/hadoop-ozone/dist/src/main/compose/upgrade/upgrades/rolling-upgrade/driver.sh +++ b/hadoop-ozone/dist/src/main/compose/upgrade/upgrades/rolling-upgrade/driver.sh @@ -54,20 +54,13 @@ rolling_restart_service() { fi fi - # The data generation/validation is doing S3 API tests, so skip it in case the S3 gateway is updated - # TODO find a better solution - if [[ ${SERVICE} != "s3g" ]]; then - callback before_service_restart - fi + callback before_service_restart # Restart service with new image. prepare_for_image "${OZONE_UPGRADE_TO}" create_containers "${SERVICE}" - # The data generation/validation is doing S3 API tests, so skip it in case the S3 gateway is updated - if [[ ${SERVICE} != "s3g" ]]; then - callback after_service_restart - fi + callback after_service_restart # Service-specific readiness checks. case "${SERVICE}" in @@ -112,14 +105,17 @@ for s in dn1 dn2 dn3 dn4 dn5; do rolling_restart_service "$s" "$OZONE_UPGRADE_TO" done +# OMs for s in om1 om2 om3; do OUTPUT_NAME="${OZONE_UPGRADE_FROM}-${OZONE_UPGRADE_TO}-2-${s}" rolling_restart_service "$s" "$OZONE_UPGRADE_TO" done -# S3 Gateway -OUTPUT_NAME="${OZONE_UPGRADE_FROM}-${OZONE_UPGRADE_TO}-2-s3g" -rolling_restart_service "s3g" "$OZONE_UPGRADE_TO" +# S3 Gateways (s3g is HAProxy and does not need to be upgraded) +for s in s3g1 s3g2 s3g3; do + OUTPUT_NAME="${OZONE_UPGRADE_FROM}-${OZONE_UPGRADE_TO}-2-${s}" + rolling_restart_service "$s" "$OZONE_UPGRADE_TO" +done # TODO Add downgrade scenario From 7e2c25ead70e2f6200d9b6c7bc74ec558d24daaa Mon Sep 17 00:00:00 2001 From: Zita Dombi Date: Wed, 1 Apr 2026 13:34:54 +0200 Subject: [PATCH 2/5] Comment out rolling upgrade run --- hadoop-ozone/dist/src/main/compose/upgrade/test.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/hadoop-ozone/dist/src/main/compose/upgrade/test.sh b/hadoop-ozone/dist/src/main/compose/upgrade/test.sh index 51c61d2e80fa..014d1ba81e21 100755 --- a/hadoop-ozone/dist/src/main/compose/upgrade/test.sh +++ b/hadoop-ozone/dist/src/main/compose/upgrade/test.sh @@ -33,7 +33,7 @@ RESULT_DIR="$ALL_RESULT_DIR" create_results_dir # This is the version of Ozone that should use the runner image to run the # code that was built. Other versions will pull images from docker hub. -# run_test ha non-rolling-upgrade 2.1.0 "$OZONE_CURRENT_VERSION" +run_test ha non-rolling-upgrade 2.1.0 "$OZONE_CURRENT_VERSION" # run_test ha non-rolling-upgrade 2.0.0 "$OZONE_CURRENT_VERSION" #run_test non-ha non-rolling-upgrade 1.4.1 "$OZONE_CURRENT_VERSION" #run_test ha non-rolling-upgrade 1.4.1 "$OZONE_CURRENT_VERSION" @@ -43,7 +43,7 @@ RESULT_DIR="$ALL_RESULT_DIR" create_results_dir # run_test om-ha non-rolling-upgrade 1.1.0 "$OZONE_CURRENT_VERSION" # Rolling upgrade test, commented out for now -run_test ha rolling-upgrade "$OZONE_CURRENT_VERSION" "$OZONE_CURRENT_VERSION" +# run_test ha rolling-upgrade "$OZONE_CURRENT_VERSION" "$OZONE_CURRENT_VERSION" generate_report "upgrade" "$ALL_RESULT_DIR" From 215088b47a72ee21e63c15a98ec195fb82094686 Mon Sep 17 00:00:00 2001 From: Zita Dombi Date: Fri, 3 Apr 2026 00:15:47 +0200 Subject: [PATCH 3/5] Remove unnecessary s3g data directory --- hadoop-ozone/dist/src/main/compose/upgrade/compose/ha/load.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hadoop-ozone/dist/src/main/compose/upgrade/compose/ha/load.sh b/hadoop-ozone/dist/src/main/compose/upgrade/compose/ha/load.sh index e275ee59b40b..4b38c86a73d0 100755 --- a/hadoop-ozone/dist/src/main/compose/upgrade/compose/ha/load.sh +++ b/hadoop-ozone/dist/src/main/compose/upgrade/compose/ha/load.sh @@ -27,6 +27,6 @@ export COMPOSE_FILE="$TEST_DIR/compose/ha/docker-compose.yaml" export OM_SERVICE_ID=omservice export SECURITY_ENABLED="true" -create_data_dirs dn{1..5} kms om{1..3} recon s3g s3g{1..3} scm{1..3} +create_data_dirs dn{1..5} kms om{1..3} recon s3g{1..3} scm{1..3} echo "Using docker cluster defined in $COMPOSE_FILE" From 0e3f71d66dad88d2b4aabb6d1ddb71eaecb7c583 Mon Sep 17 00:00:00 2001 From: Zita Dombi Date: Mon, 13 Apr 2026 23:43:03 +0200 Subject: [PATCH 4/5] Address review comments and run test suite --- .../src/main/compose/common/s3-haproxy.cfg | 20 +++++++++++++------ hadoop-ozone/dist/src/main/compose/testlib.sh | 4 ++-- .../upgrade/compose/ha/docker-compose.yaml | 2 +- .../dist/src/main/compose/upgrade/test.sh | 4 ++-- .../upgrades/rolling-upgrade/driver.sh | 3 +++ 5 files changed, 22 insertions(+), 11 deletions(-) diff --git a/hadoop-ozone/dist/src/main/compose/common/s3-haproxy.cfg b/hadoop-ozone/dist/src/main/compose/common/s3-haproxy.cfg index 6e8728945f29..a88ceaca9678 100644 --- a/hadoop-ozone/dist/src/main/compose/common/s3-haproxy.cfg +++ b/hadoop-ozone/dist/src/main/compose/common/s3-haproxy.cfg @@ -33,9 +33,13 @@ frontend http-in backend servers balance roundrobin - server server1 s3g1:9878 maxconn 32 - server server2 s3g2:9878 maxconn 32 - server server3 s3g3:9878 maxconn 32 + option httpchk GET / + http-check expect ! rstatus ^5 + server server1 s3g1:9878 maxconn 32 check + server server2 s3g2:9878 maxconn 32 check + server server3 s3g3:9878 maxconn 32 check + retries 3 + option redispatch frontend webadmin bind *:19878 @@ -43,6 +47,10 @@ frontend webadmin backend webadmin-servers balance roundrobin - server server1 s3g1:19878 maxconn 32 - server server2 s3g2:19878 maxconn 32 - server server3 s3g3:19878 maxconn 32 + option httpchk GET / + http-check expect ! rstatus ^5 + server server1 s3g1:19878 maxconn 32 check + server server2 s3g2:19878 maxconn 32 check + server server3 s3g3:19878 maxconn 32 check + retries 3 + option redispatch diff --git a/hadoop-ozone/dist/src/main/compose/testlib.sh b/hadoop-ozone/dist/src/main/compose/testlib.sh index f21330de0215..99c486d2faca 100755 --- a/hadoop-ozone/dist/src/main/compose/testlib.sh +++ b/hadoop-ozone/dist/src/main/compose/testlib.sh @@ -273,12 +273,12 @@ reorder_om_nodes() { local new_order="$1" if [[ -n "${new_order}" ]] && [[ "${new_order}" != "om1,om2,om3" ]]; then - for c in $(docker-compose ps | cut -f1 -d' ' | grep -v -e '^NAME$' -e '^om'); do + for c in $(docker-compose ps | cut -f1 -d' ' | grep -v -e '^NAME$' -e '^om' -e 's3g-haproxy'); do docker exec "${c}" bash -c \ "if [[ -f /etc/hadoop/ozone-site.xml ]]; then \ sed -i -e 's/om1,om2,om3/${new_order}/' /etc/hadoop/ozone-site.xml; \ echo 'Replaced OM order with ${new_order} in ${c}'; \ - fi" || true + fi done fi } diff --git a/hadoop-ozone/dist/src/main/compose/upgrade/compose/ha/docker-compose.yaml b/hadoop-ozone/dist/src/main/compose/upgrade/compose/ha/docker-compose.yaml index e389943572a4..af34b86566f0 100644 --- a/hadoop-ozone/dist/src/main/compose/upgrade/compose/ha/docker-compose.yaml +++ b/hadoop-ozone/dist/src/main/compose/upgrade/compose/ha/docker-compose.yaml @@ -274,7 +274,7 @@ services: - *krb5conf - *ozone-dir - *transformation - s3g: + s3g-haproxy: image: haproxy:lts-alpine hostname: s3g networks: diff --git a/hadoop-ozone/dist/src/main/compose/upgrade/test.sh b/hadoop-ozone/dist/src/main/compose/upgrade/test.sh index 014d1ba81e21..3c079cd884fd 100755 --- a/hadoop-ozone/dist/src/main/compose/upgrade/test.sh +++ b/hadoop-ozone/dist/src/main/compose/upgrade/test.sh @@ -33,7 +33,7 @@ RESULT_DIR="$ALL_RESULT_DIR" create_results_dir # This is the version of Ozone that should use the runner image to run the # code that was built. Other versions will pull images from docker hub. -run_test ha non-rolling-upgrade 2.1.0 "$OZONE_CURRENT_VERSION" +# run_test ha non-rolling-upgrade 2.1.0 "$OZONE_CURRENT_VERSION" # run_test ha non-rolling-upgrade 2.0.0 "$OZONE_CURRENT_VERSION" #run_test non-ha non-rolling-upgrade 1.4.1 "$OZONE_CURRENT_VERSION" #run_test ha non-rolling-upgrade 1.4.1 "$OZONE_CURRENT_VERSION" @@ -43,7 +43,7 @@ run_test ha non-rolling-upgrade 2.1.0 "$OZONE_CURRENT_VERSION" # run_test om-ha non-rolling-upgrade 1.1.0 "$OZONE_CURRENT_VERSION" # Rolling upgrade test, commented out for now -# run_test ha rolling-upgrade "$OZONE_CURRENT_VERSION" "$OZONE_CURRENT_VERSION" + run_test ha rolling-upgrade "$OZONE_CURRENT_VERSION" "$OZONE_CURRENT_VERSION" generate_report "upgrade" "$ALL_RESULT_DIR" diff --git a/hadoop-ozone/dist/src/main/compose/upgrade/upgrades/rolling-upgrade/driver.sh b/hadoop-ozone/dist/src/main/compose/upgrade/upgrades/rolling-upgrade/driver.sh index 0f5fadfb29e2..e2972a403e5d 100755 --- a/hadoop-ozone/dist/src/main/compose/upgrade/upgrades/rolling-upgrade/driver.sh +++ b/hadoop-ozone/dist/src/main/compose/upgrade/upgrades/rolling-upgrade/driver.sh @@ -74,6 +74,9 @@ rolling_restart_service() { dn*) wait_for_port "${SERVICE}" 9882 120 ;; + s3g*) + wait_for_port "${SERVICE}" 9878 120 + ;; esac } From 03017115181af31c6113a1335d88802aaa9e83b7 Mon Sep 17 00:00:00 2001 From: Zita Dombi Date: Mon, 13 Apr 2026 23:48:58 +0200 Subject: [PATCH 5/5] Add missing quote --- hadoop-ozone/dist/src/main/compose/testlib.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hadoop-ozone/dist/src/main/compose/testlib.sh b/hadoop-ozone/dist/src/main/compose/testlib.sh index 99c486d2faca..68ad87bfac8c 100755 --- a/hadoop-ozone/dist/src/main/compose/testlib.sh +++ b/hadoop-ozone/dist/src/main/compose/testlib.sh @@ -278,7 +278,7 @@ reorder_om_nodes() { "if [[ -f /etc/hadoop/ozone-site.xml ]]; then \ sed -i -e 's/om1,om2,om3/${new_order}/' /etc/hadoop/ozone-site.xml; \ echo 'Replaced OM order with ${new_order} in ${c}'; \ - fi + fi" done fi }