HDDS-14897. Add multiple S3 gateways to the rolling-upgrade suite (#10028)

Author: dombizita

hadoop-ozone/dist/src/main/compose/common/s3-haproxy.cfg

@@ -33,16 +33,24 @@ frontend http-in
 
 backend servers
     balance roundrobin
-    server server1 s3g1:9878 maxconn 32
-    server server2 s3g2:9878 maxconn 32
-    server server3 s3g3:9878 maxconn 32
+    option httpchk GET /
+    http-check expect ! rstatus ^5
+    server server1 s3g1:9878 maxconn 32 check
+    server server2 s3g2:9878 maxconn 32 check
+    server server3 s3g3:9878 maxconn 32 check
+    retries 3
+    option redispatch
 
 frontend webadmin
     bind *:19878
     default_backend webadmin-servers
 
 backend webadmin-servers
     balance roundrobin
-    server server1 s3g1:19878 maxconn 32
-    server server2 s3g2:19878 maxconn 32
-    server server3 s3g3:19878 maxconn 32
+    option httpchk GET /
+    http-check expect ! rstatus ^5
+    server server1 s3g1:19878 maxconn 32 check 
+    server server2 s3g2:19878 maxconn 32 check
+    server server3 s3g3:19878 maxconn 32 check
+    retries 3
+    option redispatch

hadoop-ozone/dist/src/main/compose/testlib.sh

@@ -273,7 +273,7 @@ reorder_om_nodes() {
   local new_order="$1"
 
   if [[ -n "${new_order}" ]] && [[ "${new_order}" != "om1,om2,om3" ]]; then
-    for c in $(docker-compose ps | cut -f1 -d' ' | grep -v -e '^NAME$' -e '^om'); do
+    for c in $(docker-compose ps | cut -f1 -d' ' | grep -v -e '^NAME$' -e '^om' -e 's3g-haproxy'); do
       docker exec "${c}" bash -c \
         "if [[ -f /etc/hadoop/ozone-site.xml ]]; then \
           sed -i -e 's/om1,om2,om3/${new_order}/' /etc/hadoop/ozone-site.xml; \

hadoop-ozone/dist/src/main/compose/upgrade/compose/ha/docker-compose.yaml

@@ -72,6 +72,16 @@ x-volumes:
     - &ozone-dir ../../../..:${OZONE_DIR}
     - &transformation ../../../../libexec/transformation.py:/opt/hadoop/libexec/transformation.py
 
+x-s3-worker:
+  &s3-worker
+  command: ["ozone","s3g"]
+  <<: *common-config
+  environment:
+    <<: *environment
+  ports:
+    - 9878
+    - 19878
+
 services:
   kdc:
     command: ["/opt/hadoop/compose/common/init-kdc.sh"]
@@ -264,19 +274,50 @@ services:
       - *krb5conf
       - *ozone-dir
       - *transformation
-  s3g:
-    command: ["ozone","s3g"]
-    <<: *common-config
-    environment:
-      <<: *environment
+  s3g-haproxy:
+    image: haproxy:lts-alpine
     hostname: s3g
     networks:
       net:
         ipv4_address: 10.9.0.23
     ports:
       - 9878:9878
+      - 19878:19878
+    volumes:
+      - ../../../common/s3-haproxy.cfg:/usr/local/etc/haproxy/haproxy.cfg
+    command: ["haproxy", "-f", "/usr/local/etc/haproxy/haproxy.cfg"]
+  s3g1:
+    <<: *s3-worker
+    hostname: s3g1
+    networks:
+      net:
+        ipv4_address: 10.9.0.24
+    volumes:
+      - ${OZONE_VOLUME}/s3g1:/data
+      - *keytabs
+      - *krb5conf
+      - *ozone-dir
+      - *transformation
+  s3g2:
+    <<: *s3-worker
+    hostname: s3g2
+    networks:
+      net:
+        ipv4_address: 10.9.0.25
+    volumes:
+      - ${OZONE_VOLUME}/s3g2:/data
+      - *keytabs
+      - *krb5conf
+      - *ozone-dir
+      - *transformation
+  s3g3:
+    <<: *s3-worker
+    hostname: s3g3
+    networks:
+      net:
+        ipv4_address: 10.9.0.26
     volumes:
-      - ${OZONE_VOLUME}/s3g:/data
+      - ${OZONE_VOLUME}/s3g3:/data
       - *keytabs
       - *krb5conf
       - *ozone-dir

hadoop-ozone/dist/src/main/compose/upgrade/compose/ha/load.sh

@@ -27,6 +27,6 @@ export COMPOSE_FILE="$TEST_DIR/compose/ha/docker-compose.yaml"
 export OM_SERVICE_ID=omservice
 export SECURITY_ENABLED="true"
 
-create_data_dirs dn{1..5} kms om{1..3} recon s3g scm{1..3}
+create_data_dirs dn{1..5} kms om{1..3} recon s3g{1..3} scm{1..3}
 
 echo "Using docker cluster defined in $COMPOSE_FILE"

hadoop-ozone/dist/src/main/compose/upgrade/test.sh

@@ -33,7 +33,7 @@ RESULT_DIR="$ALL_RESULT_DIR" create_results_dir
 
 # This is the version of Ozone that should use the runner image to run the
 # code that was built. Other versions will pull images from docker hub.
-run_test ha     non-rolling-upgrade 2.1.0 "$OZONE_CURRENT_VERSION"
+# run_test ha     non-rolling-upgrade 2.1.0 "$OZONE_CURRENT_VERSION"
 # run_test ha     non-rolling-upgrade 2.0.0 "$OZONE_CURRENT_VERSION"
 #run_test non-ha non-rolling-upgrade 1.4.1 "$OZONE_CURRENT_VERSION"
 #run_test ha     non-rolling-upgrade 1.4.1 "$OZONE_CURRENT_VERSION"
@@ -43,7 +43,7 @@ run_test ha     non-rolling-upgrade 2.1.0 "$OZONE_CURRENT_VERSION"
 # run_test om-ha non-rolling-upgrade 1.1.0 "$OZONE_CURRENT_VERSION"
 
 # Rolling upgrade test, commented out for now
-# run_test ha     rolling-upgrade "$OZONE_CURRENT_VERSION" "$OZONE_CURRENT_VERSION"
+ run_test ha     rolling-upgrade "$OZONE_CURRENT_VERSION" "$OZONE_CURRENT_VERSION"
 
 generate_report "upgrade" "$ALL_RESULT_DIR"
 

hadoop-ozone/dist/src/main/compose/upgrade/upgrades/rolling-upgrade/driver.sh

@@ -54,20 +54,13 @@ rolling_restart_service() {
     fi
   fi
 
-  # The data generation/validation is doing S3 API tests, so skip it in case the S3 gateway is updated
-  # TODO find a better solution
-  if [[ ${SERVICE} != "s3g" ]]; then
-    callback before_service_restart
-  fi
+  callback before_service_restart
 
   # Restart service with new image.
   prepare_for_image "${OZONE_UPGRADE_TO}"
   create_containers "${SERVICE}"
 
-  # The data generation/validation is doing S3 API tests, so skip it in case the S3 gateway is updated
-  if [[ ${SERVICE} != "s3g" ]]; then
-    callback after_service_restart
-  fi
+  callback after_service_restart
 
   # Service-specific readiness checks.
   case "${SERVICE}" in
@@ -81,6 +74,9 @@ rolling_restart_service() {
     dn*)
       wait_for_port "${SERVICE}" 9882 120
       ;;
+    s3g*)
+      wait_for_port "${SERVICE}" 9878 120
+      ;;
   esac
 }
 
@@ -112,14 +108,17 @@ for s in dn1 dn2 dn3 dn4 dn5; do
   rolling_restart_service "$s" "$OZONE_UPGRADE_TO"
 done
 
+# OMs
 for s in om1 om2 om3; do
   OUTPUT_NAME="${OZONE_UPGRADE_FROM}-${OZONE_UPGRADE_TO}-2-${s}"
   rolling_restart_service "$s" "$OZONE_UPGRADE_TO"
 done
 
-# S3 Gateway
-OUTPUT_NAME="${OZONE_UPGRADE_FROM}-${OZONE_UPGRADE_TO}-2-s3g"
-rolling_restart_service "s3g" "$OZONE_UPGRADE_TO"
+# S3 Gateways (s3g is HAProxy and does not need to be upgraded)
+for s in s3g1 s3g2 s3g3; do
+  OUTPUT_NAME="${OZONE_UPGRADE_FROM}-${OZONE_UPGRADE_TO}-2-${s}"
+  rolling_restart_service "$s" "$OZONE_UPGRADE_TO"
+done
 
 # TODO Add downgrade scenario