HDDS-14897. Add multiple S3 gateways to the rolling-upgrade suite

hadoop-ozone/dist/src/main/compose/testlib.sh

@@ -278,7 +278,7 @@ reorder_om_nodes() {
         "if [[ -f /etc/hadoop/ozone-site.xml ]]; then \
           sed -i -e 's/om1,om2,om3/${new_order}/' /etc/hadoop/ozone-site.xml; \
           echo 'Replaced OM order with ${new_order} in ${c}'; \
-        fi"
+        fi" || true
     done
   fi
 }

hadoop-ozone/dist/src/main/compose/upgrade/compose/ha/docker-compose.yaml

@@ -72,6 +72,16 @@ x-volumes:
     - &ozone-dir ../../../..:${OZONE_DIR}
     - &transformation ../../../../libexec/transformation.py:/opt/hadoop/libexec/transformation.py
 
+x-s3-worker:
+  &s3-worker
+  command: ["ozone","s3g"]
+  <<: *common-config
+  environment:
+    <<: *environment
+  ports:
+    - 9878
+    - 19878
+
 services:
   kdc:
     command: ["/opt/hadoop/compose/common/init-kdc.sh"]
@@ -265,18 +275,49 @@ services:
       - *ozone-dir
       - *transformation
   s3g:
-    command: ["ozone","s3g"]
-    <<: *common-config
-    environment:
-      <<: *environment
+    image: haproxy:lts-alpine
     hostname: s3g
     networks:
       net:
         ipv4_address: 10.9.0.23
     ports:
       - 9878:9878
+      - 19878:19878
+    volumes:
+      - ../../../common/s3-haproxy.cfg:/usr/local/etc/haproxy/haproxy.cfg
+    command: ["haproxy", "-f", "/usr/local/etc/haproxy/haproxy.cfg"]
+  s3g1:
+    <<: *s3-worker
+    hostname: s3g1
+    networks:
+      net:
+        ipv4_address: 10.9.0.24
+    volumes:
+      - ${OZONE_VOLUME}/s3g1:/data
+      - *keytabs
+      - *krb5conf
+      - *ozone-dir
+      - *transformation
+  s3g2:
+    <<: *s3-worker
+    hostname: s3g2
+    networks:
+      net:
+        ipv4_address: 10.9.0.25
+    volumes:
+      - ${OZONE_VOLUME}/s3g2:/data
+      - *keytabs
+      - *krb5conf
+      - *ozone-dir
+      - *transformation
+  s3g3:
+    <<: *s3-worker
+    hostname: s3g3
+    networks:
+      net:
+        ipv4_address: 10.9.0.26
     volumes:
-      - ${OZONE_VOLUME}/s3g:/data
+      - ${OZONE_VOLUME}/s3g3:/data
       - *keytabs
       - *krb5conf
       - *ozone-dir

hadoop-ozone/dist/src/main/compose/upgrade/compose/ha/load.sh

@@ -27,6 +27,6 @@ export COMPOSE_FILE="$TEST_DIR/compose/ha/docker-compose.yaml"
 export OM_SERVICE_ID=omservice
 export SECURITY_ENABLED="true"
 
-create_data_dirs dn{1..5} kms om{1..3} recon s3g scm{1..3}
+create_data_dirs dn{1..5} kms om{1..3} recon s3g s3g{1..3} scm{1..3}
 
 echo "Using docker cluster defined in $COMPOSE_FILE"

hadoop-ozone/dist/src/main/compose/upgrade/upgrades/rolling-upgrade/driver.sh

@@ -54,20 +54,13 @@ rolling_restart_service() {
     fi
   fi
 
-  # The data generation/validation is doing S3 API tests, so skip it in case the S3 gateway is updated
-  # TODO find a better solution
-  if [[ ${SERVICE} != "s3g" ]]; then
-    callback before_service_restart
-  fi
+  callback before_service_restart
 
   # Restart service with new image.
   prepare_for_image "${OZONE_UPGRADE_TO}"
   create_containers "${SERVICE}"
 
-  # The data generation/validation is doing S3 API tests, so skip it in case the S3 gateway is updated
-  if [[ ${SERVICE} != "s3g" ]]; then
-    callback after_service_restart
-  fi
+  callback after_service_restart
 
   # Service-specific readiness checks.
   case "${SERVICE}" in
@@ -112,14 +105,17 @@ for s in dn1 dn2 dn3 dn4 dn5; do
   rolling_restart_service "$s" "$OZONE_UPGRADE_TO"
 done
 
+# OMs
 for s in om1 om2 om3; do
   OUTPUT_NAME="${OZONE_UPGRADE_FROM}-${OZONE_UPGRADE_TO}-2-${s}"
   rolling_restart_service "$s" "$OZONE_UPGRADE_TO"
 done
 
-# S3 Gateway
-OUTPUT_NAME="${OZONE_UPGRADE_FROM}-${OZONE_UPGRADE_TO}-2-s3g"
-rolling_restart_service "s3g" "$OZONE_UPGRADE_TO"
+# S3 Gateways (s3g is HAProxy and does not need to be upgraded)
+for s in s3g1 s3g2 s3g3; do
+  OUTPUT_NAME="${OZONE_UPGRADE_FROM}-${OZONE_UPGRADE_TO}-2-${s}"
+  rolling_restart_service "$s" "$OZONE_UPGRADE_TO"
+done
 
 # TODO Add downgrade scenario