HDDS-14896. Use separate container for clients in rolling-upgrade suite (#10032)

Author: dombizita

hadoop-ozone/dist/src/main/compose/common/init-kdc.sh

@@ -31,13 +31,15 @@ export_keytab HTTP/scm scm
 export_keytab testuser/scm scm
 export_keytab testuser2/scm scm
 
+export_keytab testuser/client testuser
 export_keytab testuser/dn testuser
 export_keytab testuser/httpfs testuser
 export_keytab testuser/om testuser
 export_keytab testuser/recon testuser
 export_keytab testuser/s3g testuser
 export_keytab testuser/scm testuser
 
+export_keytab testuser2/client testuser2
 export_keytab testuser2/dn testuser2
 export_keytab testuser2/httpfs testuser2
 export_keytab testuser2/om testuser2

hadoop-ozone/dist/src/main/compose/upgrade/README.md

@@ -100,6 +100,7 @@ Docker compose cluster definitions to be used in upgrade testing are defined in
   - `SCM`: The name of the SCM container to run robot tests from.
     - This can be passed as the first argument to `execute_robot_test`.
     - This allows the same tests to work with and without SCM HA.
+  - `CLIENT`: The dedicated client container used to run Robot tests in rolling-upgrade HA.
 
 ### Testing New Versions
 

hadoop-ozone/dist/src/main/compose/upgrade/compose/ha/docker-compose.yaml

@@ -322,6 +322,17 @@ services:
       - *krb5conf
       - *ozone-dir
       - *transformation
+  client:
+    command: ["sleep","1000000"]
+    <<: *common-config
+    hostname: client
+    networks:
+      net:
+        ipv4_address: 10.9.0.27
+    volumes:
+      - *keytabs
+      - *krb5conf
+      - *ozone-dir
 
 networks:
   net:

hadoop-ozone/dist/src/main/compose/upgrade/compose/ha/load.sh

@@ -24,6 +24,7 @@ set +u
 source "$TEST_DIR/testlib.sh"
 
 export COMPOSE_FILE="$TEST_DIR/compose/ha/docker-compose.yaml"
+export CLIENT=client
 export OM_SERVICE_ID=omservice
 export SECURITY_ENABLED="true"
 

hadoop-ozone/dist/src/main/compose/upgrade/testlib.sh

@@ -117,16 +117,18 @@ run_test() {
 
 ## @description Generates data on the cluster.
 ## @param The prefix to use for data generated.
-## @param All parameters after the first one are passed directly to the robot command,
+## @param The container to run the robot test in.
+## @param All remaining parameters are passed directly to the robot command,
 ##        see https://robotframework.org/robotframework/latest/RobotFrameworkUserGuide.html#all-command-line-options
 generate() {
-  execute_robot_test "$SCM" -N "${OUTPUT_NAME}-generate-${1}" -v PREFIX:"$1" ${@:2} upgrade/generate.robot
+  execute_robot_test "${2}" -N "${OUTPUT_NAME}-generate-${1}" -v PREFIX:"$1" ${@:3} upgrade/generate.robot
 }
 
 ## @description Validates that data exists on the cluster.
 ## @param The prefix of the data to be validated.
-## @param All parameters after the first one are passed directly to the robot command,
+## @param The container to run the robot test in.
+## @param All remaining parameters are passed directly to the robot command,
 ##        see https://robotframework.org/robotframework/latest/RobotFrameworkUserGuide.html#all-command-line-options
 validate() {
-  execute_robot_test "$SCM" -N "${OUTPUT_NAME}-validate-${1}" -v PREFIX:"$1" ${@:2} upgrade/validate.robot
+  execute_robot_test "${2}" -N "${OUTPUT_NAME}-validate-${1}" -v PREFIX:"$1" ${@:3} upgrade/validate.robot
 }

hadoop-ozone/dist/src/main/compose/upgrade/upgrades/non-rolling-upgrade/callbacks/common/callback.sh

@@ -21,40 +21,40 @@ source "$TEST_DIR"/testlib.sh
 
 with_old_version() {
   execute_robot_test "$SCM" -N "${OUTPUT_NAME}-check-finalization" --include finalized upgrade/check-finalization.robot
-  generate old1
-  validate old1
+  generate old1 "$SCM"
+  validate old1 "$SCM"
 }
 
 with_this_version_pre_finalized() {
   # No check for pre-finalized status here, because the release may not have
   # added layout features to OM or HDDS.
-  validate old1
+  validate old1 "$SCM"
   # HDDS-6261: overwrite the same keys intentionally
-  generate old1 --exclude create-volume-and-bucket
+  generate old1 "$SCM" --exclude create-volume-and-bucket
 
-  generate new1
-  validate new1
+  generate new1 "$SCM"
+  validate new1 "$SCM"
 }
 
 with_old_version_downgraded() {
   execute_robot_test "$SCM" -N "${OUTPUT_NAME}-check-finalization" --include finalized upgrade/check-finalization.robot
-  validate old1
-  validate new1
+  validate old1 "$SCM"
+  validate new1 "$SCM"
 
-  generate old2
-  validate old2
+  generate old2 "$SCM"
+  validate old2 "$SCM"
 
   # HDDS-6261: overwrite the same keys again to trigger the precondition check
   # that exists <= 1.1.0 OM
-  generate old1 --exclude create-volume-and-bucket
+  generate old1 "$SCM" --exclude create-volume-and-bucket
 }
 
 with_this_version_finalized() {
   execute_robot_test "$SCM" -N "${OUTPUT_NAME}-check-finalization" --include finalized upgrade/check-finalization.robot
-  validate old1
-  validate new1
-  validate old2
+  validate old1 "$SCM"
+  validate new1 "$SCM"
+  validate old2 "$SCM"
 
-  generate new2
-  validate new2
+  generate new2 "$SCM"
+  validate new2 "$SCM"
 }

hadoop-ozone/dist/src/main/compose/upgrade/upgrades/rolling-upgrade/callbacks/common/callback.sh

@@ -20,9 +20,9 @@ source "$TEST_DIR"/testlib.sh
 ### CALLBACKS ###
 
 before_service_restart() {
-  generate "generate-${SERVICE}"
+  generate "generate-${SERVICE}" "$CLIENT"
 }
 
 after_service_restart() {
-  validate "generate-${SERVICE}"
+  validate "generate-${SERVICE}" "$CLIENT"
 }

hadoop-ozone/dist/src/main/compose/upgrade/upgrades/rolling-upgrade/driver.sh

@@ -27,6 +27,7 @@ set -u
 : "${OZONE_UPGRADE_TO}"
 : "${TEST_DIR}"
 : "${SCM}"
+: "${CLIENT}"
 : "${OZONE_CURRENT_VERSION}"
 set +u
 
@@ -43,17 +44,6 @@ rolling_restart_service() {
   # Stop service
   stop_containers "${SERVICE}"
 
-  # Check if this SCM container is running, as during a rolling upgrade it does stop-start one-by-one and
-  # we want to run write/read tests while one service is unavailable. Choose SCM (the container where the generate and
-  # validate robot tests are running) considering availability.
-  if [[ "$(docker inspect -f '{{.State.Running}}' "ha-${SCM}-1" 2>/dev/null)" != "true" ]]; then
-    local fallback_scm
-    fallback_scm="$(docker-compose --project-directory="$TEST_DIR/compose/ha" config --services | grep scm | grep -v "^${SCM}$" | head -n1)"
-    if [[ -n "$fallback_scm" ]]; then
-      export SCM="$fallback_scm"
-    fi
-  fi
-
   callback before_service_restart
 
   # Restart service with new image.
@@ -122,10 +112,18 @@ done
 
 # TODO Add downgrade scenario
 
+# Upgrade client after all server components are upgraded but before finalization,
+# so new client APIs can be exercised against pre-finalized servers.
+echo "--- UPGRADING CLIENT TO $OZONE_UPGRADE_TO ---"
+OUTPUT_NAME="${OZONE_UPGRADE_FROM}-${OZONE_UPGRADE_TO}-2-client"
+stop_containers "$CLIENT"
+prepare_for_image "${OZONE_UPGRADE_TO}"
+create_containers "$CLIENT"
+
 echo "--- RUNNING WITH NEW VERSION $OZONE_UPGRADE_TO FINALIZED ---"
 OUTPUT_NAME="${OZONE_UPGRADE_FROM}-${OZONE_UPGRADE_TO}-3-finalized"
 
 # TODO Add validation for pre-finalized state
 
 # Sends commands to finalize OM and SCM.
-execute_robot_test "$SCM" -N "${OUTPUT_NAME}-finalize" upgrade/finalize.robot
+execute_robot_test "$CLIENT" -N "${OUTPUT_NAME}-finalize" upgrade/finalize.robot