HDDS-14821. Version specific data validation in rolling-upgrade suite

hadoop-ozone/dist/src/main/compose/upgrade/compose/ha/docker-compose.yaml

@@ -325,6 +325,8 @@ services:
   client:
     command: ["sleep","1000000"]
     <<: *common-config
+    environment:
+      <<: *environment
     hostname: client
     networks:
       net:

hadoop-ozone/dist/src/main/compose/upgrade/testlib.sh

@@ -49,8 +49,10 @@ prepare_for_image() {
 ##   the upgrade being tested if one exists. If neither exists, print a
 ##   warning that nothing was tested.
 ## @param The name of the function to run.
+## @param All remaining parameters are passed as parameters to the called function
 callback() {
   local func="$1"
+  shift
 
   set -u
   : "${OZONE_UPGRADE_CALLBACK}"
@@ -61,7 +63,7 @@ callback() {
     # Common callback always exists.
     source "$OZONE_COMMON_CALLBACK"
     if [[ "$(type -t "$func")" = function ]]; then
-      "$func"
+      "$func" "$@"
     fi
   )
 
@@ -70,7 +72,7 @@ callback() {
     if [[ -f "$OZONE_UPGRADE_CALLBACK" ]]; then
       source "$OZONE_UPGRADE_CALLBACK"
       if [[ "$(type -t "$func")" = function ]]; then
-        "$func"
+        "$func" "$@"
       fi
     fi
   )

hadoop-ozone/dist/src/main/compose/upgrade/upgrades/rolling-upgrade/callbacks/common/callback.sh

@@ -19,10 +19,58 @@ source "$TEST_DIR"/testlib.sh
 
 ### CALLBACKS ###
 
-before_service_restart() {
-  generate "generate-${SERVICE}" "$CLIENT"
+## @description Run while a service is stopped, it'll generate data on the cluster (upgrade/generate.robot)
+## @param The prefix to use for data generated.
+with_service_stopped() {
+  generate "${1}" "$CLIENT"
 }
 
-after_service_restart() {
-  validate "generate-${SERVICE}" "$CLIENT"
+## @description Run while a service is started again (with new image), it'll validate data on the cluster (upgrade/validate.robot)
+## @param The prefix to use for data generated.
+with_service_restarted() {
+  validate "${1}" "$CLIENT"
+}
+
+with_old_version() {
+  execute_robot_test "$CLIENT" -N "${OUTPUT_NAME}-check-finalization" --include finalized upgrade/check-finalization.robot
+  generate old1 "$CLIENT"
+  validate old1 "$CLIENT"
+
+  generate overwrite "$CLIENT"
+}
+
+with_this_version_pre_finalized() {
+  # No check for pre-finalized status here, because the release may not have
+  # added layout features to OM or HDDS.
+  validate old1 "$CLIENT"
+
+  generate new1 "$CLIENT"
+  validate new1 "$CLIENT"
+
+  validate overwrite "$CLIENT"
+  generate overwrite "$CLIENT" --exclude create-volume-and-bucket
+}
+
+with_old_version_downgraded() {
+  execute_robot_test "$CLIENT" -N "${OUTPUT_NAME}-check-finalization" --include finalized upgrade/check-finalization.robot
+  validate old1 "$CLIENT"
+  validate new1 "$CLIENT"
+
+  generate old2 "$CLIENT"
+  validate old2 "$CLIENT"
+
+  validate overwrite "$CLIENT"
+  generate overwrite "$CLIENT" --exclude create-volume-and-bucket
+}
+
+with_this_version_finalized() {
+  execute_robot_test "$CLIENT" -N "${OUTPUT_NAME}-check-finalization" --include finalized upgrade/check-finalization.robot
+  validate old1 "$CLIENT"
+  validate new1 "$CLIENT"
+  validate old2 "$CLIENT"
+
+  generate new2 "$CLIENT"
+  validate new2 "$CLIENT"
+
+  validate overwrite "$CLIENT"
 }

hadoop-ozone/dist/src/main/compose/upgrade/upgrades/rolling-upgrade/driver.sh

@@ -26,7 +26,6 @@ set -u
 : "${OZONE_UPGRADE_FROM}"
 : "${OZONE_UPGRADE_TO}"
 : "${TEST_DIR}"
-: "${SCM}"
 : "${CLIENT}"
 : "${OZONE_CURRENT_VERSION}"
 set +u
@@ -35,95 +34,124 @@ echo "--- RUNNING ROLLING UPGRADE TEST FROM $OZONE_UPGRADE_FROM TO $OZONE_UPGRAD
 
 source "$TEST_DIR"/testlib.sh
 
-# Restart one service with the target image.
+## @description Restart one service with the target image.
+## @param name of the service
+## @param stage prefix, used for the data prefix
 rolling_restart_service() {
-  SERVICE="$1"
+  local service="$1"
+  local stage_prefix="$2"
 
-  echo "--- RESTARTING ${SERVICE} WITH IMAGE ${OZONE_UPGRADE_TO} ---"
+  local callback_data_prefix="generate-${service}-${stage_prefix}"
 
   # Stop service
-  stop_containers "${SERVICE}"
+  stop_containers "${service}"
 
-  callback before_service_restart
+  callback with_service_stopped "$callback_data_prefix"
 
-  # Restart service with new image.
-  prepare_for_image "${OZONE_UPGRADE_TO}"
-  create_containers "${SERVICE}"
+  # Start service
+  create_containers "${service}"
 
-  callback after_service_restart
+  callback with_service_restarted "$callback_data_prefix"
 
   # Service-specific readiness checks.
-  case "${SERVICE}" in
+  case "${service}" in
     om*)
-      wait_for_port "${SERVICE}" 9862 120
+      wait_for_port "${service}" 9862 120
       ;;
     scm*)
       # SCM hostnames in this compose are scmX.org
-      wait_for_port "${SERVICE}.org" 9876 120
+      wait_for_port "${service}.org" 9876 120
       ;;
     dn*)
-      wait_for_port "${SERVICE}" 9882 120
+      wait_for_port "${service}" 9882 120
       ;;
     s3g*)
-      wait_for_port "${SERVICE}" 9878 120
+      wait_for_port "${service}" 9878 120
       ;;
   esac
 }
 
+## @description Restart all services with the target image.
+## @param stage prefix, used for the generated data as prefix, also for the OUTPUT_NAME
+## @param target image
+rolling_restart_all_services() {
+  local stage_prefix="$1"
+  local target_image="$2"
+  local s
+
+  # Prepare the requested image
+  prepare_for_image "${target_image}"
+  echo "--- PREPARED ${target_image} IMAGE ---"
+
+  # SCMs first
+  for s in scm2 scm1 scm3; do
+    OUTPUT_NAME="${OZONE_UPGRADE_FROM}-${OZONE_UPGRADE_TO}-${stage_prefix}-${s}"
+    echo "--- RESTARTING ${s} WITH IMAGE ${target_image} ---"
+    rolling_restart_service "$s" "$stage_prefix"
+  done
+
+  # Recon
+  OUTPUT_NAME="${OZONE_UPGRADE_FROM}-${OZONE_UPGRADE_TO}-${stage_prefix}-recon"
+  echo "--- RESTARTING recon WITH IMAGE ${target_image} ---"
+  rolling_restart_service "recon" "$stage_prefix"
+
+  # DNs
+  for s in dn1 dn2 dn3 dn4 dn5; do
+    OUTPUT_NAME="${OZONE_UPGRADE_FROM}-${OZONE_UPGRADE_TO}-${stage_prefix}-${s}"
+    echo "--- RESTARTING ${s} WITH IMAGE ${target_image} ---"
+    rolling_restart_service "$s" "$stage_prefix"
+  done
+
+  # OMs
+  for s in om1 om2 om3; do
+    OUTPUT_NAME="${OZONE_UPGRADE_FROM}-${OZONE_UPGRADE_TO}-${stage_prefix}-${s}"
+    echo "--- RESTARTING ${s} WITH IMAGE ${target_image} ---"
+    rolling_restart_service "$s" "$stage_prefix"
+  done
+
+  # S3 Gateways (s3g is HAProxy and does not need to be upgraded)
+  for s in s3g1 s3g2 s3g3; do
+    OUTPUT_NAME="${OZONE_UPGRADE_FROM}-${OZONE_UPGRADE_TO}-${stage_prefix}-${s}"
+    echo "--- RESTARTING ${s} WITH IMAGE ${target_image} ---"
+    rolling_restart_service "$s" "$stage_prefix"
+  done
+}
+
 echo "--- SETTING UP OLD VERSION $OZONE_UPGRADE_FROM ---"
 OUTPUT_NAME="${OZONE_UPGRADE_FROM}-${OZONE_UPGRADE_TO}-1-original"
-export OM_HA_ARGS='--'
 prepare_for_image "$OZONE_UPGRADE_FROM"
 
 echo "--- RUNNING WITH OLD VERSION $OZONE_UPGRADE_FROM ---"
 start_docker_env
 
-# TODO Add old data generation
+callback with_old_version
 
 echo "--- ROLLING UPGRADE TO $OZONE_UPGRADE_TO PRE-FINALIZED ---"
+rolling_restart_all_services "2-upgrade" "$OZONE_UPGRADE_TO"
 
-# SCMs first
-for s in scm2 scm1 scm3; do
-  OUTPUT_NAME="${OZONE_UPGRADE_FROM}-${OZONE_UPGRADE_TO}-2-${s}"
-  rolling_restart_service "$s" "$OZONE_UPGRADE_TO"
-done
-
-# Recon
-OUTPUT_NAME="${OZONE_UPGRADE_FROM}-${OZONE_UPGRADE_TO}-2-recon"
-rolling_restart_service "recon" "$OZONE_UPGRADE_TO"
+OUTPUT_NAME="${OZONE_UPGRADE_FROM}-${OZONE_UPGRADE_TO}-2-pre-finalized"
+callback with_this_version_pre_finalized
 
-# DNs
-for s in dn1 dn2 dn3 dn4 dn5; do
-  OUTPUT_NAME="${OZONE_UPGRADE_FROM}-${OZONE_UPGRADE_TO}-2-${s}"
-  rolling_restart_service "$s" "$OZONE_UPGRADE_TO"
-done
+echo "--- ROLLING DOWNGRADE TO $OZONE_UPGRADE_FROM ---"
+rolling_restart_all_services "3-downgrade" "$OZONE_UPGRADE_FROM"
 
-# OMs
-for s in om1 om2 om3; do
-  OUTPUT_NAME="${OZONE_UPGRADE_FROM}-${OZONE_UPGRADE_TO}-2-${s}"
-  rolling_restart_service "$s" "$OZONE_UPGRADE_TO"
-done
+OUTPUT_NAME="${OZONE_UPGRADE_FROM}-${OZONE_UPGRADE_TO}-3-downgraded"
+callback with_old_version_downgraded
 
-# S3 Gateways (s3g is HAProxy and does not need to be upgraded)
-for s in s3g1 s3g2 s3g3; do
-  OUTPUT_NAME="${OZONE_UPGRADE_FROM}-${OZONE_UPGRADE_TO}-2-${s}"
-  rolling_restart_service "$s" "$OZONE_UPGRADE_TO"
-done
-
-# TODO Add downgrade scenario
+echo "--- ROLLING UPGRADE TO $OZONE_UPGRADE_TO ---"
+rolling_restart_all_services "4-upgrade" "$OZONE_UPGRADE_TO"
 
 # Upgrade client after all server components are upgraded but before finalization,
 # so new client APIs can be exercised against pre-finalized servers.
 echo "--- UPGRADING CLIENT TO $OZONE_UPGRADE_TO ---"
-OUTPUT_NAME="${OZONE_UPGRADE_FROM}-${OZONE_UPGRADE_TO}-2-client"
+OUTPUT_NAME="${OZONE_UPGRADE_FROM}-${OZONE_UPGRADE_TO}-4-upgrade-client"
 stop_containers "$CLIENT"
 prepare_for_image "${OZONE_UPGRADE_TO}"
 create_containers "$CLIENT"
 
 echo "--- RUNNING WITH NEW VERSION $OZONE_UPGRADE_TO FINALIZED ---"
-OUTPUT_NAME="${OZONE_UPGRADE_FROM}-${OZONE_UPGRADE_TO}-3-finalized"
-
-# TODO Add validation for pre-finalized state
+OUTPUT_NAME="${OZONE_UPGRADE_FROM}-${OZONE_UPGRADE_TO}-5-finalized"
 
 # Sends commands to finalize OM and SCM.
 execute_robot_test "$CLIENT" -N "${OUTPUT_NAME}-finalize" upgrade/finalize.robot
+callback with_this_version_finalized