diff --git a/hadoop-ozone/dist/src/main/compose/common/init-kdc.sh b/hadoop-ozone/dist/src/main/compose/common/init-kdc.sh index 3fd00a2b4f36..95a052ffb8b6 100755 --- a/hadoop-ozone/dist/src/main/compose/common/init-kdc.sh +++ b/hadoop-ozone/dist/src/main/compose/common/init-kdc.sh @@ -31,6 +31,7 @@ export_keytab HTTP/scm scm export_keytab testuser/scm scm export_keytab testuser2/scm scm +export_keytab testuser/client testuser export_keytab testuser/dn testuser export_keytab testuser/httpfs testuser export_keytab testuser/om testuser @@ -38,6 +39,7 @@ export_keytab testuser/recon testuser export_keytab testuser/s3g testuser export_keytab testuser/scm testuser +export_keytab testuser2/client testuser2 export_keytab testuser2/dn testuser2 export_keytab testuser2/httpfs testuser2 export_keytab testuser2/om testuser2 diff --git a/hadoop-ozone/dist/src/main/compose/upgrade/README.md b/hadoop-ozone/dist/src/main/compose/upgrade/README.md index 3565d0e067ea..6138a7b38305 100644 --- a/hadoop-ozone/dist/src/main/compose/upgrade/README.md +++ b/hadoop-ozone/dist/src/main/compose/upgrade/README.md @@ -100,6 +100,7 @@ Docker compose cluster definitions to be used in upgrade testing are defined in - `SCM`: The name of the SCM container to run robot tests from. - This can be passed as the first argument to `execute_robot_test`. - This allows the same tests to work with and without SCM HA. + - `CLIENT`: The dedicated client container used to run Robot tests in rolling-upgrade HA. ### Testing New Versions diff --git a/hadoop-ozone/dist/src/main/compose/upgrade/compose/ha/docker-compose.yaml b/hadoop-ozone/dist/src/main/compose/upgrade/compose/ha/docker-compose.yaml index af34b86566f0..c9db073f825b 100644 --- a/hadoop-ozone/dist/src/main/compose/upgrade/compose/ha/docker-compose.yaml +++ b/hadoop-ozone/dist/src/main/compose/upgrade/compose/ha/docker-compose.yaml @@ -322,6 +322,17 @@ services: - *krb5conf - *ozone-dir - *transformation + client: + command: ["sleep","1000000"] + <<: *common-config + hostname: client + networks: + net: + ipv4_address: 10.9.0.27 + volumes: + - *keytabs + - *krb5conf + - *ozone-dir networks: net: diff --git a/hadoop-ozone/dist/src/main/compose/upgrade/compose/ha/load.sh b/hadoop-ozone/dist/src/main/compose/upgrade/compose/ha/load.sh index 4b38c86a73d0..7356c6911ab5 100755 --- a/hadoop-ozone/dist/src/main/compose/upgrade/compose/ha/load.sh +++ b/hadoop-ozone/dist/src/main/compose/upgrade/compose/ha/load.sh @@ -24,6 +24,7 @@ set +u source "$TEST_DIR/testlib.sh" export COMPOSE_FILE="$TEST_DIR/compose/ha/docker-compose.yaml" +export CLIENT=client export OM_SERVICE_ID=omservice export SECURITY_ENABLED="true" diff --git a/hadoop-ozone/dist/src/main/compose/upgrade/testlib.sh b/hadoop-ozone/dist/src/main/compose/upgrade/testlib.sh index a418a96423a2..c83fc2850144 100755 --- a/hadoop-ozone/dist/src/main/compose/upgrade/testlib.sh +++ b/hadoop-ozone/dist/src/main/compose/upgrade/testlib.sh @@ -117,16 +117,18 @@ run_test() { ## @description Generates data on the cluster. ## @param The prefix to use for data generated. -## @param All parameters after the first one are passed directly to the robot command, +## @param The container to run the robot test in. +## @param All remaining parameters are passed directly to the robot command, ## see https://robotframework.org/robotframework/latest/RobotFrameworkUserGuide.html#all-command-line-options generate() { - execute_robot_test "$SCM" -N "${OUTPUT_NAME}-generate-${1}" -v PREFIX:"$1" ${@:2} upgrade/generate.robot + execute_robot_test "${2}" -N "${OUTPUT_NAME}-generate-${1}" -v PREFIX:"$1" ${@:3} upgrade/generate.robot } ## @description Validates that data exists on the cluster. ## @param The prefix of the data to be validated. -## @param All parameters after the first one are passed directly to the robot command, +## @param The container to run the robot test in. +## @param All remaining parameters are passed directly to the robot command, ## see https://robotframework.org/robotframework/latest/RobotFrameworkUserGuide.html#all-command-line-options validate() { - execute_robot_test "$SCM" -N "${OUTPUT_NAME}-validate-${1}" -v PREFIX:"$1" ${@:2} upgrade/validate.robot + execute_robot_test "${2}" -N "${OUTPUT_NAME}-validate-${1}" -v PREFIX:"$1" ${@:3} upgrade/validate.robot } diff --git a/hadoop-ozone/dist/src/main/compose/upgrade/upgrades/non-rolling-upgrade/callbacks/common/callback.sh b/hadoop-ozone/dist/src/main/compose/upgrade/upgrades/non-rolling-upgrade/callbacks/common/callback.sh index c2159ca0cd24..e4a419f00df1 100755 --- a/hadoop-ozone/dist/src/main/compose/upgrade/upgrades/non-rolling-upgrade/callbacks/common/callback.sh +++ b/hadoop-ozone/dist/src/main/compose/upgrade/upgrades/non-rolling-upgrade/callbacks/common/callback.sh @@ -21,40 +21,40 @@ source "$TEST_DIR"/testlib.sh with_old_version() { execute_robot_test "$SCM" -N "${OUTPUT_NAME}-check-finalization" --include finalized upgrade/check-finalization.robot - generate old1 - validate old1 + generate old1 "$SCM" + validate old1 "$SCM" } with_this_version_pre_finalized() { # No check for pre-finalized status here, because the release may not have # added layout features to OM or HDDS. - validate old1 + validate old1 "$SCM" # HDDS-6261: overwrite the same keys intentionally - generate old1 --exclude create-volume-and-bucket + generate old1 "$SCM" --exclude create-volume-and-bucket - generate new1 - validate new1 + generate new1 "$SCM" + validate new1 "$SCM" } with_old_version_downgraded() { execute_robot_test "$SCM" -N "${OUTPUT_NAME}-check-finalization" --include finalized upgrade/check-finalization.robot - validate old1 - validate new1 + validate old1 "$SCM" + validate new1 "$SCM" - generate old2 - validate old2 + generate old2 "$SCM" + validate old2 "$SCM" # HDDS-6261: overwrite the same keys again to trigger the precondition check # that exists <= 1.1.0 OM - generate old1 --exclude create-volume-and-bucket + generate old1 "$SCM" --exclude create-volume-and-bucket } with_this_version_finalized() { execute_robot_test "$SCM" -N "${OUTPUT_NAME}-check-finalization" --include finalized upgrade/check-finalization.robot - validate old1 - validate new1 - validate old2 + validate old1 "$SCM" + validate new1 "$SCM" + validate old2 "$SCM" - generate new2 - validate new2 + generate new2 "$SCM" + validate new2 "$SCM" } diff --git a/hadoop-ozone/dist/src/main/compose/upgrade/upgrades/rolling-upgrade/callbacks/common/callback.sh b/hadoop-ozone/dist/src/main/compose/upgrade/upgrades/rolling-upgrade/callbacks/common/callback.sh index 5160b4ef384b..e5e693e41b51 100644 --- a/hadoop-ozone/dist/src/main/compose/upgrade/upgrades/rolling-upgrade/callbacks/common/callback.sh +++ b/hadoop-ozone/dist/src/main/compose/upgrade/upgrades/rolling-upgrade/callbacks/common/callback.sh @@ -20,9 +20,9 @@ source "$TEST_DIR"/testlib.sh ### CALLBACKS ### before_service_restart() { - generate "generate-${SERVICE}" + generate "generate-${SERVICE}" "$CLIENT" } after_service_restart() { - validate "generate-${SERVICE}" + validate "generate-${SERVICE}" "$CLIENT" } diff --git a/hadoop-ozone/dist/src/main/compose/upgrade/upgrades/rolling-upgrade/driver.sh b/hadoop-ozone/dist/src/main/compose/upgrade/upgrades/rolling-upgrade/driver.sh index e2972a403e5d..a9bf70300213 100755 --- a/hadoop-ozone/dist/src/main/compose/upgrade/upgrades/rolling-upgrade/driver.sh +++ b/hadoop-ozone/dist/src/main/compose/upgrade/upgrades/rolling-upgrade/driver.sh @@ -27,6 +27,7 @@ set -u : "${OZONE_UPGRADE_TO}" : "${TEST_DIR}" : "${SCM}" +: "${CLIENT}" : "${OZONE_CURRENT_VERSION}" set +u @@ -43,17 +44,6 @@ rolling_restart_service() { # Stop service stop_containers "${SERVICE}" - # Check if this SCM container is running, as during a rolling upgrade it does stop-start one-by-one and - # we want to run write/read tests while one service is unavailable. Choose SCM (the container where the generate and - # validate robot tests are running) considering availability. - if [[ "$(docker inspect -f '{{.State.Running}}' "ha-${SCM}-1" 2>/dev/null)" != "true" ]]; then - local fallback_scm - fallback_scm="$(docker-compose --project-directory="$TEST_DIR/compose/ha" config --services | grep scm | grep -v "^${SCM}$" | head -n1)" - if [[ -n "$fallback_scm" ]]; then - export SCM="$fallback_scm" - fi - fi - callback before_service_restart # Restart service with new image. @@ -122,10 +112,18 @@ done # TODO Add downgrade scenario +# Upgrade client after all server components are upgraded but before finalization, +# so new client APIs can be exercised against pre-finalized servers. +echo "--- UPGRADING CLIENT TO $OZONE_UPGRADE_TO ---" +OUTPUT_NAME="${OZONE_UPGRADE_FROM}-${OZONE_UPGRADE_TO}-2-client" +stop_containers "$CLIENT" +prepare_for_image "${OZONE_UPGRADE_TO}" +create_containers "$CLIENT" + echo "--- RUNNING WITH NEW VERSION $OZONE_UPGRADE_TO FINALIZED ---" OUTPUT_NAME="${OZONE_UPGRADE_FROM}-${OZONE_UPGRADE_TO}-3-finalized" # TODO Add validation for pre-finalized state # Sends commands to finalize OM and SCM. -execute_robot_test "$SCM" -N "${OUTPUT_NAME}-finalize" upgrade/finalize.robot +execute_robot_test "$CLIENT" -N "${OUTPUT_NAME}-finalize" upgrade/finalize.robot