diff --git a/doc/local-tests.md b/doc/local-tests.md index 411418f3b..fb13a79ed 100644 --- a/doc/local-tests.md +++ b/doc/local-tests.md @@ -34,3 +34,11 @@ TEST_MYSQL_IMAGE="mysql-server:8.0.16" ./script/docker-gh-ost-replica-tests up # cleanup containers ./script/docker-gh-ost-replica-tests down ``` + +Pass the `-t` flag to run the tests with a toxiproxy between gh-ost and the MySQL replica. This simulates network conditions where MySQL connections are closed unexpectedly. + +```shell +# run tests with toxiproxy +./script/docker-gh-ost-replica-tests up -t +./script/docker-gh-ost-replica-tests run -t +``` diff --git a/go/cmd/gh-ost/main.go b/go/cmd/gh-ost/main.go index 027bfc86e..feabd8658 100644 --- a/go/cmd/gh-ost/main.go +++ b/go/cmd/gh-ost/main.go @@ -143,6 +143,7 @@ func main() { flag.BoolVar(&migrationContext.IncludeTriggers, "include-triggers", false, "When true, the triggers (if exist) will be created on the new table") flag.StringVar(&migrationContext.TriggerSuffix, "trigger-suffix", "", "Add a suffix to the trigger name (i.e '_v2'). Requires '--include-triggers'") flag.BoolVar(&migrationContext.RemoveTriggerSuffix, "remove-trigger-suffix-if-exists", false, "Remove given suffix from name of trigger. Requires '--include-triggers' and '--trigger-suffix'") + flag.BoolVar(&migrationContext.SkipPortValidation, "skip-port-validation", false, "Skip port validation for MySQL connections") maxLoad := flag.String("max-load", "", "Comma delimited status-name=threshold. e.g: 'Threads_running=100,Threads_connected=500'. When status exceeds threshold, app throttles writes") criticalLoad := flag.String("critical-load", "", "Comma delimited status-name=threshold, same format as --max-load. When status exceeds threshold, app panics and quits") diff --git a/localtests/test.sh b/localtests/test.sh index f595c1b25..5cdc65f7e 100755 --- a/localtests/test.sh +++ b/localtests/test.sh @@ -12,6 +12,7 @@ test_logfile=/tmp/gh-ost-test.log default_ghost_binary=/tmp/gh-ost-test ghost_binary="" docker=false +toxiproxy=false storage_engine=innodb exec_command_file=/tmp/gh-ost-test.bash ghost_structure_output_file=/tmp/gh-ost-test.ghost.structure.sql @@ -29,7 +30,7 @@ original_sql_mode= sysbench_pid= OPTIND=1 -while getopts "b:s:d" OPTION; do +while getopts "b:s:dt" OPTION; do case $OPTION in b) ghost_binary="$OPTARG" @@ -37,6 +38,9 @@ while getopts "b:s:d" OPTION; do s) storage_engine="$OPTARG" ;; + t) + toxiproxy=true + ;; d) docker=true ;; @@ -75,6 +79,20 @@ verify_master_and_replica() { read replica_host replica_port <<<$(gh-ost-test-mysql-replica -e "select @@hostname, @@port" -ss) [ "$replica_host" == "$(hostname)" ] && replica_host="127.0.0.1" echo "# replica verified at $replica_host:$replica_port" + + if [ "$docker" = true ]; then + master_host="0.0.0.0" + master_port="3307" + echo "# using docker master at $master_host:$master_port" + replica_host="0.0.0.0" + if [ "$toxiproxy" = true ]; then + replica_port="23308" + echo "# using toxiproxy replica at $replica_host:$replica_port" + else + replica_port="3308" + echo "# using docker replica at $replica_host:$replica_port" + fi + fi } exec_cmd() { @@ -154,13 +172,6 @@ test_single() { local test_name test_name="$1" - if [ "$docker" = true ]; then - master_host="0.0.0.0" - master_port="3307" - replica_host="0.0.0.0" - replica_port="3308" - fi - if [ -f $tests_path/$test_name/ignore_versions ]; then ignore_versions=$(cat $tests_path/$test_name/ignore_versions) mysql_version=$(gh-ost-test-mysql-master -s -s -e "select @@version") @@ -199,6 +210,9 @@ test_single() { if [ -f $tests_path/$test_name/extra_args ]; then extra_args=$(cat $tests_path/$test_name/extra_args) fi + if [ "$toxiproxy" = true ]; then + extra_args+=" --skip-port-validation" + fi orig_columns="*" ghost_columns="*" order_by="" diff --git a/script/docker-gh-ost-replica-tests b/script/docker-gh-ost-replica-tests index 4469694d3..4c068e6d6 100755 --- a/script/docker-gh-ost-replica-tests +++ b/script/docker-gh-ost-replica-tests @@ -5,11 +5,15 @@ # Set the environment var TEST_MYSQL_IMAGE to change the docker image. # # Usage: -# docker-gh-ost-replica-tests up start the containers -# docker-gh-ost-replica-tests down remove the containers -# docker-gh-ost-replica-tests run run replica tests on the containers +# docker-gh-ost-replica-tests up [-t] start the containers +# docker-gh-ost-replica-tests down remove the containers +# docker-gh-ost-replica-tests run [-t] run replica tests on the containers +# +# Flags: +# -t use a toxiproxy for replica connection to simulate dropped connections set -e +toxiproxy=false GH_OST_ROOT=$(git rev-parse --show-toplevel) if [[ ":$PATH:" != *":$GH_OST_ROOT:"* ]]; then @@ -47,6 +51,22 @@ mysql-replica() { fi } +create_toxiproxy() { + curl --fail -X POST http://localhost:8474/proxies \ + -H "Content-Type: application/json" \ + -d '{"name": "mysql_proxy", + "listen": "0.0.0.0:23308", + "upstream": "host.docker.internal:3308"}' + echo + + curl --fail -X POST http://localhost:8474/proxies/mysql_proxy/toxics \ + -H "Content-Type: application/json" \ + -d '{"name": "limit_data_downstream", + "type": "limit_data", + "attributes": {"bytes": 300000}}' + echo +} + setup() { [ -z "$TEST_MYSQL_IMAGE" ] && TEST_MYSQL_IMAGE="mysql:8.0.41" @@ -59,6 +79,22 @@ setup() { MYSQL_SHA2_RSA_KEYS_FLAG="--caching-sha2-password-auto-generate-rsa-keys=ON" fi (TEST_MYSQL_IMAGE="$TEST_MYSQL_IMAGE" MYSQL_SHA2_RSA_KEYS_FLAG="$MYSQL_SHA2_RSA_KEYS_FLAG" envsubst <"$compose_file") >"$compose_file.tmp" + + if [ "$toxiproxy" = true ]; then + echo "Starting toxiproxy container..." + cat <>"$compose_file.tmp" + mysql-toxiproxy: + image: "ghcr.io/shopify/toxiproxy:latest" + container_name: mysql-toxiproxy + ports: + - '8474:8474' + - '23308:23308' + expose: + - '23308' + - '8474' +EOF + fi + docker compose -f "$compose_file.tmp" up -d --wait echo "Waiting for MySQL..." @@ -80,23 +116,36 @@ setup() { mysql-replica -e "start slave;" fi echo "OK" + + if [ "$toxiproxy" = true ]; then + echo "Creating toxiproxy..." + create_toxiproxy + echo "OK" + fi } teardown() { echo "Stopping containers..." docker stop mysql-replica docker stop mysql-primary + docker stop mysql-toxiproxy 2>/dev/null || true echo "Removing containers..." docker rm mysql-replica docker rm mysql-primary + docker rm mysql-toxiproxy 2>/dev/null || true } main() { - if [[ "$1" == "up" ]]; then + local cmd="$1" + local tflag= + if [[ "$2" == "-t" ]]; then + toxiproxy=true + fi + if [[ "$cmd" == "up" ]]; then setup - elif [[ "$1" == "down" ]]; then + elif [[ "$cmd" == "down" ]]; then teardown - elif [[ "$1" == "run" ]]; then + elif [[ "$cmd" == "run" ]]; then shift 1 "$GH_OST_ROOT/localtests/test.sh" -d "$@" fi