From aaaec7e76eab48b879270b5796cb4a0cba8a3cc8 Mon Sep 17 00:00:00 2001
From: Jan Grodowski <jan.grodowski@shopify.com>
Date: Wed, 9 Jul 2025 13:51:12 +0200
Subject: [PATCH 1/3] Panic if InitiateHeartbeat exhausts retries to avoid
 looping infinitely.

Based on experience, if the writer database fails inbeetween the copy & cutover stages (e.g. during cutover pause), the heartbeat writes will fail and stop,
then leading to throttled state and an infinite loop of throttler.shouldThrottle().

Since this state is irrecoverable, make the heartbeat writer panic if retries are exhausted, so that the migration can fail and be restarted later.
---
 go/logic/applier.go | 1 +
 1 file changed, 1 insertion(+)

diff --git a/go/logic/applier.go b/go/logic/applier.go
index a7edaed24..b3b77e98d 100644
--- a/go/logic/applier.go
+++ b/go/logic/applier.go
@@ -574,6 +574,7 @@ func (this *Applier) InitiateHeartbeat() {
 			continue
 		}
 		if err := injectHeartbeat(); err != nil {
+			this.migrationContext.PanicAbort <- fmt.Errorf("injectHeartbeat writing failed %d times, last error: %w", numSuccessiveFailures, err)
 			return
 		}
 	}

From 2f0f85bff050f17e9d3e6a3824b51a6c682d77ac Mon Sep 17 00:00:00 2001
From: meiji163 <meiji163@github.com>
Date: Wed, 8 Oct 2025 13:31:01 -0700
Subject: [PATCH 2/3] Add sysbench localtest (#1590)

* add sysbench localtest

* fix table name

* Apply suggestion from @Copilot

Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>

* ensure cleanup

---------

Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>
---
 .github/workflows/replica-tests.yml |   5 +
 localtests/sysbench/create.sql      |  13 +
 localtests/test.sh                  | 518 ++++++++++++++++------------
 3 files changed, 311 insertions(+), 225 deletions(-)
 create mode 100644 localtests/sysbench/create.sql

diff --git a/.github/workflows/replica-tests.yml b/.github/workflows/replica-tests.yml
index 7eb1deb74..9641727a2 100644
--- a/.github/workflows/replica-tests.yml
+++ b/.github/workflows/replica-tests.yml
@@ -15,6 +15,11 @@ jobs:
     steps:
       - uses: actions/checkout@v4
 
+      - name: Install sysbench
+        run: |
+          sudo apt-get update
+          sudo apt-get install -y sysbench
+
       - name: Setup environment
         run: script/docker-gh-ost-replica-tests up
 
diff --git a/localtests/sysbench/create.sql b/localtests/sysbench/create.sql
new file mode 100644
index 000000000..e26a3ef4c
--- /dev/null
+++ b/localtests/sysbench/create.sql
@@ -0,0 +1,13 @@
+/* This table will get created by `sysbench prepare` */
+/*
+CREATE TABLE `sbtest1` (
+  `id` int NOT NULL AUTO_INCREMENT,
+  `k` int NOT NULL DEFAULT '0',
+  `c` char(120) NOT NULL DEFAULT '',
+  `pad` char(60) NOT NULL DEFAULT '',
+  PRIMARY KEY (`id`),
+  KEY `k_1` (`k`)
+) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;
+*/
+
+DROP TABLE IF EXISTS `sbtest1`;
diff --git a/localtests/test.sh b/localtests/test.sh
index e467c113b..f595c1b25 100755
--- a/localtests/test.sh
+++ b/localtests/test.sh
@@ -18,174 +18,241 @@ ghost_structure_output_file=/tmp/gh-ost-test.ghost.structure.sql
 orig_content_output_file=/tmp/gh-ost-test.orig.content.csv
 ghost_content_output_file=/tmp/gh-ost-test.ghost.content.csv
 throttle_flag_file=/tmp/gh-ost-test.ghost.throttle.flag
+table_name=
+ghost_table_name=
 
 master_host=
 master_port=
 replica_host=
 replica_port=
 original_sql_mode=
+sysbench_pid=
 
 OPTIND=1
-while getopts "b:s:d" OPTION
-do
-  case $OPTION in
+while getopts "b:s:d" OPTION; do
+    case $OPTION in
     b)
-      ghost_binary="$OPTARG";;
+        ghost_binary="$OPTARG"
+        ;;
     s)
-      storage_engine="$OPTARG";;
+        storage_engine="$OPTARG"
+        ;;
     d)
-      docker=true;;
-  esac
+        docker=true
+        ;;
+    esac
 done
-shift $((OPTIND-1))
+shift $((OPTIND - 1))
 
 test_pattern="${1:-.}"
 
 verify_master_and_replica() {
-  if [ "$(gh-ost-test-mysql-master -e "select 1" -ss)" != "1" ] ; then
-    echo "Cannot verify gh-ost-test-mysql-master"
-    exit 1
-  fi
-  read master_host master_port <<< $(gh-ost-test-mysql-master -e "select @@hostname, @@port" -ss)
-  [ "$master_host" == "$(hostname)" ] && master_host="127.0.0.1"
-  echo "# master verified at $master_host:$master_port"
-  if ! gh-ost-test-mysql-master -e "set global event_scheduler := 1" ; then
-    echo "Cannot enable event_scheduler on master"
-    exit 1
-  fi
-  original_sql_mode="$(gh-ost-test-mysql-master -e "select @@global.sql_mode" -s -s)"
-  echo "sql_mode on master is ${original_sql_mode}"
-
-  echo "Gracefully sleeping for 3 seconds while replica is setting up..."
-  sleep 3
-
-  if [ "$(gh-ost-test-mysql-replica -e "select 1" -ss)" != "1" ] ; then
-    echo "Cannot verify gh-ost-test-mysql-replica"
-    exit 1
-  fi
-  if [ "$(gh-ost-test-mysql-replica -e "select @@global.binlog_format" -ss)" != "ROW" ] ; then
-    echo "Expecting test replica to have binlog_format=ROW"
-    exit 1
-  fi
-  read replica_host replica_port <<< $(gh-ost-test-mysql-replica -e "select @@hostname, @@port" -ss)
-  [ "$replica_host" == "$(hostname)" ] && replica_host="127.0.0.1"
-  echo "# replica verified at $replica_host:$replica_port"
+    if [ "$(gh-ost-test-mysql-master -e "select 1" -ss)" != "1" ]; then
+        echo "Cannot verify gh-ost-test-mysql-master"
+        exit 1
+    fi
+    read master_host master_port <<<$(gh-ost-test-mysql-master -e "select @@hostname, @@port" -ss)
+    [ "$master_host" == "$(hostname)" ] && master_host="127.0.0.1"
+    echo "# master verified at $master_host:$master_port"
+    if ! gh-ost-test-mysql-master -e "set global event_scheduler := 1"; then
+        echo "Cannot enable event_scheduler on master"
+        exit 1
+    fi
+    original_sql_mode="$(gh-ost-test-mysql-master -e "select @@global.sql_mode" -s -s)"
+    echo "sql_mode on master is ${original_sql_mode}"
+
+    echo "Gracefully sleeping for 3 seconds while replica is setting up..."
+    sleep 3
+
+    if [ "$(gh-ost-test-mysql-replica -e "select 1" -ss)" != "1" ]; then
+        echo "Cannot verify gh-ost-test-mysql-replica"
+        exit 1
+    fi
+    if [ "$(gh-ost-test-mysql-replica -e "select @@global.binlog_format" -ss)" != "ROW" ]; then
+        echo "Expecting test replica to have binlog_format=ROW"
+        exit 1
+    fi
+    read replica_host replica_port <<<$(gh-ost-test-mysql-replica -e "select @@hostname, @@port" -ss)
+    [ "$replica_host" == "$(hostname)" ] && replica_host="127.0.0.1"
+    echo "# replica verified at $replica_host:$replica_port"
 }
 
 exec_cmd() {
-  echo "$@"
-  command "$@" 1> $test_logfile 2>&1
-  return $?
+    echo "$@"
+    command "$@" 1>$test_logfile 2>&1
+    return $?
 }
 
 echo_dot() {
-  echo -n "."
+    echo -n "."
 }
 
 start_replication() {
-  mysql_version="$(gh-ost-test-mysql-replica -e  "select @@version")"
-  if [[ $mysql_version =~ "8.4" ]]; then
-    seconds_behind_source="Seconds_Behind_Source"
-    replica_terminology="replica"
-  else
-    seconds_behind_source="Seconds_Behind_Master"
-    replica_terminology="slave"
-  fi
-  gh-ost-test-mysql-replica -e "stop $replica_terminology; start $replica_terminology;"
-
-  num_attempts=0
-  while gh-ost-test-mysql-replica -e "show $replica_terminology status\G" | grep $seconds_behind_source | grep -q NULL ; do
-    ((num_attempts=num_attempts+1))
-    if [ $num_attempts -gt 10 ] ; then
-      echo
-      echo "ERROR replication failure"
-      exit 1
+    mysql_version="$(gh-ost-test-mysql-replica -e "select @@version")"
+    if [[ $mysql_version =~ "8.4" ]]; then
+        seconds_behind_source="Seconds_Behind_Source"
+        replica_terminology="replica"
+    else
+        seconds_behind_source="Seconds_Behind_Master"
+        replica_terminology="slave"
+    fi
+    gh-ost-test-mysql-replica -e "stop $replica_terminology; start $replica_terminology;"
+
+    num_attempts=0
+    while gh-ost-test-mysql-replica -e "show $replica_terminology status\G" | grep $seconds_behind_source | grep -q NULL; do
+        ((num_attempts = num_attempts + 1))
+        if [ $num_attempts -gt 10 ]; then
+            echo
+            echo "ERROR replication failure"
+            exit 1
+        fi
+        echo_dot
+        sleep 1
+    done
+}
+
+sysbench_prepare() {
+    local mysql_host="$1"
+    local mysql_port="$2"
+    sysbench oltp_write_only \
+        --mysql-host="$mysql_host" \
+        --mysql-port="$mysql_port" \
+        --mysql-user=root \
+        --mysql-password=opensesame \
+        --mysql-db=test \
+        --tables=1 \
+        --table-size=10000 \
+        prepare
+}
+
+sysbench_run_cmd() {
+    local mysql_host="$1"
+    local mysql_port="$2"
+    cmd="sysbench oltp_write_only \
+    --mysql-host="$mysql_host" \
+    --mysql-port="$mysql_port" \
+    --mysql-user=root \
+    --mysql-password=opensesame \
+    --mysql-db=test \
+    --rand-seed=163 \
+    --tables=1 \
+    --threads=2 \
+    --time=30 \
+    --report-interval=10 \
+    --rate=500 \
+    run"
+    echo $cmd
+}
+
+cleanup() {
+    if ! [ -z $sysbench_pid ] && ps -p $sysbench_pid >/dev/null; then
+        kill $sysbench_pid
     fi
-    echo_dot
-    sleep 1
-  done
 }
 
 test_single() {
-  local test_name
-  test_name="$1"
-
-  if [ "$docker" = true ]; then
-    master_host="0.0.0.0"
-    master_port="3307"
-    replica_host="0.0.0.0"
-    replica_port="3308"
-  fi
-
-  if [ -f $tests_path/$test_name/ignore_versions ] ; then
-    ignore_versions=$(cat $tests_path/$test_name/ignore_versions)
-    mysql_version=$(gh-ost-test-mysql-master -s -s -e "select @@version")
-    mysql_version_comment=$(gh-ost-test-mysql-master -s -s -e "select @@version_comment")
-    if echo "$mysql_version" | egrep -q "^${ignore_versions}" ; then
-      echo -n "Skipping: $test_name"
-      return 0
-    elif echo "$mysql_version_comment" | egrep -i -q "^${ignore_versions}" ; then
-      echo -n "Skipping: $test_name"
-      return 0
+    local test_name
+    test_name="$1"
+
+    if [ "$docker" = true ]; then
+        master_host="0.0.0.0"
+        master_port="3307"
+        replica_host="0.0.0.0"
+        replica_port="3308"
+    fi
+
+    if [ -f $tests_path/$test_name/ignore_versions ]; then
+        ignore_versions=$(cat $tests_path/$test_name/ignore_versions)
+        mysql_version=$(gh-ost-test-mysql-master -s -s -e "select @@version")
+        mysql_version_comment=$(gh-ost-test-mysql-master -s -s -e "select @@version_comment")
+        if echo "$mysql_version" | egrep -q "^${ignore_versions}"; then
+            echo -n "Skipping: $test_name"
+            return 0
+        elif echo "$mysql_version_comment" | egrep -i -q "^${ignore_versions}"; then
+            echo -n "Skipping: $test_name"
+            return 0
+        fi
+    fi
+
+    echo -n "Testing: $test_name"
+
+    echo_dot
+    start_replication
+    echo_dot
+
+    if [ -f $tests_path/$test_name/sql_mode ]; then
+        gh-ost-test-mysql-master --default-character-set=utf8mb4 test -e "set @@global.sql_mode='$(cat $tests_path/$test_name/sql_mode)'"
+        gh-ost-test-mysql-replica --default-character-set=utf8mb4 test -e "set @@global.sql_mode='$(cat $tests_path/$test_name/sql_mode)'"
+    fi
+
+    gh-ost-test-mysql-master --default-character-set=utf8mb4 test <$tests_path/$test_name/create.sql
+    test_create_result=$?
+
+    if [ $test_create_result -ne 0 ]; then
+        echo
+        echo "ERROR $test_name create failure. cat $tests_path/$test_name/create.sql:"
+        cat $tests_path/$test_name/create.sql
+        return 1
+    fi
+
+    extra_args=""
+    if [ -f $tests_path/$test_name/extra_args ]; then
+        extra_args=$(cat $tests_path/$test_name/extra_args)
     fi
-  fi
-
-  echo -n "Testing: $test_name"
-
-  echo_dot
-  start_replication
-  echo_dot
-
-  if [ -f $tests_path/$test_name/sql_mode ] ; then
-    gh-ost-test-mysql-master --default-character-set=utf8mb4 test -e "set @@global.sql_mode='$(cat $tests_path/$test_name/sql_mode)'"
-    gh-ost-test-mysql-replica --default-character-set=utf8mb4 test -e "set @@global.sql_mode='$(cat $tests_path/$test_name/sql_mode)'"
-  fi
-
-  gh-ost-test-mysql-master --default-character-set=utf8mb4 test < $tests_path/$test_name/create.sql
-  test_create_result=$?
-
-  if [ $test_create_result -ne 0 ] ; then
-    echo
-    echo "ERROR $test_name create failure. cat $tests_path/$test_name/create.sql:"
-    cat $tests_path/$test_name/create.sql
-    return 1
-  fi
-
-  extra_args=""
-  if [ -f $tests_path/$test_name/extra_args ] ; then
-    extra_args=$(cat $tests_path/$test_name/extra_args)
-  fi
-  orig_columns="*"
-  ghost_columns="*"
-  order_by=""
-  if [ -f $tests_path/$test_name/orig_columns ] ; then
-    orig_columns=$(cat $tests_path/$test_name/orig_columns)
-  fi
-  if [ -f $tests_path/$test_name/ghost_columns ] ; then
-    ghost_columns=$(cat $tests_path/$test_name/ghost_columns)
-  fi
-  if [ -f $tests_path/$test_name/order_by ] ; then
-    order_by="order by $(cat $tests_path/$test_name/order_by)"
-  fi
-  # graceful sleep for replica to catch up
-  echo_dot
-  sleep 1
-  #
-  cmd="$ghost_binary \
+    orig_columns="*"
+    ghost_columns="*"
+    order_by=""
+    if [ -f $tests_path/$test_name/orig_columns ]; then
+        orig_columns=$(cat $tests_path/$test_name/orig_columns)
+    fi
+    if [ -f $tests_path/$test_name/ghost_columns ]; then
+        ghost_columns=$(cat $tests_path/$test_name/ghost_columns)
+    fi
+    if [ -f $tests_path/$test_name/order_by ]; then
+        order_by="order by $(cat $tests_path/$test_name/order_by)"
+    fi
+    # graceful sleep for replica to catch up
+    echo_dot
+    sleep 1
+
+    table_name="gh_ost_test"
+    ghost_table_name="_gh_ost_test_gho"
+    trap cleanup EXIT INT TERM
+    # test with sysbench oltp write load
+    if [[ "$test_name" == "sysbench" ]]; then
+        if ! command -v sysbench &>/dev/null; then
+            echo "skipping"
+            return 0
+        fi
+        table_name="sbtest1"
+        ghost_table_name="_${table_name}_gho"
+        echo "Preparing sysbench..."
+        sysbench_prepare "$master_host" "$master_port"
+
+        load_cmd="$(sysbench_run_cmd $master_host $master_port)"
+        eval "$load_cmd" &
+        sysbench_pid=$!
+        echo
+        echo -n "Started sysbench (PID $sysbench_pid):  "
+        echo $load_cmd
+    fi
+
+    #
+    cmd="$ghost_binary \
     --user=gh-ost \
     --password=gh-ost \
     --host=$replica_host \
     --port=$replica_port \
     --assume-master-host=${master_host}:${master_port}
     --database=test \
-    --table=gh_ost_test \
+    --table=${table_name} \
     --storage-engine=${storage_engine} \
     --alter='engine=${storage_engine}' \
     --exact-rowcount \
     --assume-rbr \
     --initially-drop-old-table \
     --initially-drop-ghost-table \
-    --throttle-query='select timestampdiff(second, min(last_update), now()) < 5 from _gh_ost_test_ghc' \
+    --throttle-query='select timestampdiff(second, min(last_update), now()) < 5 from _${table_name}_ghc' \
     --throttle-flag-file=$throttle_flag_file \
     --serve-socket-file=/tmp/gh-ost.test.sock \
     --initially-drop-socket-file \
@@ -196,118 +263,119 @@ test_single() {
     --debug \
     --stack \
     --execute ${extra_args[@]}"
-  echo_dot
-  echo $cmd > $exec_command_file
-  echo_dot
-  bash $exec_command_file 1> $test_logfile 2>&1
-
-  execution_result=$?
-
-  if [ -f $tests_path/$test_name/sql_mode ] ; then
-    gh-ost-test-mysql-master --default-character-set=utf8mb4 test -e "set @@global.sql_mode='${original_sql_mode}'"
-    gh-ost-test-mysql-replica --default-character-set=utf8mb4 test -e "set @@global.sql_mode='${original_sql_mode}'"
-  fi
-
-  if [ -f $tests_path/$test_name/destroy.sql ] ; then
-    gh-ost-test-mysql-master --default-character-set=utf8mb4 test < $tests_path/$test_name/destroy.sql
-  fi
-
-  if [ -f $tests_path/$test_name/expect_failure ] ; then
-    if [ $execution_result -eq 0 ] ; then
-      echo
-      echo "ERROR $test_name execution was expected to exit on error but did not. cat $test_logfile"
-      return 1
+    echo_dot
+    echo $cmd >$exec_command_file
+    echo_dot
+    bash $exec_command_file 1>$test_logfile 2>&1
+
+    execution_result=$?
+    cleanup
+
+    if [ -f $tests_path/$test_name/sql_mode ]; then
+        gh-ost-test-mysql-master --default-character-set=utf8mb4 test -e "set @@global.sql_mode='${original_sql_mode}'"
+        gh-ost-test-mysql-replica --default-character-set=utf8mb4 test -e "set @@global.sql_mode='${original_sql_mode}'"
     fi
-    if [ -s $tests_path/$test_name/expect_failure ] ; then
-      # 'expect_failure' file has content. We expect to find this content in the log.
-      expected_error_message="$(cat $tests_path/$test_name/expect_failure)"
-      if grep -q "$expected_error_message" $test_logfile ; then
-          return 0
-      fi
-      echo
-      echo "ERROR $test_name execution was expected to exit with error message '${expected_error_message}' but did not. cat $test_logfile"
-      return 1
+
+    if [ -f $tests_path/$test_name/destroy.sql ]; then
+        gh-ost-test-mysql-master --default-character-set=utf8mb4 test <$tests_path/$test_name/destroy.sql
     fi
-    # 'expect_failure' file has no content. We generally agree that the failure is correct
-    return 0
-  fi
-
-  if [ $execution_result -ne 0 ] ; then
-    echo
-    echo "ERROR $test_name execution failure. cat $test_logfile:"
-    cat $test_logfile
-    return 1
-  fi
-
-  gh-ost-test-mysql-replica --default-character-set=utf8mb4 test -e "show create table _gh_ost_test_gho\G" -ss > $ghost_structure_output_file
-
-  if [ -f $tests_path/$test_name/expect_table_structure ] ; then
-    expected_table_structure="$(cat $tests_path/$test_name/expect_table_structure)"
-    if ! grep -q "$expected_table_structure" $ghost_structure_output_file ; then
-      echo
-      echo "ERROR $test_name: table structure was expected to include ${expected_table_structure} but did not. cat $ghost_structure_output_file:"
-      cat $ghost_structure_output_file
-      return 1
+
+    if [ -f $tests_path/$test_name/expect_failure ]; then
+        if [ $execution_result -eq 0 ]; then
+            echo
+            echo "ERROR $test_name execution was expected to exit on error but did not. cat $test_logfile"
+            return 1
+        fi
+        if [ -s $tests_path/$test_name/expect_failure ]; then
+            # 'expect_failure' file has content. We expect to find this content in the log.
+            expected_error_message="$(cat $tests_path/$test_name/expect_failure)"
+            if grep -q "$expected_error_message" $test_logfile; then
+                return 0
+            fi
+            echo
+            echo "ERROR $test_name execution was expected to exit with error message '${expected_error_message}' but did not. cat $test_logfile"
+            return 1
+        fi
+        # 'expect_failure' file has no content. We generally agree that the failure is correct
+        return 0
     fi
-  fi
 
-  echo_dot
-  gh-ost-test-mysql-replica --default-character-set=utf8mb4 test -e "select ${orig_columns} from gh_ost_test ${order_by}" -ss > $orig_content_output_file
-  gh-ost-test-mysql-replica --default-character-set=utf8mb4 test -e "select ${ghost_columns} from _gh_ost_test_gho ${order_by}" -ss > $ghost_content_output_file
-  orig_checksum=$(cat $orig_content_output_file | md5sum)
-  ghost_checksum=$(cat $ghost_content_output_file | md5sum)
+    if [ $execution_result -ne 0 ]; then
+        echo
+        echo "ERROR $test_name execution failure. cat $test_logfile:"
+        cat $test_logfile
+        return 1
+    fi
 
-  if [ "$orig_checksum" != "$ghost_checksum" ] ; then
-    gh-ost-test-mysql-replica --default-character-set=utf8mb4 test -e "select ${orig_columns} from gh_ost_test" -ss > $orig_content_output_file
-    gh-ost-test-mysql-replica --default-character-set=utf8mb4 test -e "select ${ghost_columns} from _gh_ost_test_gho" -ss > $ghost_content_output_file
-    echo "ERROR $test_name: checksum mismatch"
-    echo "---"
-    diff $orig_content_output_file $ghost_content_output_file
+    gh-ost-test-mysql-replica --default-character-set=utf8mb4 test -e "show create table ${ghost_table_name}\G" -ss >$ghost_structure_output_file
 
-    echo "diff $orig_content_output_file $ghost_content_output_file"
+    if [ -f $tests_path/$test_name/expect_table_structure ]; then
+        expected_table_structure="$(cat $tests_path/$test_name/expect_table_structure)"
+        if ! grep -q "$expected_table_structure" $ghost_structure_output_file; then
+            echo
+            echo "ERROR $test_name: table structure was expected to include ${expected_table_structure} but did not. cat $ghost_structure_output_file:"
+            cat $ghost_structure_output_file
+            return 1
+        fi
+    fi
+
+    echo_dot
+    gh-ost-test-mysql-replica --default-character-set=utf8mb4 test -e "select ${orig_columns} from ${table_name} ${order_by}" -ss >$orig_content_output_file
+    gh-ost-test-mysql-replica --default-character-set=utf8mb4 test -e "select ${ghost_columns} from ${ghost_table_name} ${order_by}" -ss >$ghost_content_output_file
+    orig_checksum=$(cat $orig_content_output_file | md5sum)
+    ghost_checksum=$(cat $ghost_content_output_file | md5sum)
+
+    if [ "$orig_checksum" != "$ghost_checksum" ]; then
+        gh-ost-test-mysql-replica --default-character-set=utf8mb4 test -e "select ${orig_columns} from ${table_name}" -ss >$orig_content_output_file
+        gh-ost-test-mysql-replica --default-character-set=utf8mb4 test -e "select ${ghost_columns} from ${ghost_table_name}" -ss >$ghost_content_output_file
+        echo "ERROR $test_name: checksum mismatch"
+        echo "---"
+        diff $orig_content_output_file $ghost_content_output_file
 
-    return 1
-  fi
+        echo "diff $orig_content_output_file $ghost_content_output_file"
+
+        return 1
+    fi
 }
 
 build_binary() {
-  echo "Building"
-  rm -f $default_ghost_binary
-  [ "$ghost_binary" == "" ] && ghost_binary="$default_ghost_binary"
-  if [ -f "$ghost_binary" ] ; then
-    echo "Using binary: $ghost_binary"
-    return 0
-  fi
-
-  go build -o $ghost_binary go/cmd/gh-ost/main.go
-
-  if [ $? -ne 0 ] ; then
-    echo "Build failure"
-    exit 1
-  fi
+    echo "Building"
+    rm -f $default_ghost_binary
+    [ "$ghost_binary" == "" ] && ghost_binary="$default_ghost_binary"
+    if [ -f "$ghost_binary" ]; then
+        echo "Using binary: $ghost_binary"
+        return 0
+    fi
+
+    go build -o $ghost_binary go/cmd/gh-ost/main.go
+
+    if [ $? -ne 0 ]; then
+        echo "Build failure"
+        exit 1
+    fi
 }
 
 test_all() {
-  build_binary
-  test_dirs=$(find "$tests_path" -mindepth 1 -maxdepth 1 ! -path . -type d | grep "$test_pattern" | sort)
-  while read -r test_dir; do
-    test_name=$(basename "$test_dir")
-    if ! test_single "$test_name" ; then
-      create_statement=$(gh-ost-test-mysql-replica test -t -e "show create table _gh_ost_test_gho \G")
-      echo "$create_statement" >> $test_logfile
-      echo "+ FAIL"
-      return 1
-    else
-      echo
-      echo "+ pass"
-    fi
-    mysql_version="$(gh-ost-test-mysql-replica -e  "select @@version")"
-    replica_terminology="slave"
-    if [[ $mysql_version =~ "8.4" ]]; then
-      replica_terminology="replica"
-    fi
-    gh-ost-test-mysql-replica -e "start $replica_terminology"
-  done <<< "$test_dirs"
+    build_binary
+    test_dirs=$(find "$tests_path" -mindepth 1 -maxdepth 1 ! -path . -type d | grep "$test_pattern" | sort)
+    while read -r test_dir; do
+        test_name=$(basename "$test_dir")
+        if ! test_single "$test_name"; then
+            create_statement=$(gh-ost-test-mysql-replica test -t -e "show create table ${ghost_table_name} \G")
+            echo "$create_statement" >>$test_logfile
+            echo "+ FAIL"
+            return 1
+        else
+            echo
+            echo "+ pass"
+        fi
+        mysql_version="$(gh-ost-test-mysql-replica -e "select @@version")"
+        replica_terminology="slave"
+        if [[ $mysql_version =~ "8.4" ]]; then
+            replica_terminology="replica"
+        fi
+        gh-ost-test-mysql-replica -e "start $replica_terminology"
+    done <<<"$test_dirs"
 }
 
 verify_master_and_replica

From 601d1f9ec5de6f11569b7731c1107bdbe61a9f08 Mon Sep 17 00:00:00 2001
From: meiji163 <meiji163@github.com>
Date: Fri, 10 Oct 2025 09:08:36 -0700
Subject: [PATCH 3/3] add toxiproxy option for localtests (#1591)

---
 doc/local-tests.md                 |  8 ++++
 go/cmd/gh-ost/main.go              |  1 +
 localtests/test.sh                 | 30 +++++++++++----
 script/docker-gh-ost-replica-tests | 61 +++++++++++++++++++++++++++---
 4 files changed, 86 insertions(+), 14 deletions(-)

diff --git a/doc/local-tests.md b/doc/local-tests.md
index 411418f3b..fb13a79ed 100644
--- a/doc/local-tests.md
+++ b/doc/local-tests.md
@@ -34,3 +34,11 @@ TEST_MYSQL_IMAGE="mysql-server:8.0.16" ./script/docker-gh-ost-replica-tests up
 # cleanup containers
 ./script/docker-gh-ost-replica-tests down
 ```
+
+Pass the `-t` flag to run the tests with a toxiproxy between gh-ost and the MySQL replica. This simulates network conditions where MySQL connections are closed unexpectedly.
+
+```shell
+# run tests with toxiproxy
+./script/docker-gh-ost-replica-tests up -t
+./script/docker-gh-ost-replica-tests run -t
+```
diff --git a/go/cmd/gh-ost/main.go b/go/cmd/gh-ost/main.go
index 027bfc86e..feabd8658 100644
--- a/go/cmd/gh-ost/main.go
+++ b/go/cmd/gh-ost/main.go
@@ -143,6 +143,7 @@ func main() {
 	flag.BoolVar(&migrationContext.IncludeTriggers, "include-triggers", false, "When true, the triggers (if exist) will be created on the new table")
 	flag.StringVar(&migrationContext.TriggerSuffix, "trigger-suffix", "", "Add a suffix to the trigger name (i.e '_v2'). Requires '--include-triggers'")
 	flag.BoolVar(&migrationContext.RemoveTriggerSuffix, "remove-trigger-suffix-if-exists", false, "Remove given suffix from name of trigger. Requires '--include-triggers' and '--trigger-suffix'")
+	flag.BoolVar(&migrationContext.SkipPortValidation, "skip-port-validation", false, "Skip port validation for MySQL connections")
 
 	maxLoad := flag.String("max-load", "", "Comma delimited status-name=threshold. e.g: 'Threads_running=100,Threads_connected=500'. When status exceeds threshold, app throttles writes")
 	criticalLoad := flag.String("critical-load", "", "Comma delimited status-name=threshold, same format as --max-load. When status exceeds threshold, app panics and quits")
diff --git a/localtests/test.sh b/localtests/test.sh
index f595c1b25..5cdc65f7e 100755
--- a/localtests/test.sh
+++ b/localtests/test.sh
@@ -12,6 +12,7 @@ test_logfile=/tmp/gh-ost-test.log
 default_ghost_binary=/tmp/gh-ost-test
 ghost_binary=""
 docker=false
+toxiproxy=false
 storage_engine=innodb
 exec_command_file=/tmp/gh-ost-test.bash
 ghost_structure_output_file=/tmp/gh-ost-test.ghost.structure.sql
@@ -29,7 +30,7 @@ original_sql_mode=
 sysbench_pid=
 
 OPTIND=1
-while getopts "b:s:d" OPTION; do
+while getopts "b:s:dt" OPTION; do
     case $OPTION in
     b)
         ghost_binary="$OPTARG"
@@ -37,6 +38,9 @@ while getopts "b:s:d" OPTION; do
     s)
         storage_engine="$OPTARG"
         ;;
+    t)
+        toxiproxy=true
+        ;;
     d)
         docker=true
         ;;
@@ -75,6 +79,20 @@ verify_master_and_replica() {
     read replica_host replica_port <<<$(gh-ost-test-mysql-replica -e "select @@hostname, @@port" -ss)
     [ "$replica_host" == "$(hostname)" ] && replica_host="127.0.0.1"
     echo "# replica verified at $replica_host:$replica_port"
+
+    if [ "$docker" = true ]; then
+        master_host="0.0.0.0"
+        master_port="3307"
+        echo "# using docker master at $master_host:$master_port"
+        replica_host="0.0.0.0"
+        if [ "$toxiproxy" = true ]; then
+            replica_port="23308"
+            echo "# using toxiproxy replica at $replica_host:$replica_port"
+        else
+            replica_port="3308"
+            echo "# using docker replica at $replica_host:$replica_port"
+        fi
+    fi
 }
 
 exec_cmd() {
@@ -154,13 +172,6 @@ test_single() {
     local test_name
     test_name="$1"
 
-    if [ "$docker" = true ]; then
-        master_host="0.0.0.0"
-        master_port="3307"
-        replica_host="0.0.0.0"
-        replica_port="3308"
-    fi
-
     if [ -f $tests_path/$test_name/ignore_versions ]; then
         ignore_versions=$(cat $tests_path/$test_name/ignore_versions)
         mysql_version=$(gh-ost-test-mysql-master -s -s -e "select @@version")
@@ -199,6 +210,9 @@ test_single() {
     if [ -f $tests_path/$test_name/extra_args ]; then
         extra_args=$(cat $tests_path/$test_name/extra_args)
     fi
+    if [ "$toxiproxy" = true ]; then
+        extra_args+=" --skip-port-validation"
+    fi
     orig_columns="*"
     ghost_columns="*"
     order_by=""
diff --git a/script/docker-gh-ost-replica-tests b/script/docker-gh-ost-replica-tests
index 4469694d3..4c068e6d6 100755
--- a/script/docker-gh-ost-replica-tests
+++ b/script/docker-gh-ost-replica-tests
@@ -5,11 +5,15 @@
 # Set the environment var TEST_MYSQL_IMAGE to change the docker image.
 #
 # Usage:
-#   docker-gh-ost-replica-tests up      start the containers
-#   docker-gh-ost-replica-tests down    remove the containers
-#   docker-gh-ost-replica-tests run     run replica tests on the containers
+#   docker-gh-ost-replica-tests up [-t]     start the containers
+#   docker-gh-ost-replica-tests down        remove the containers
+#   docker-gh-ost-replica-tests run [-t]    run replica tests on the containers
+#
+# Flags:
+#   -t      use a toxiproxy for replica connection to simulate dropped connections
 
 set -e
+toxiproxy=false
 
 GH_OST_ROOT=$(git rev-parse --show-toplevel)
 if [[ ":$PATH:" != *":$GH_OST_ROOT:"* ]]; then
@@ -47,6 +51,22 @@ mysql-replica() {
     fi
 }
 
+create_toxiproxy() {
+    curl --fail -X POST http://localhost:8474/proxies \
+        -H "Content-Type: application/json" \
+        -d '{"name": "mysql_proxy",
+            "listen": "0.0.0.0:23308",
+            "upstream": "host.docker.internal:3308"}'
+    echo
+
+    curl --fail -X POST http://localhost:8474/proxies/mysql_proxy/toxics \
+        -H "Content-Type: application/json" \
+        -d '{"name": "limit_data_downstream",
+            "type": "limit_data",
+            "attributes": {"bytes": 300000}}'
+    echo
+}
+
 setup() {
     [ -z "$TEST_MYSQL_IMAGE" ] && TEST_MYSQL_IMAGE="mysql:8.0.41"
 
@@ -59,6 +79,22 @@ setup() {
         MYSQL_SHA2_RSA_KEYS_FLAG="--caching-sha2-password-auto-generate-rsa-keys=ON"
     fi
     (TEST_MYSQL_IMAGE="$TEST_MYSQL_IMAGE" MYSQL_SHA2_RSA_KEYS_FLAG="$MYSQL_SHA2_RSA_KEYS_FLAG" envsubst <"$compose_file") >"$compose_file.tmp"
+
+    if [ "$toxiproxy" = true ]; then
+        echo "Starting toxiproxy container..."
+        cat <<EOF >>"$compose_file.tmp"
+  mysql-toxiproxy:
+    image: "ghcr.io/shopify/toxiproxy:latest"
+    container_name: mysql-toxiproxy
+    ports:
+      - '8474:8474'
+      - '23308:23308'
+    expose:
+      - '23308'
+      - '8474'
+EOF
+    fi
+
     docker compose -f "$compose_file.tmp" up -d --wait
 
     echo "Waiting for MySQL..."
@@ -80,23 +116,36 @@ setup() {
         mysql-replica -e "start slave;"
     fi
     echo "OK"
+
+    if [ "$toxiproxy" = true ]; then
+        echo "Creating toxiproxy..."
+        create_toxiproxy
+        echo "OK"
+    fi
 }
 
 teardown() {
     echo "Stopping containers..."
     docker stop mysql-replica
     docker stop mysql-primary
+    docker stop mysql-toxiproxy 2>/dev/null || true
     echo "Removing containers..."
     docker rm mysql-replica
     docker rm mysql-primary
+    docker rm mysql-toxiproxy 2>/dev/null || true
 }
 
 main() {
-    if [[ "$1" == "up" ]]; then
+    local cmd="$1"
+    local tflag=
+    if [[ "$2" == "-t" ]]; then
+        toxiproxy=true
+    fi
+    if [[ "$cmd" == "up" ]]; then
         setup
-    elif [[ "$1" == "down" ]]; then
+    elif [[ "$cmd" == "down" ]]; then
         teardown
-    elif [[ "$1" == "run" ]]; then
+    elif [[ "$cmd" == "run" ]]; then
         shift 1
         "$GH_OST_ROOT/localtests/test.sh" -d "$@"
     fi