Skip to content

Commit 15b8b9a

Browse files
chriskirklandchriskirkland
committed
3. resume-panic-before-on-success-hook
1 parent f1dd6fd commit 15b8b9a

12 files changed

Lines changed: 175 additions & 9 deletions

File tree

‎go/logic/migrator.go‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1405,6 +1405,8 @@ func (mgtr *Migrator) moveTablesCutOver() (err error) {
14051405
atomic.StoreInt64(&mgtr.migrationContext.CutOverCompleteFlag, 1)
14061406
mgtr.migrationContext.Log.Debugf("T4: CutOverCompleteFlag set")
14071407

1408+
mgtr.migrationContext.NewFailPoint("panic-before-on-success-hook", base.WithFailPointWait(2*time.Second))
1409+
14081410
// ----- T5: on-success hook -----
14091411
// Hook unlocks user_rw@target via db-user-management and flips the
14101412
// write_cutover? feature flag. Standard env vars only — GH_OST_DRAIN_GTID +

‎localtests/move-tables-test.sh‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -365,7 +365,7 @@ test_single() {
365365

366366
else
367367

368-
# kick off the on_test script for the test. this enables arbitrary custom logic
368+
# kick off the on_test script for the test. this enables arbitrary custom logic
369369
# concurrent with the gh-ost process. this enables additional scenarios like
370370
# streaming of writes prior to the write cutover.
371371
#
@@ -378,7 +378,7 @@ test_single() {
378378

379379
# queue up removal of the postpone cutover flag, otherwise gh-ost hangs on the cutover
380380
(
381-
sleep 1;
381+
sleep 1;
382382
echo "⏩ Sending unpostpone cutover"
383383
rm $postpone_cutover_flag_file &> /dev/null;
384384
) &

localtests/move-tables/fail-resume-before-drain-complete/create.sql renamed to localtests/move-tables/resume-panic-before-drain-complete/create.sql

File renamed without changes.

localtests/move-tables/fail-resume-before-drain-complete/tables.txt renamed to localtests/move-tables/resume-panic-before-drain-complete/tables.txt

File renamed without changes.

localtests/move-tables/fail-resume-before-drain-complete/test.sh renamed to localtests/move-tables/resume-panic-before-drain-complete/test.sh

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11

22
#!/bin/bash
3-
# Custom test:
3+
# Custom test:
44
# - panic after RENAME (T1) and prior to drain completion (T3), prior to cutover completion
55
# - validate RENAME and source writes are not possible
66
# - resume and complete the migration
@@ -23,7 +23,7 @@ GO_FAILPOINTS="github.com/github/gh-ost/go/base/panic-before-drain-completion=re
2323

2424
# queue up removal of the postpone cutover flag, otherwise gh-ost hangs on the cutover
2525
(
26-
sleep 2;
26+
sleep 2;
2727
echo "⏩ Sending unpostpone cutover"
2828
rm $postpone_cutover_flag_file &> /dev/null;
2929
) &
@@ -60,7 +60,7 @@ if [ $? -gt 0 ]; then
6060
return 1
6161
fi
6262

63-
# Table not writeable on source
63+
# Table not writeable on source
6464
mysql-exec target source $database -sNe "INSERT INTO ${table_name} VALUES (NULL, 1021, 2001, 2400001, 201, 1700000041, 1700000041);"
6565
if [ $? -eq 0 ]; then
6666
echo "ERROR: Table '${table_name}' was writeable on source but should not be!."
@@ -90,7 +90,7 @@ cmd="$cmd --resume"
9090

9191
# queue up removal of the postpone cutover flag, otherwise gh-ost hangs on the cutover
9292
(
93-
sleep 2;
93+
sleep 2;
9494
echo "⏩ Sending unpostpone cutover"
9595
rm $postpone_cutover_flag_file &> /dev/null;
9696
) &

localtests/move-tables/fail-resume-on-row-copy/create.sql renamed to localtests/move-tables/resume-panic-before-on-success-hook/create.sql

File renamed without changes.

localtests/move-tables/fail-resume-on-row-copy/tables.txt renamed to localtests/move-tables/resume-panic-before-on-success-hook/tables.txt

File renamed without changes.

‎localtests/move-tables/resume-panic-before-on-success-hook/test.sh‎

Lines changed: 129 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,129 @@
1+
2+
#!/bin/bash
3+
# Custom test:
4+
# - panic after drain (T4) and prior to on-success (T5), prior to cutover completion
5+
# - validate RENAME and source writes are not possible
6+
# - validate contents of source and target are the same
7+
# - resume and complete the migration
8+
9+
database=test
10+
table_name=gh_ost_test
11+
12+
# Build gh-ost command from scratch using framework function (required to inject failpoints)
13+
rm $ghost_binary
14+
build_binary
15+
16+
######################################################################################################
17+
### Run #1: Should panic after drain (T4) and before on-success (T5)
18+
######################################################################################################
19+
20+
echo "⚙️ Starting migration with failpoint (run #1)..."
21+
22+
# Build the gh-ost command using the framework function
23+
GO_FAILPOINTS="github.com/github/gh-ost/go/base/panic-before-on-success-hook=return(true)" build_ghost_command
24+
25+
# queue up removal of the postpone cutover flag, otherwise gh-ost hangs on the cutover
26+
(
27+
sleep 2;
28+
echo "⏩ Sending unpostpone cutover"
29+
rm $postpone_cutover_flag_file &> /dev/null;
30+
) &
31+
32+
# drive some concurrent writes to the table to exercise queue drain (T3/T4)
33+
(
34+
DATABASE=test script/move-tables/insert-source-primary-loop 100 0.1 10 &>/dev/null &
35+
writes_pid=$!
36+
sleep 3
37+
kill $writes_pid
38+
) &
39+
40+
# Run the gh-ost command, expecting panic on the failpoint the first time
41+
echo_dot
42+
echo > $test_logfile
43+
bash -c "$cmd" >>$test_logfile 2>&1
44+
ghost_result=$?
45+
46+
if [ $ghost_result -eq 0 ]; then
47+
echo "ERROR: gh-ost should have failed but did not."
48+
return 1
49+
fi
50+
51+
echo -e "\n\n\n\n\n"
52+
53+
######################################################################################################
54+
### Intermediate validation
55+
######################################################################################################
56+
57+
echo "⚙️ Validating checkpointed state on unexpected exit..."
58+
59+
# Table was renamed on source
60+
mysql-exec source primary $database -sNe "SELECT 1 FROM ${table_name} LIMIT 1;"
61+
if [ $? -eq 0 ]; then
62+
echo "ERROR: Table '${table_name}' exists on source but show have been renamed."
63+
return 1
64+
fi
65+
66+
mysql-exec source primary $database -sNe "SELECT 1 FROM _${table_name}_del LIMIT 1;"
67+
if [ $? -gt 0 ]; then
68+
echo "ERROR: Renamed table '_${table_name}_del' does not exist on source."
69+
return 1
70+
fi
71+
72+
# Table not writeable on source
73+
mysql-exec target source $database -sNe "INSERT INTO ${table_name} VALUES (NULL, 1021, 2001, 2400001, 201, 1700000041, 1700000041);"
74+
if [ $? -eq 0 ]; then
75+
echo "ERROR: Table '${table_name}' was writeable on source but should not be!."
76+
return 1
77+
fi
78+
79+
# Table still exists on target
80+
mysql-exec target primary $database -sNe "SELECT 1 FROM ${table_name} LIMIT 1;"
81+
if [ $? -gt 0 ]; then
82+
echo "ERROR: Table '${table_name}' does not exist on target."
83+
return 1
84+
fi
85+
86+
# contents of table on source and target are the same
87+
source_contents_file=/tmp/gh-ost-test.resume-panic-before-on-success-hook-source_contents.txt
88+
target_contents_file=/tmp/gh-ost-test.resume-panic-before-on-success-hook-target_contents.txt
89+
mysql-exec source primary $database -sNe "SELECT * FROM _${table_name}_del;" > $source_contents_file
90+
mysql-exec target primary $database -sNe "SELECT * FROM ${table_name};" > $target_contents_file
91+
92+
if ! diff $source_contents_file $target_contents_file; then
93+
echo "ERROR: Contents of table '${table_name}' are not the same on source and target."
94+
echo "---- DIFF -----"
95+
diff --side-by-side $source_contents_file $target_contents_file
96+
echo "---------------"
97+
return 1
98+
fi
99+
100+
echo "✅ Validated checkpointed state on unexpected exit..."
101+
102+
echo -e "\n\n\n\n\n"
103+
104+
######################################################################################################
105+
### Run #2: Resume and complete the migration
106+
######################################################################################################
107+
108+
echo "⚙️ Resuming migration (run #2)..."
109+
110+
# resume migration
111+
build_ghost_command
112+
cmd="$cmd --resume"
113+
114+
# queue up removal of the postpone cutover flag, otherwise gh-ost hangs on the cutover
115+
(
116+
sleep 2;
117+
echo "⏩ Sending unpostpone cutover"
118+
rm $postpone_cutover_flag_file &> /dev/null;
119+
) &
120+
121+
bash -c "$cmd" >>$test_logfile 2>&1
122+
ghost_result=$?
123+
124+
if [ $ghost_result -neq 0 ]; then
125+
echo "ERROR: gh-ost should have succeeded but did not. ($ghost_result)"
126+
return 1
127+
fi
128+
129+
echo -e "\n\n\n\n\n"

‎localtests/move-tables/resume-panic-on-row-copy/create.sql‎

Lines changed: 34 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,34 @@
1+
drop table if exists gh_ost_test;
2+
create table gh_ost_test (
3+
id bigint(20) NOT NULL AUTO_INCREMENT,
4+
column1 int(11) NOT NULL,
5+
column2 smallint(5) unsigned NOT NULL,
6+
column3 mediumint(8) unsigned NOT NULL,
7+
column4 tinyint(3) unsigned NOT NULL,
8+
column5 int(11) NOT NULL,
9+
column6 int(11) NOT NULL,
10+
PRIMARY KEY (id),
11+
KEY c12_ix (column1, column2)
12+
) auto_increment=1;
13+
14+
insert into gh_ost_test values
15+
(NULL, 1001, 100, 500000, 10, 1700000001, 1700000002),
16+
(NULL, 1002, 200, 600000, 20, 1700000003, 1700000004),
17+
(NULL, 1003, 300, 700000, 30, 1700000005, 1700000006),
18+
(NULL, 1004, 400, 800000, 40, 1700000007, 1700000008),
19+
(NULL, 1005, 500, 900000, 50, 1700000009, 1700000010),
20+
(NULL, 1006, 600, 1000000, 60, 1700000011, 1700000012),
21+
(NULL, 1007, 700, 1100000, 70, 1700000013, 1700000014),
22+
(NULL, 1008, 800, 1200000, 80, 1700000015, 1700000016),
23+
(NULL, 1009, 900, 1300000, 90, 1700000017, 1700000018),
24+
(NULL, 1010, 1000, 1400000, 100, 1700000019, 1700000020),
25+
(NULL, 1011, 1100, 1500000, 110, 1700000021, 1700000022),
26+
(NULL, 1012, 1200, 1600000, 120, 1700000023, 1700000024),
27+
(NULL, 1013, 1300, 1700000, 130, 1700000025, 1700000026),
28+
(NULL, 1014, 1400, 1800000, 140, 1700000027, 1700000028),
29+
(NULL, 1015, 1500, 1900000, 150, 1700000029, 1700000030),
30+
(NULL, 1016, 1600, 2000000, 160, 1700000031, 1700000032),
31+
(NULL, 1017, 1700, 2100000, 170, 1700000033, 1700000034),
32+
(NULL, 1018, 1800, 2200000, 180, 1700000035, 1700000036),
33+
(NULL, 1019, 1900, 2300000, 190, 1700000037, 1700000038),
34+
(NULL, 1020, 2000, 2400000, 200, 1700000039, 1700000040);

‎localtests/move-tables/resume-panic-on-row-copy/tables.txt‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
gh_ost_test

0 commit comments

Comments
 (0)