Skip to content

Commit f1dd6fd

Browse files
chriskirklandchriskirkland
committed
2. fail-resume-before-drain-complete
1 parent 85eed06 commit f1dd6fd

6 files changed

Lines changed: 156 additions & 12 deletions

File tree

‎go/logic/migrator.go‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1385,6 +1385,9 @@ func (mgtr *Migrator) moveTablesCutOver() (err error) {
13851385
}
13861386
}
13871387

1388+
mgtr.migrationContext.NewFailPoint("panic-before-drain-completion", base.WithFailPointWait(2*time.Second))
1389+
1390+
// ------ T3: draining applier to drain GTID -----------
13881391
if err := mgtr.drainMoveTablesCutOver(drainGTID); err != nil {
13891392
return err
13901393
}

‎localtests/move-tables-test.sh‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -206,6 +206,7 @@ build_ghost_command() {
206206
--checkpoint \
207207
--postpone-cut-over-flag-file=$postpone_cutover_flag_file \
208208
--unsafe-fail-points-enabled \
209+
--checkpoint-seconds=1 \
209210
--execute ${extra_args[@]}"
210211

211212
if [ -n "$GO_FAILPOINTS" ]; then

‎localtests/move-tables/fail-resume-before-drain-complete/create.sql‎

Lines changed: 34 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,34 @@
1+
drop table if exists gh_ost_test;
2+
create table gh_ost_test (
3+
id bigint(20) NOT NULL AUTO_INCREMENT,
4+
column1 int(11) NOT NULL,
5+
column2 smallint(5) unsigned NOT NULL,
6+
column3 mediumint(8) unsigned NOT NULL,
7+
column4 tinyint(3) unsigned NOT NULL,
8+
column5 int(11) NOT NULL,
9+
column6 int(11) NOT NULL,
10+
PRIMARY KEY (id),
11+
KEY c12_ix (column1, column2)
12+
) auto_increment=1;
13+
14+
insert into gh_ost_test values
15+
(NULL, 1001, 100, 500000, 10, 1700000001, 1700000002),
16+
(NULL, 1002, 200, 600000, 20, 1700000003, 1700000004),
17+
(NULL, 1003, 300, 700000, 30, 1700000005, 1700000006),
18+
(NULL, 1004, 400, 800000, 40, 1700000007, 1700000008),
19+
(NULL, 1005, 500, 900000, 50, 1700000009, 1700000010),
20+
(NULL, 1006, 600, 1000000, 60, 1700000011, 1700000012),
21+
(NULL, 1007, 700, 1100000, 70, 1700000013, 1700000014),
22+
(NULL, 1008, 800, 1200000, 80, 1700000015, 1700000016),
23+
(NULL, 1009, 900, 1300000, 90, 1700000017, 1700000018),
24+
(NULL, 1010, 1000, 1400000, 100, 1700000019, 1700000020),
25+
(NULL, 1011, 1100, 1500000, 110, 1700000021, 1700000022),
26+
(NULL, 1012, 1200, 1600000, 120, 1700000023, 1700000024),
27+
(NULL, 1013, 1300, 1700000, 130, 1700000025, 1700000026),
28+
(NULL, 1014, 1400, 1800000, 140, 1700000027, 1700000028),
29+
(NULL, 1015, 1500, 1900000, 150, 1700000029, 1700000030),
30+
(NULL, 1016, 1600, 2000000, 160, 1700000031, 1700000032),
31+
(NULL, 1017, 1700, 2100000, 170, 1700000033, 1700000034),
32+
(NULL, 1018, 1800, 2200000, 180, 1700000035, 1700000036),
33+
(NULL, 1019, 1900, 2300000, 190, 1700000037, 1700000038),
34+
(NULL, 1020, 2000, 2400000, 200, 1700000039, 1700000040);

‎localtests/move-tables/fail-resume-before-drain-complete/tables.txt‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
gh_ost_test

‎localtests/move-tables/fail-resume-before-drain-complete/test.sh‎

Lines changed: 106 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,106 @@
1+
2+
#!/bin/bash
3+
# Custom test:
4+
# - panic after RENAME (T1) and prior to drain completion (T3), prior to cutover completion
5+
# - validate RENAME and source writes are not possible
6+
# - resume and complete the migration
7+
8+
database=test
9+
table_name=gh_ost_test
10+
11+
# Build gh-ost command from scratch using framework function (required to inject failpoints)
12+
rm $ghost_binary
13+
build_binary
14+
15+
######################################################################################################
16+
### Run #1: Should panic after RENAME (T1) and before drain completion (T3)
17+
######################################################################################################
18+
19+
echo "⚙️ Starting migration with failpoint (run #1)..."
20+
21+
# Build the gh-ost command using the framework function
22+
GO_FAILPOINTS="github.com/github/gh-ost/go/base/panic-before-drain-completion=return(true)" build_ghost_command
23+
24+
# queue up removal of the postpone cutover flag, otherwise gh-ost hangs on the cutover
25+
(
26+
sleep 2;
27+
echo "⏩ Sending unpostpone cutover"
28+
rm $postpone_cutover_flag_file &> /dev/null;
29+
) &
30+
31+
# Run the gh-ost command, expecting panic on the failpoint the first time
32+
echo_dot
33+
echo > $test_logfile
34+
bash -c "$cmd" >>$test_logfile 2>&1
35+
ghost_result=$?
36+
37+
if [ $ghost_result -eq 0 ]; then
38+
echo "ERROR: gh-ost should have failed but did not."
39+
return 1
40+
fi
41+
42+
echo -e "\n\n\n\n\n"
43+
44+
######################################################################################################
45+
### Intermediate validation
46+
######################################################################################################
47+
48+
echo "⚙️ Validating checkpointed state on unexpected exit..."
49+
50+
# Table was renamed on source
51+
mysql-exec source primary $database -sNe "SELECT 1 FROM ${table_name} LIMIT 1;"
52+
if [ $? -eq 0 ]; then
53+
echo "ERROR: Table '${table_name}' exists on source but show have been renamed."
54+
return 1
55+
fi
56+
57+
mysql-exec source primary $database -sNe "SELECT 1 FROM _${table_name}_del LIMIT 1;"
58+
if [ $? -gt 0 ]; then
59+
echo "ERROR: Renamed table '_${table_name}_del' does not exist on source."
60+
return 1
61+
fi
62+
63+
# Table not writeable on source
64+
mysql-exec target source $database -sNe "INSERT INTO ${table_name} VALUES (NULL, 1021, 2001, 2400001, 201, 1700000041, 1700000041);"
65+
if [ $? -eq 0 ]; then
66+
echo "ERROR: Table '${table_name}' was writeable on source but should not be!."
67+
return 1
68+
fi
69+
70+
# Table still exists on target
71+
mysql-exec target primary $database -sNe "SELECT 1 FROM ${table_name} LIMIT 1;"
72+
if [ $? -gt 0 ]; then
73+
echo "ERROR: Table '${table_name}' does not exist on target."
74+
return 1
75+
fi
76+
77+
echo "✅ Validated checkpointed state on unexpected exit..."
78+
79+
echo -e "\n\n\n\n\n"
80+
81+
######################################################################################################
82+
### Run #2: Resume and complete the migration
83+
######################################################################################################
84+
85+
echo "⚙️ Resuming migration (run #2)..."
86+
87+
# resume migration
88+
build_ghost_command
89+
cmd="$cmd --resume"
90+
91+
# queue up removal of the postpone cutover flag, otherwise gh-ost hangs on the cutover
92+
(
93+
sleep 2;
94+
echo "⏩ Sending unpostpone cutover"
95+
rm $postpone_cutover_flag_file &> /dev/null;
96+
) &
97+
98+
bash -c "$cmd" >>$test_logfile 2>&1
99+
ghost_result=$?
100+
101+
if [ $ghost_result -neq 0 ]; then
102+
echo "ERROR: gh-ost should have succeeded but did not. ($ghost_result)"
103+
return 1
104+
fi
105+
106+
echo -e "\n\n\n\n\n"

‎localtests/move-tables/fail-resume-on-row-copy/test.sh‎

Lines changed: 11 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -19,7 +19,6 @@ echo "⚙️ Starting migration with failpoint (run #1)..."
1919

2020
# Build the gh-ost command using the framework function
2121
GO_FAILPOINTS="github.com/github/gh-ost/go/base/panic-after-row-copy=return(true)" build_ghost_command
22-
cmd="$cmd --checkpoint-seconds=1"
2322

2423
# Run the gh-ost command, expecting panic on the failpoint the first time
2524
echo_dot
@@ -41,22 +40,22 @@ echo -e "\n\n\n\n\n"
4140
echo "⚙️ Validating checkpointed state on unexpected exit..."
4241

4342
# checkpoint file exists on target and is non-empty
44-
results=$(mysql-exec target primary $database -sNe "SELECT 1 FROM _${table_name}_ghk LIMIT 1;")
45-
if [ -z "$results" ]; then
43+
mysql-exec target primary $database -sNe "SELECT 1 FROM _${table_name}_ghk LIMIT 1;"
44+
if [ $? -gt 0 ]; then
4645
echo "ERROR: Checkpoint file is empty or does not exist."
4746
return 1
4847
fi
4948

5049
# original table still exists on source
51-
results=$(mysql-exec source replica $database -sNe "SELECT 1 FROM ${table_name} LIMIT 1;")
52-
if [ -z "$results" ]; then
50+
mysql-exec source replica $database -sNe "SELECT 1 FROM ${table_name} LIMIT 1;"
51+
if [ $? -gt 0 ]; then
5352
echo "ERROR: Table '${table_name}' does not exist on the source cluster."
5453
return 1
5554
fi
5655

5756
# original table exists on the target
58-
results=$(mysql-exec target replica $database -sNe "SELECT 1 FROM ${table_name} LIMIT 1;")
59-
if [ -z "$results" ]; then
57+
mysql-exec target replica $database -sNe "SELECT 1 FROM ${table_name} LIMIT 1;"
58+
if [ $? -gt 0 ]; then
6059
echo "ERROR: Table '${table_name}' does not exist on the target cluster."
6160
return 1
6261
fi
@@ -71,17 +70,17 @@ echo -e "\n\n\n\n\n"
7170

7271
echo "⚙️ Resuming migration (run #2)..."
7372

73+
# resume migration
74+
build_ghost_command
75+
cmd="$cmd --resume"
76+
7477
# queue up removal of the postpone cutover flag, otherwise gh-ost hangs on the cutover
7578
(
76-
sleep 1;
79+
sleep 2;
7780
echo "⏩ Sending unpostpone cutover"
7881
rm $postpone_cutover_flag_file &> /dev/null;
7982
) &
8083

81-
# resume migration
82-
build_ghost_command
83-
cmd="$cmd --resume"
84-
8584
bash -c "$cmd" >>$test_logfile 2>&1
8685
ghost_result=$?
8786

0 commit comments

Comments
 (0)