Skip to content

Commit 85eed06

Browse files
chriskirklandchriskirkland
committed
1. fail-resume-on-row-copy
1 parent 7bd8059 commit 85eed06

8 files changed

Lines changed: 196 additions & 47 deletions

File tree

‎go/base/context.go‎

Lines changed: 23 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1154,11 +1154,33 @@ func SendWithContext[T any](ctx context.Context, ch chan<- T, val T) error {
11541154
}
11551155
}
11561156

1157-
func (mctx *MigrationContext) NewFailPoint(name string) {
1157+
type failPointOpts struct {
1158+
wait time.Duration
1159+
}
1160+
1161+
type FailPointOpt func(*failPointOpts)
1162+
1163+
// WithFailPointWait sets the time for a fail point to wait before exiting.
1164+
func WithFailPointWait(wait time.Duration) FailPointOpt {
1165+
return func(opts *failPointOpts) {
1166+
opts.wait = wait
1167+
}
1168+
}
1169+
1170+
func (mctx *MigrationContext) NewFailPoint(name string, opts ...FailPointOpt) {
11581171
if mctx.UnsafeFailPointsEnabled {
11591172
mctx.Log.Debugf("[TEST] Injecting fail point: '%s'", name)
11601173

1174+
var fpo failPointOpts
1175+
for _, opt := range opts {
1176+
opt(&fpo)
1177+
}
1178+
11611179
failpoint.Inject(name, func(_ failpoint.Value) {
1180+
if fpo.wait > 0 {
1181+
<-time.After(fpo.wait)
1182+
}
1183+
11621184
panic(fmt.Sprintf("[TEST] Encountered fail point: '%s'", name))
11631185
})
11641186
}

‎go/cmd/gh-ost/main.go‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -348,7 +348,9 @@ func main() {
348348
if *storageEngine == "rocksdb" {
349349
migrationContext.Log.Warning("RocksDB storage engine support is experimental")
350350
}
351-
if migrationContext.CheckpointIntervalSeconds < 10 {
351+
// ignore low checkpoint intervals in unsafe mode as frequent checkpoints are required to reliably
352+
// reduce test duration
353+
if migrationContext.CheckpointIntervalSeconds < 10 && !migrationContext.UnsafeFailPointsEnabled {
352354
migrationContext.Log.Fatalf("--checkpoint-seconds should be >=10")
353355
}
354356
if migrationContext.CountTableRows && migrationContext.PanicOnWarnings {

‎go/logic/applier.go‎

Lines changed: 0 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -1451,17 +1451,6 @@ func (apl *Applier) ApplyIterationInsertQuery() (chunkSize int64, rowsAffected i
14511451
// ApplyIterationMoveTableCopyQueries issues a SELECT query on the original table and an INSERT query on the target table,
14521452
// copying a chunk of rows. It is used when `--move-tables` is specified, instead of ApplyIterationInsertQuery.
14531453
func (apl *Applier) ApplyIterationMoveTableCopyQueries(sourceDB *gosql.DB) (chunkSize int64, rowsAffected int64, duration time.Duration, err error) {
1454-
apl.migrationContext.NewFailPoint("panic-on-row-copy")
1455-
//if apl.migrationContext.UnsafeFailPointsEnabled {
1456-
// name := "panic-on-row-copy"
1457-
// apl.migrationContext.Log.Debugf("Injecting fail point: %s", name)
1458-
1459-
// failpoint.Inject(name, func(_ failpoint.Value) {
1460-
// apl.migrationContext.Log.Debugf("Encountered fail point: %s", name)
1461-
// panic(fmt.Sprintf("encountered fail point: '%s'", name))
1462-
// })
1463-
//}
1464-
14651454
startTime := time.Now()
14661455
chunkSize = atomic.LoadInt64(&apl.migrationContext.ChunkSize)
14671456
if sourceDB == nil {

‎go/logic/migrator.go‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2471,6 +2471,8 @@ func (mgtr *Migrator) iterateChunks() error {
24712471
}
24722472
return terminateRowIteration(err)
24732473
}
2474+
2475+
mgtr.migrationContext.NewFailPoint("panic-after-row-copy", base.WithFailPointWait(2*time.Second))
24742476
}
24752477
}
24762478

‎localtests/move-tables-test.sh‎

Lines changed: 40 additions & 34 deletions
Original file line numberDiff line numberDiff line change
@@ -183,7 +183,7 @@ build_ghost_command() {
183183
move_tables_arg=$(IFS=, ; echo "${tables_to_migrate[*]}")
184184

185185
# NOTE(chriskirkland): fully qualified package name + failpoint name
186-
cmd="GOTRACEBACK=crash GO_FAILPOINTS=\"github.com/github/gh-ost/go/base/panic-on-row-copy=return(true)\" $ghost_binary \
186+
cmd="GOTRACEBACK=crash $ghost_binary \
187187
--move-tables=$move_tables_arg \
188188
--user=root \
189189
--password=opensesame \
@@ -207,6 +207,10 @@ build_ghost_command() {
207207
--postpone-cut-over-flag-file=$postpone_cutover_flag_file \
208208
--unsafe-fail-points-enabled \
209209
--execute ${extra_args[@]}"
210+
211+
if [ -n "$GO_FAILPOINTS" ]; then
212+
cmd="GO_FAILPOINTS=\"$GO_FAILPOINTS\" $cmd --unsafe-fail-points-enabled"
213+
fi
210214
}
211215

212216
print_log_excerpt() {
@@ -357,45 +361,47 @@ test_single() {
357361
wait $test_pid 2>/dev/null
358362
execution_result=$?
359363
return $execution_result
360-
fi
361364

362-
# kick off the on_test script for the test. this enables arbitrary custom logic
363-
# concurrent with the gh-ost process. this enables additional scenarios like
364-
# streaming of writes prior to the write cutover.
365-
#
366-
# IMPORTANT: The on-test script is executed in the background and will be killed as soon
367-
# as the gh-ost process terminates.
368-
if [ -f $tests_path/$test_name/on_test.sh ]; then
369-
$tests_path/$test_name/on_test.sh &> /dev/null &
370-
on_test_pid=$!
371-
fi
365+
else
372366

373-
# queue up removal of the postpone cutover flag, otherwise gh-ost hangs on the cutover
374-
(
375-
sleep 1;
376-
echo "⏩ Sending unpostpone cutover"
377-
rm $postpone_cutover_flag_file &> /dev/null;
378-
) &
367+
# kick off the on_test script for the test. this enables arbitrary custom logic
368+
# concurrent with the gh-ost process. this enables additional scenarios like
369+
# streaming of writes prior to the write cutover.
370+
#
371+
# IMPORTANT: The on-test script is executed in the background and will be killed as soon
372+
# as the gh-ost process terminates.
373+
if [ -f $tests_path/$test_name/on_test.sh ]; then
374+
$tests_path/$test_name/on_test.sh &> /dev/null &
375+
on_test_pid=$!
376+
fi
379377

380-
# Build and execute gh-ost command
381-
build_ghost_command
382-
echo_dot
383-
echo $cmd >$exec_command_file
384-
echo_dot
385-
timeout $test_timeout bash $exec_command_file >$test_logfile 2>&1
378+
# queue up removal of the postpone cutover flag, otherwise gh-ost hangs on the cutover
379+
(
380+
sleep 1;
381+
echo "⏩ Sending unpostpone cutover"
382+
rm $postpone_cutover_flag_file &> /dev/null;
383+
) &
386384

387-
execution_result=$?
385+
# Build and execute gh-ost command
386+
build_ghost_command
387+
echo_dot
388+
echo $cmd >$exec_command_file
389+
echo_dot
390+
timeout $test_timeout bash $exec_command_file >$test_logfile 2>&1
388391

389-
if [ -n "$on_test_pid" ]; then
390-
kill -KILL $on_test_pid &>/dev/null
391-
fi
392+
execution_result=$?
392393

393-
# Check for timeout (exit code 124)
394-
if [ $execution_result -eq 124 ]; then
395-
echo
396-
echo "ERROR $test_name execution timed out"
397-
print_log_excerpt
398-
return 1
394+
if [ -n "$on_test_pid" ]; then
395+
kill -KILL $on_test_pid &>/dev/null
396+
fi
397+
398+
# Check for timeout (exit code 124)
399+
if [ $execution_result -eq 124 ]; then
400+
echo
401+
echo "ERROR $test_name execution timed out"
402+
print_log_excerpt
403+
return 1
404+
fi
399405
fi
400406

401407
if [ -f $tests_path/$test_name/sql_mode ]; then

‎localtests/move-tables/fail-resume-on-row-copy/create.sql‎

Lines changed: 34 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,34 @@
1+
drop table if exists gh_ost_test;
2+
create table gh_ost_test (
3+
id bigint(20) NOT NULL AUTO_INCREMENT,
4+
column1 int(11) NOT NULL,
5+
column2 smallint(5) unsigned NOT NULL,
6+
column3 mediumint(8) unsigned NOT NULL,
7+
column4 tinyint(3) unsigned NOT NULL,
8+
column5 int(11) NOT NULL,
9+
column6 int(11) NOT NULL,
10+
PRIMARY KEY (id),
11+
KEY c12_ix (column1, column2)
12+
) auto_increment=1;
13+
14+
insert into gh_ost_test values
15+
(NULL, 1001, 100, 500000, 10, 1700000001, 1700000002),
16+
(NULL, 1002, 200, 600000, 20, 1700000003, 1700000004),
17+
(NULL, 1003, 300, 700000, 30, 1700000005, 1700000006),
18+
(NULL, 1004, 400, 800000, 40, 1700000007, 1700000008),
19+
(NULL, 1005, 500, 900000, 50, 1700000009, 1700000010),
20+
(NULL, 1006, 600, 1000000, 60, 1700000011, 1700000012),
21+
(NULL, 1007, 700, 1100000, 70, 1700000013, 1700000014),
22+
(NULL, 1008, 800, 1200000, 80, 1700000015, 1700000016),
23+
(NULL, 1009, 900, 1300000, 90, 1700000017, 1700000018),
24+
(NULL, 1010, 1000, 1400000, 100, 1700000019, 1700000020),
25+
(NULL, 1011, 1100, 1500000, 110, 1700000021, 1700000022),
26+
(NULL, 1012, 1200, 1600000, 120, 1700000023, 1700000024),
27+
(NULL, 1013, 1300, 1700000, 130, 1700000025, 1700000026),
28+
(NULL, 1014, 1400, 1800000, 140, 1700000027, 1700000028),
29+
(NULL, 1015, 1500, 1900000, 150, 1700000029, 1700000030),
30+
(NULL, 1016, 1600, 2000000, 160, 1700000031, 1700000032),
31+
(NULL, 1017, 1700, 2100000, 170, 1700000033, 1700000034),
32+
(NULL, 1018, 1800, 2200000, 180, 1700000035, 1700000036),
33+
(NULL, 1019, 1900, 2300000, 190, 1700000037, 1700000038),
34+
(NULL, 1020, 2000, 2400000, 200, 1700000039, 1700000040);

‎localtests/move-tables/fail-resume-on-row-copy/tables.txt‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
gh_ost_test

‎localtests/move-tables/fail-resume-on-row-copy/test.sh‎

Lines changed: 93 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,93 @@
1+
2+
#!/bin/bash
3+
# Custom test:
4+
# - panic during row copy stage, prior to cutover
5+
# - resume and complete the migration
6+
7+
database=test
8+
table_name=gh_ost_test
9+
10+
# Build gh-ost command from scratch using framework function (required to inject failpoints)
11+
rm $ghost_binary
12+
build_binary
13+
14+
######################################################################################################
15+
### Run #1: Should panic after first row copy and migration will not complete
16+
######################################################################################################
17+
18+
echo "⚙️ Starting migration with failpoint (run #1)..."
19+
20+
# Build the gh-ost command using the framework function
21+
GO_FAILPOINTS="github.com/github/gh-ost/go/base/panic-after-row-copy=return(true)" build_ghost_command
22+
cmd="$cmd --checkpoint-seconds=1"
23+
24+
# Run the gh-ost command, expecting panic on the failpoint the first time
25+
echo_dot
26+
echo > $test_logfile
27+
bash -c "$cmd" >>$test_logfile 2>&1
28+
ghost_result=$?
29+
30+
if [ $ghost_result -eq 0 ]; then
31+
echo "ERROR: gh-ost should have failed but did not."
32+
return 1
33+
fi
34+
35+
echo -e "\n\n\n\n\n"
36+
37+
######################################################################################################
38+
### Intermediate validation
39+
######################################################################################################
40+
41+
echo "⚙️ Validating checkpointed state on unexpected exit..."
42+
43+
# checkpoint file exists on target and is non-empty
44+
results=$(mysql-exec target primary $database -sNe "SELECT 1 FROM _${table_name}_ghk LIMIT 1;")
45+
if [ -z "$results" ]; then
46+
echo "ERROR: Checkpoint file is empty or does not exist."
47+
return 1
48+
fi
49+
50+
# original table still exists on source
51+
results=$(mysql-exec source replica $database -sNe "SELECT 1 FROM ${table_name} LIMIT 1;")
52+
if [ -z "$results" ]; then
53+
echo "ERROR: Table '${table_name}' does not exist on the source cluster."
54+
return 1
55+
fi
56+
57+
# original table exists on the target
58+
results=$(mysql-exec target replica $database -sNe "SELECT 1 FROM ${table_name} LIMIT 1;")
59+
if [ -z "$results" ]; then
60+
echo "ERROR: Table '${table_name}' does not exist on the target cluster."
61+
return 1
62+
fi
63+
64+
echo "✅ Validating checkpointed state on unexpected exit..."
65+
66+
echo -e "\n\n\n\n\n"
67+
68+
######################################################################################################
69+
### Run #2: Resume and complete the migration
70+
######################################################################################################
71+
72+
echo "⚙️ Resuming migration (run #2)..."
73+
74+
# queue up removal of the postpone cutover flag, otherwise gh-ost hangs on the cutover
75+
(
76+
sleep 1;
77+
echo "⏩ Sending unpostpone cutover"
78+
rm $postpone_cutover_flag_file &> /dev/null;
79+
) &
80+
81+
# resume migration
82+
build_ghost_command
83+
cmd="$cmd --resume"
84+
85+
bash -c "$cmd" >>$test_logfile 2>&1
86+
ghost_result=$?
87+
88+
if [ $ghost_result -neq 0 ]; then
89+
echo "ERROR: gh-ost should have succeeded but did not. ($ghost_result)"
90+
return 1
91+
fi
92+
93+
echo -e "\n\n\n\n\n"

0 commit comments

Comments
 (0)