Skip to content

Commit 764d2db

Browse files
committed
[FLINK-39198]Add test for restoring from early checkpoint with TIMESTAMP starting offset
Signed-off-by: danzhewuju <1668926294@qq.com>
1 parent 0c3dd4c commit 764d2db

2 files changed

Lines changed: 75 additions & 0 deletions

File tree

flink-cdc-connect/flink-cdc-source-connectors/flink-connector-mysql-cdc/src/main/java/org/apache/flink/cdc/connectors/mysql/debezium/reader/BinlogSplitReader.java

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -408,6 +408,9 @@ private Predicate<Event> createEventFilter() {
408408
+ "configuration. ");
409409
}
410410
if (startingOffset.getOffsetKind() != BinlogOffsetKind.TIMESTAMP) {
411+
LOG.info(
412+
"The starting offset kind is {}, no need to apply timestamp filter",
413+
startingOffset.getOffsetKind());
411414
return event -> true;
412415
}
413416

flink-cdc-connect/flink-cdc-source-connectors/flink-connector-mysql-cdc/src/test/java/org/apache/flink/cdc/connectors/mysql/debezium/reader/BinlogSplitReaderTest.java

Lines changed: 72 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,7 @@
2727
import org.apache.flink.cdc.connectors.mysql.source.config.MySqlSourceConfig;
2828
import org.apache.flink.cdc.connectors.mysql.source.config.MySqlSourceConfigFactory;
2929
import org.apache.flink.cdc.connectors.mysql.source.offset.BinlogOffset;
30+
import org.apache.flink.cdc.connectors.mysql.source.offset.BinlogOffsetKind;
3031
import org.apache.flink.cdc.connectors.mysql.source.split.FinishedSnapshotSplitInfo;
3132
import org.apache.flink.cdc.connectors.mysql.source.split.MySqlBinlogSplit;
3233
import org.apache.flink.cdc.connectors.mysql.source.split.MySqlSnapshotSplit;
@@ -1143,6 +1144,77 @@ void testRestoreFromCheckpointWithTimestampStartingOffset() throws Exception {
11431144
assertThat(eventFilter.test(event)).isTrue();
11441145
}
11451146

1147+
@Test
1148+
void testRestoreFromEarlyCheckpointWithTimestampStartingOffset() throws Exception {
1149+
// This covers the "early checkpoint / savepoint" case: the checkpoint is taken before the
1150+
// binlog split has processed any record, so the restored split's starting offset is still
1151+
// BinlogOffsetKind.TIMESTAMP with the original submission-time timestamp (T1). The job is
1152+
// then resubmitted with a config that carries a different, later TIMESTAMP option (T2 >
1153+
// T1).
1154+
// The event filter must anchor on the restored split timestamp T1, not the newly submitted
1155+
// config timestamp T2.
1156+
1157+
// Preparations
1158+
inventoryDatabase8.createAndInitialize();
1159+
MySqlSourceConfig connectionConfig =
1160+
getConfig(MYSQL8_CONTAINER, inventoryDatabase8, new String[] {"products"});
1161+
binaryLogClient = DebeziumUtils.createBinaryClient(connectionConfig.getDbzConfiguration());
1162+
mySqlConnection = DebeziumUtils.createMySqlConnection(connectionConfig);
1163+
1164+
// T1 is the original submission-time timestamp captured in the checkpoint, while T2 is a
1165+
// different (later) timestamp coming from the resubmitted config. T2 > T1 on purpose so we
1166+
// can tell which one the filter uses.
1167+
long restoredTimestampMs = 15_000L;
1168+
long resubmittedTimestampMs = 30_000L;
1169+
1170+
// The resubmitted job config still runs in TIMESTAMP mode, but with a different timestamp.
1171+
MySqlSourceConfig sourceConfig =
1172+
getConfig(
1173+
MYSQL8_CONTAINER,
1174+
inventoryDatabase8,
1175+
StartupOptions.timestamp(resubmittedTimestampMs),
1176+
new String[] {"products"});
1177+
1178+
// The restored split still carries a TIMESTAMP starting offset (T1), which is exactly the
1179+
// state an early checkpoint / savepoint would hold.
1180+
BinlogSplitReader binlogReader = createBinlogReader(sourceConfig);
1181+
MySqlBinlogSplit checkpointSplit =
1182+
createBinlogSplit(
1183+
getConfig(
1184+
MYSQL8_CONTAINER,
1185+
inventoryDatabase8,
1186+
StartupOptions.timestamp(restoredTimestampMs),
1187+
new String[] {"products"}));
1188+
assertThat(checkpointSplit.getStartingOffset().getOffsetKind())
1189+
.isEqualTo(BinlogOffsetKind.TIMESTAMP);
1190+
assertThat(checkpointSplit.getStartingOffset().getTimestampSec())
1191+
.isEqualTo(restoredTimestampMs / 1000);
1192+
1193+
// Restore binlog reader from the early checkpoint
1194+
binlogReader.submitSplit(checkpointSplit);
1195+
1196+
Predicate<Event> eventFilter = binlogReader.getBinlogSplitReadTask().getEventFilter();
1197+
1198+
// An event between T1 and T2 must be kept, because the filter should use the restored split
1199+
// timestamp T1 (>= 15000ms) rather than the resubmitted config timestamp T2 (>= 30000ms).
1200+
Event eventBetweenT1AndT2 =
1201+
mockWriteRowsEvent(restoredTimestampMs + Duration.ofSeconds(5).toMillis());
1202+
assertThat(eventFilter.test(eventBetweenT1AndT2)).isTrue();
1203+
1204+
// An event before T1 must still be dropped, proving the timestamp filter is actually active
1205+
// on T1 and did not degrade into a pass-through filter.
1206+
Event eventBeforeT1 =
1207+
mockWriteRowsEvent(restoredTimestampMs - Duration.ofSeconds(5).toMillis());
1208+
assertThat(eventFilter.test(eventBeforeT1)).isFalse();
1209+
}
1210+
1211+
private static Event mockWriteRowsEvent(long timestampMs) {
1212+
EventHeaderV4 header = new EventHeaderV4();
1213+
header.setEventType(EventType.WRITE_ROWS);
1214+
header.setTimestamp(timestampMs);
1215+
return new Event(header, new WriteRowsEventData());
1216+
}
1217+
11461218
@Test
11471219
public void testBinlogOffsetCompareWithSnapshotAndBinlogPhase() throws Exception {
11481220
// Preparations

0 commit comments

Comments
 (0)