Skip to content

Commit 79cf736

Browse files
committed
[FLINK-39198]Add test for restoring from early checkpoint with TIMESTAMP starting offset
Signed-off-by: danzhewuju <1668926294@qq.com>
1 parent 0c3dd4c commit 79cf736

2 files changed

Lines changed: 73 additions & 0 deletions

File tree

flink-cdc-connect/flink-cdc-source-connectors/flink-connector-mysql-cdc/src/main/java/org/apache/flink/cdc/connectors/mysql/debezium/reader/BinlogSplitReader.java

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -408,6 +408,8 @@ private Predicate<Event> createEventFilter() {
408408
+ "configuration. ");
409409
}
410410
if (startingOffset.getOffsetKind() != BinlogOffsetKind.TIMESTAMP) {
411+
LOG.info("The starting offset kind is {}, no need to apply timestamp filter",
412+
startingOffset.getOffsetKind());
411413
return event -> true;
412414
}
413415

flink-cdc-connect/flink-cdc-source-connectors/flink-connector-mysql-cdc/src/test/java/org/apache/flink/cdc/connectors/mysql/debezium/reader/BinlogSplitReaderTest.java

Lines changed: 71 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,7 @@
2727
import org.apache.flink.cdc.connectors.mysql.source.config.MySqlSourceConfig;
2828
import org.apache.flink.cdc.connectors.mysql.source.config.MySqlSourceConfigFactory;
2929
import org.apache.flink.cdc.connectors.mysql.source.offset.BinlogOffset;
30+
import org.apache.flink.cdc.connectors.mysql.source.offset.BinlogOffsetKind;
3031
import org.apache.flink.cdc.connectors.mysql.source.split.FinishedSnapshotSplitInfo;
3132
import org.apache.flink.cdc.connectors.mysql.source.split.MySqlBinlogSplit;
3233
import org.apache.flink.cdc.connectors.mysql.source.split.MySqlSnapshotSplit;
@@ -1143,6 +1144,76 @@ void testRestoreFromCheckpointWithTimestampStartingOffset() throws Exception {
11431144
assertThat(eventFilter.test(event)).isTrue();
11441145
}
11451146

1147+
@Test
1148+
void testRestoreFromEarlyCheckpointWithTimestampStartingOffset() throws Exception {
1149+
// This covers the "early checkpoint / savepoint" case: the checkpoint is taken before the
1150+
// binlog split has processed any record, so the restored split's starting offset is still
1151+
// BinlogOffsetKind.TIMESTAMP with the original submission-time timestamp (T1). The job is
1152+
// then resubmitted with a config that carries a different, later TIMESTAMP option (T2 > T1).
1153+
// The event filter must anchor on the restored split timestamp T1, not the newly submitted
1154+
// config timestamp T2.
1155+
1156+
// Preparations
1157+
inventoryDatabase8.createAndInitialize();
1158+
MySqlSourceConfig connectionConfig =
1159+
getConfig(MYSQL8_CONTAINER, inventoryDatabase8, new String[] {"products"});
1160+
binaryLogClient = DebeziumUtils.createBinaryClient(connectionConfig.getDbzConfiguration());
1161+
mySqlConnection = DebeziumUtils.createMySqlConnection(connectionConfig);
1162+
1163+
// T1 is the original submission-time timestamp captured in the checkpoint, while T2 is a
1164+
// different (later) timestamp coming from the resubmitted config. T2 > T1 on purpose so we
1165+
// can tell which one the filter uses.
1166+
long restoredTimestampMs = 15_000L;
1167+
long resubmittedTimestampMs = 30_000L;
1168+
1169+
// The resubmitted job config still runs in TIMESTAMP mode, but with a different timestamp.
1170+
MySqlSourceConfig sourceConfig =
1171+
getConfig(
1172+
MYSQL8_CONTAINER,
1173+
inventoryDatabase8,
1174+
StartupOptions.timestamp(resubmittedTimestampMs),
1175+
new String[] {"products"});
1176+
1177+
// The restored split still carries a TIMESTAMP starting offset (T1), which is exactly the
1178+
// state an early checkpoint / savepoint would hold.
1179+
BinlogSplitReader binlogReader = createBinlogReader(sourceConfig);
1180+
MySqlBinlogSplit checkpointSplit =
1181+
createBinlogSplit(
1182+
getConfig(
1183+
MYSQL8_CONTAINER,
1184+
inventoryDatabase8,
1185+
StartupOptions.timestamp(restoredTimestampMs),
1186+
new String[] {"products"}));
1187+
assertThat(checkpointSplit.getStartingOffset().getOffsetKind())
1188+
.isEqualTo(BinlogOffsetKind.TIMESTAMP);
1189+
assertThat(checkpointSplit.getStartingOffset().getTimestampSec())
1190+
.isEqualTo(restoredTimestampMs / 1000);
1191+
1192+
// Restore binlog reader from the early checkpoint
1193+
binlogReader.submitSplit(checkpointSplit);
1194+
1195+
Predicate<Event> eventFilter = binlogReader.getBinlogSplitReadTask().getEventFilter();
1196+
1197+
// An event between T1 and T2 must be kept, because the filter should use the restored split
1198+
// timestamp T1 (>= 15000ms) rather than the resubmitted config timestamp T2 (>= 30000ms).
1199+
Event eventBetweenT1AndT2 =
1200+
mockWriteRowsEvent(restoredTimestampMs + Duration.ofSeconds(5).toMillis());
1201+
assertThat(eventFilter.test(eventBetweenT1AndT2)).isTrue();
1202+
1203+
// An event before T1 must still be dropped, proving the timestamp filter is actually active
1204+
// on T1 and did not degrade into a pass-through filter.
1205+
Event eventBeforeT1 =
1206+
mockWriteRowsEvent(restoredTimestampMs - Duration.ofSeconds(5).toMillis());
1207+
assertThat(eventFilter.test(eventBeforeT1)).isFalse();
1208+
}
1209+
1210+
private static Event mockWriteRowsEvent(long timestampMs) {
1211+
EventHeaderV4 header = new EventHeaderV4();
1212+
header.setEventType(EventType.WRITE_ROWS);
1213+
header.setTimestamp(timestampMs);
1214+
return new Event(header, new WriteRowsEventData());
1215+
}
1216+
11461217
@Test
11471218
public void testBinlogOffsetCompareWithSnapshotAndBinlogPhase() throws Exception {
11481219
// Preparations

0 commit comments

Comments
 (0)