Skip to content

Commit cdea685

Browse files
committed
enable flink-cdc pipeline mode
1 parent cd6289a commit cdea685

1 file changed

Lines changed: 10 additions & 19 deletions

File tree

  • tis-incr/tis-realtime-flink/src/main/java/com/qlangtech/plugins/incr/flink/cdc

tis-incr/tis-realtime-flink/src/main/java/com/qlangtech/plugins/incr/flink/cdc/SourceChannel.java

Lines changed: 10 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -79,22 +79,16 @@ public SourceChannel(boolean flinkCDCPipelineEnable, ReaderSource sourceFunction
7979
this(flinkCDCPipelineEnable, Collections.singletonList(sourceFunction));
8080
}
8181

82-
public static List<ReaderSource> getSourceFunction(
83-
DataSourceFactory dsFactory, List<ISelectedTab> tabs, ReaderSourceCreator sourceFunctionCreator) {
82+
public static List<ReaderSource> getSourceFunction(DataSourceFactory dsFactory, List<ISelectedTab> tabs, ReaderSourceCreator sourceFunctionCreator) {
8483

85-
final Optional<DataSourceFactory.ISchemaSupported>
86-
schemaSupport = DataSourceFactory.ISchemaSupported.schemaSupported(dsFactory);
84+
final Optional<DataSourceFactory.ISchemaSupported> schemaSupport = DataSourceFactory.ISchemaSupported.schemaSupported(dsFactory);
8785

8886
return getSourceFunction(dsFactory, (tab) -> {
8987
TableInDB tabsInDB = dsFactory.getTablesInDB();
90-
DataXJobInfo jobInfo = tabsInDB.createDataXJobInfo(
91-
DataXJobSubmit.TableDataXEntity
92-
.createTableEntity(null, tab.jdbcUrl, tab.getTabName()), true);
88+
DataXJobInfo jobInfo = tabsInDB.createDataXJobInfo(DataXJobSubmit.TableDataXEntity.createTableEntity(null, tab.jdbcUrl, tab.getTabName()), true);
9389
Optional<String[]> targetTableNames = jobInfo.getTargetTableNames();
9490

95-
List<String> physicsTabNames = targetTableNames
96-
.map((tabNames) -> Lists.newArrayList(tabNames))
97-
.orElseGet(() -> Lists.newArrayList(tab.getTabName()));
91+
List<String> physicsTabNames = targetTableNames.map((tabNames) -> Lists.newArrayList(tabNames)).orElseGet(() -> Lists.newArrayList(tab.getTabName()));
9892

9993
return physicsTabNames.stream().map((t) -> {
10094
EntityName entity = EntityName.parse(t, true);
@@ -108,9 +102,7 @@ public static List<ReaderSource> getSourceFunction(
108102

109103

110104
//https://ververica.github.io/flink-cdc-connectors/master/
111-
private static List<ReaderSource> getSourceFunction(
112-
DataSourceFactory dsFactory, Function<DBTable, Stream<String>> tabnameCreator
113-
, List<ISelectedTab> tabs, ReaderSourceCreator sourceFunctionCreator) {
105+
private static List<ReaderSource> getSourceFunction(DataSourceFactory dsFactory, Function<DBTable, Stream<String>> tabnameCreator, List<ISelectedTab> tabs, ReaderSourceCreator sourceFunctionCreator) {
114106

115107
try {
116108
DBConfig dbConfig = dsFactory.getDbConfig();
@@ -178,12 +170,11 @@ public void setFocusTabs(List<ISelectedTab> tabs, TableAliasMapper tabAliasMappe
178170
// ? Stream.of(DTOSourceTagProcessFunction.KEY_MERGE_ALL_TABS_IN_ONE_BUS)
179171
// : tabs.stream().map((t) -> t.getName())).collect(Collectors.toSet());
180172

181-
Map<TableAlias, DTOStream> tab2StreamMapper = this.focusTabs.stream().collect(
182-
Collectors.toMap(
183-
(name) -> (tabAliasMapper.getWithCheckNotNull(name))
184-
, (name) -> dtoStreamCreator.apply(name)));
185-
this.tab2OutputTag
186-
= new Tab2OutputTag<>(tab2StreamMapper);
173+
Map<TableAlias, DTOStream> tab2StreamMapper
174+
= this.focusTabs.stream().collect(Collectors.toMap(
175+
(name) -> (flinkCDCPipelineEnable ? TableAlias.create(name, name) : tabAliasMapper.getWithCheckNotNull(name))
176+
, (name) -> dtoStreamCreator.apply(name)));
177+
this.tab2OutputTag = new Tab2OutputTag<>(tab2StreamMapper);
187178
}
188179

189180

0 commit comments

Comments
 (0)