In the case where doris schema needs to be created, it is best to create it through this
+ * factory
+ */
+public class DorisSchemaFactory {
+
+ public static TableSchema createTableSchema(
+ String database,
+ String table,
+ Map columnFields,
+ List pkKeys,
+ DorisTableConfig dorisTableConfig,
+ String tableComment) {
+ TableSchema tableSchema = new TableSchema();
+ tableSchema.setDatabase(database);
+ tableSchema.setTable(table);
+ tableSchema.setModel(
+ CollectionUtils.isEmpty(pkKeys) ? DataModel.DUPLICATE : DataModel.UNIQUE);
+ tableSchema.setFields(columnFields);
+ tableSchema.setKeys(buildKeys(pkKeys, columnFields));
+ tableSchema.setTableComment(tableComment);
+ tableSchema.setDistributeKeys(buildDistributeKeys(pkKeys, columnFields));
+ if (Objects.nonNull(dorisTableConfig)) {
+ tableSchema.setProperties(dorisTableConfig.getTableProperties());
+ tableSchema.setTableBuckets(
+ parseTableSchemaBuckets(dorisTableConfig.getTableBuckets(), table));
+ if (ObjectUtils.isNotEmpty(dorisTableConfig.getTablePartitions())
+ && dorisTableConfig.getTablePartitions().containsKey(table)) {
+ tableSchema.setPartitionInfo(dorisTableConfig.getTablePartitions().get(table));
+ }
+ }
+ return tableSchema;
+ }
+
+ private static List buildDistributeKeys(
+ List primaryKeys, Map fields) {
+ return buildKeys(primaryKeys, fields);
+ }
+
+ /**
+ * Theoretically, the duplicate table of doris does not need to distinguish the key column, but
+ * in the actual table creation statement, the key column will be automatically added. So if it
+ * is a duplicate table, primaryKeys is empty, and we uniformly take the first field as the key.
+ */
+ private static List buildKeys(
+ List primaryKeys, Map fields) {
+ if (CollectionUtils.isNotEmpty(primaryKeys)) {
+ return primaryKeys;
+ }
+ if (!fields.isEmpty()) {
+ Entry firstField = fields.entrySet().iterator().next();
+ return Collections.singletonList(firstField.getKey());
+ }
+ return new ArrayList<>();
+ }
+
+ @VisibleForTesting
+ public static Integer parseTableSchemaBuckets(
+ Map tableBucketsMap, String tableName) {
+ if (MapUtils.isNotEmpty(tableBucketsMap)) {
+ // Firstly, if the table name is in the table-buckets map, set the buckets of the table.
+ if (tableBucketsMap.containsKey(tableName)) {
+ return tableBucketsMap.get(tableName);
+ }
+ // Secondly, iterate over the map to find a corresponding regular expression match.
+ for (Entry entry : tableBucketsMap.entrySet()) {
+ Pattern pattern = Pattern.compile(entry.getKey());
+ if (pattern.matcher(tableName).matches()) {
+ return entry.getValue();
+ }
+ }
+ }
+ return null;
+ }
+
+ public static String generateCreateTableDDL(TableSchema schema) {
+ StringBuilder sb = new StringBuilder("CREATE TABLE IF NOT EXISTS ");
+ sb.append(identifier(schema.getDatabase()))
+ .append(".")
+ .append(identifier(schema.getTable()))
+ .append("(");
+
+ Map fields = schema.getFields();
+ List keys = schema.getKeys();
+ // append keys
+ for (String key : keys) {
+ if (!fields.containsKey(key)) {
+ throw new CreateTableException("key " + key + " not found in column list");
+ }
+ FieldSchema field = fields.get(key);
+ buildColumn(sb, field, true, false);
+ }
+
+ // append partition column, auto partition column must be in keys
+ if (schema.getPartitionInfo() != null) {
+ String partitionCol = schema.getPartitionInfo().f0;
+ FieldSchema field = fields.get(partitionCol);
+ buildColumn(sb, field, true, true);
+ }
+
+ // append values
+ for (Map.Entry entry : fields.entrySet()) {
+ // skip key column
+ if (keys.contains(entry.getKey())) {
+ continue;
+ }
+ // skip partition column
+ if (schema.getPartitionInfo() != null
+ && entry.getKey().equals(schema.getPartitionInfo().f0)) {
+ continue;
+ }
+ FieldSchema field = entry.getValue();
+ buildColumn(sb, field, false, false);
+ }
+ sb = sb.deleteCharAt(sb.length() - 1);
+ sb.append(" ) ");
+ // append uniq model
+ if (DataModel.UNIQUE.equals(schema.getModel())) {
+ sb.append(schema.getModel().name())
+ .append(" KEY(")
+ .append(String.join(",", identifier(schema.getKeys())));
+
+ if (schema.getPartitionInfo() != null) {
+ sb.append(",").append(identifier(schema.getPartitionInfo().f0));
+ }
+
+ sb.append(")");
+ }
+
+ // append table comment
+ if (!StringUtils.isNullOrWhitespaceOnly(schema.getTableComment())) {
+ sb.append(" COMMENT '").append(quoteComment(schema.getTableComment())).append("' ");
+ }
+
+ // append partition info if exists
+ if (schema.getPartitionInfo() != null) {
+ sb.append(" AUTO PARTITION BY RANGE ")
+ .append(
+ String.format(
+ "(date_trunc(`%s`, '%s'))",
+ schema.getPartitionInfo().f0, schema.getPartitionInfo().f1))
+ .append("()");
+ }
+
+ // append distribute key
+ sb.append(" DISTRIBUTED BY HASH(")
+ .append(String.join(",", identifier(schema.getDistributeKeys())))
+ .append(")");
+
+ Map properties = schema.getProperties();
+ if (schema.getTableBuckets() != null) {
+
+ int bucketsNum = schema.getTableBuckets();
+ if (bucketsNum <= 0) {
+ throw new CreateTableException("The number of buckets must be positive.");
+ }
+ sb.append(" BUCKETS ").append(bucketsNum);
+ } else {
+ sb.append(" BUCKETS AUTO ");
+ }
+
+ // append properties
+ int index = 0;
+ for (Map.Entry entry : properties.entrySet()) {
+ if (index == 0) {
+ sb.append(" PROPERTIES (");
+ }
+ if (index > 0) {
+ sb.append(",");
+ }
+ sb.append(quoteProperties(entry.getKey()))
+ .append("=")
+ .append(quoteProperties(entry.getValue()));
+ index++;
+
+ if (index == (schema.getProperties().size())) {
+ sb.append(")");
+ }
+ }
+ return sb.toString();
+ }
+
+ private static void buildColumn(
+ StringBuilder sql, FieldSchema field, boolean isKey, boolean autoPartitionCol) {
+ String fieldType = field.getTypeString();
+ if (isKey && DorisType.STRING.equals(fieldType)) {
+ fieldType = String.format("%s(%s)", DorisType.VARCHAR, 65533);
+ }
+ sql.append(identifier(field.getName())).append(" ").append(fieldType);
+
+ // auto partition need set partition-column not null
+ if (autoPartitionCol) {
+ sql.append(" NOT NULL ");
+ }
+
+ if (field.getDefaultValue() != null) {
+ sql.append(" DEFAULT " + quoteDefaultValue(field.getDefaultValue()));
+ }
+ sql.append(" COMMENT '").append(quoteComment(field.getComment())).append("',");
+ }
+
+ private static String quoteProperties(String name) {
+ return "'" + name + "'";
+ }
+
+ private static List identifier(List names) {
+ return names.stream().map(DorisSchemaFactory::identifier).collect(Collectors.toList());
+ }
+
+ public static String identifier(String name) {
+ if (name.startsWith("`") && name.endsWith("`")) {
+ return name;
+ }
+ return "`" + name + "`";
+ }
+
+ public static String quoteDefaultValue(String defaultValue) {
+ // DEFAULT current_timestamp or null not need quote
+ if (defaultValue.equalsIgnoreCase("current_timestamp")
+ || defaultValue.equalsIgnoreCase("null")) {
+ return defaultValue;
+ }
+
+ return "'" + defaultValue + "'";
+ }
+
+ public static String quoteComment(String comment) {
+ if (comment == null) {
+ return "";
+ } else {
+ return comment.replaceAll("'", "\\\\'");
+ }
+ }
+
+ public static String quoteTableIdentifier(String tableIdentifier) {
+ String[] dbTable = tableIdentifier.split("\\.");
+ Preconditions.checkArgument(dbTable.length == 2);
+ return identifier(dbTable[0]) + "." + identifier(dbTable[1]);
+ }
+}
diff --git a/flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/catalog/doris/DorisSystem.java b/flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/catalog/doris/DorisSystem.java
new file mode 100644
index 000000000..9f7ed8d59
--- /dev/null
+++ b/flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/catalog/doris/DorisSystem.java
@@ -0,0 +1,193 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.flink.catalog.doris;
+
+import org.apache.flink.annotation.Public;
+import org.apache.flink.util.StringUtils;
+
+import org.apache.commons.compress.utils.Lists;
+import org.apache.doris.flink.cfg.DorisConnectionOptions;
+import org.apache.doris.flink.connection.JdbcConnectionProvider;
+import org.apache.doris.flink.connection.SimpleJdbcConnectionProvider;
+import org.apache.doris.flink.exception.DorisRuntimeException;
+import org.apache.doris.flink.exception.DorisSystemException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.Serializable;
+import java.sql.Connection;
+import java.sql.PreparedStatement;
+import java.sql.ResultSet;
+import java.sql.Statement;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+import java.util.function.Predicate;
+
+import static org.apache.flink.util.Preconditions.checkArgument;
+
+/** Doris System Operate. */
+@Public
+public class DorisSystem implements Serializable {
+ private static final long serialVersionUID = 1L;
+ private static final Logger LOG = LoggerFactory.getLogger(DorisSystem.class);
+ private final JdbcConnectionProvider jdbcConnectionProvider;
+ private static final List builtinDatabases =
+ Collections.singletonList("information_schema");
+
+ public DorisSystem(DorisConnectionOptions options) {
+ this.jdbcConnectionProvider = new SimpleJdbcConnectionProvider(options);
+ }
+
+ public List listDatabases() {
+ return extractColumnValuesBySQL(
+ "SELECT `SCHEMA_NAME` FROM `INFORMATION_SCHEMA`.`SCHEMATA`;",
+ 1,
+ dbName -> !builtinDatabases.contains(dbName));
+ }
+
+ public boolean databaseExists(String database) {
+ checkArgument(!StringUtils.isNullOrWhitespaceOnly(database));
+ return listDatabases().contains(database);
+ }
+
+ public boolean createDatabase(String database) {
+ execute(String.format("CREATE DATABASE IF NOT EXISTS %s", database));
+ return true;
+ }
+
+ public boolean dropDatabase(String database) {
+ execute(String.format("DROP DATABASE IF EXISTS %s", database));
+ return true;
+ }
+
+ public boolean tableExists(String database, String table) {
+ return databaseExists(database) && listTables(database).contains(table);
+ }
+
+ public List listTables(String databaseName) {
+ if (!databaseExists(databaseName)) {
+ throw new DorisRuntimeException("database" + databaseName + " is not exists");
+ }
+ return extractColumnValuesBySQL(
+ "SELECT TABLE_NAME FROM information_schema.`TABLES` WHERE TABLE_SCHEMA = ?",
+ 1,
+ null,
+ databaseName);
+ }
+
+ public void dropTable(String tableName) {
+ execute(String.format("DROP TABLE IF EXISTS %s", tableName));
+ }
+
+ public void createTable(TableSchema schema) {
+ String ddl = buildCreateTableDDL(schema);
+ LOG.info("Create table with ddl:{}", ddl);
+ execute(ddl);
+ }
+
+ public void execute(String sql) {
+ try (Connection connection = jdbcConnectionProvider.getOrEstablishConnection();
+ Statement statement = connection.createStatement()) {
+ statement.execute(sql);
+ } catch (Exception e) {
+ LOG.error("SQL query could not be executed: {}", sql, e);
+ throw new DorisSystemException(
+ String.format("SQL query could not be executed: %s", sql), e);
+ }
+ }
+
+ public List extractColumnValuesBySQL(
+ String sql, int columnIndex, Predicate filterFunc, Object... params) {
+
+ List columnValues = Lists.newArrayList();
+ try (Connection connection = jdbcConnectionProvider.getOrEstablishConnection();
+ PreparedStatement ps = connection.prepareStatement(sql)) {
+ if (Objects.nonNull(params) && params.length > 0) {
+ for (int i = 0; i < params.length; i++) {
+ ps.setObject(i + 1, params[i]);
+ }
+ }
+ try (ResultSet rs = ps.executeQuery()) {
+ while (rs.next()) {
+ String columnValue = rs.getString(columnIndex);
+ if (filterFunc == null || filterFunc.test(columnValue)) {
+ columnValues.add(columnValue);
+ }
+ }
+ }
+ return columnValues;
+ } catch (Exception e) {
+ throw new DorisSystemException(
+ String.format("The following SQL query could not be executed: %s", sql), e);
+ }
+ }
+
+ public static String buildCreateTableDDL(TableSchema schema) {
+ return DorisSchemaFactory.generateCreateTableDDL(schema);
+ }
+
+ public Map getTableFieldNames(String databaseName, String tableName) {
+ if (!databaseExists(databaseName)) {
+ throw new DorisRuntimeException("database" + databaseName + " is not exists");
+ }
+ String sql =
+ String.format(
+ "SELECT COLUMN_NAME,DATA_TYPE "
+ + "FROM `information_schema`.`COLUMNS` WHERE `TABLE_SCHEMA`= '%s' AND `TABLE_NAME`= '%s'",
+ databaseName, tableName);
+
+ Map columnValues = new HashMap<>();
+ try (Connection connection = jdbcConnectionProvider.getOrEstablishConnection();
+ PreparedStatement ps = connection.prepareStatement(sql);
+ ResultSet rs = ps.executeQuery()) {
+ while (rs.next()) {
+ String fieldName = rs.getString(1);
+ String datatype = rs.getString(2);
+ columnValues.put(fieldName, datatype);
+ }
+ return columnValues;
+ } catch (Exception e) {
+ LOG.error("SQL query could not be executed: {}", sql, e);
+ throw new DorisSystemException(
+ String.format("The following SQL query could not be executed: %s", sql), e);
+ }
+ }
+
+ @Deprecated
+ public static String quoteDefaultValue(String defaultValue) {
+ return DorisSchemaFactory.quoteDefaultValue(defaultValue);
+ }
+
+ @Deprecated
+ public static String quoteComment(String comment) {
+ return DorisSchemaFactory.quoteComment(comment);
+ }
+
+ @Deprecated
+ public static String identifier(String name) {
+ return DorisSchemaFactory.identifier(name);
+ }
+
+ @Deprecated
+ public static String quoteTableIdentifier(String tableIdentifier) {
+ return DorisSchemaFactory.quoteTableIdentifier(tableIdentifier);
+ }
+}
diff --git a/flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/catalog/doris/DorisType.java b/flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/catalog/doris/DorisType.java
new file mode 100644
index 000000000..7bc9fd0b2
--- /dev/null
+++ b/flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/catalog/doris/DorisType.java
@@ -0,0 +1,50 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.flink.catalog.doris;
+
+public class DorisType {
+ public static final String BOOLEAN = "BOOLEAN";
+ public static final String TINYINT = "TINYINT";
+ public static final String SMALLINT = "SMALLINT";
+ public static final String INT = "INT";
+ public static final String BIGINT = "BIGINT";
+ public static final String LARGEINT = "LARGEINT";
+ // largeint is bigint unsigned in information_schema.COLUMNS
+ public static final String BIGINT_UNSIGNED = "BIGINT UNSIGNED";
+ public static final String FLOAT = "FLOAT";
+ public static final String DOUBLE = "DOUBLE";
+ public static final String DECIMAL = "DECIMAL";
+ public static final String DECIMAL_V3 = "DECIMALV3";
+ public static final String DATE = "DATE";
+ public static final String DATE_V2 = "DATEV2";
+ public static final String DATETIME = "DATETIME";
+ public static final String DATETIME_V2 = "DATETIMEV2";
+ public static final String CHAR = "CHAR";
+ public static final String VARCHAR = "VARCHAR";
+ public static final String STRING = "STRING";
+ public static final String HLL = "HLL";
+ public static final String BITMAP = "BITMAP";
+ public static final String ARRAY = "ARRAY";
+ public static final String JSONB = "JSONB";
+ public static final String JSON = "JSON";
+ public static final String MAP = "MAP";
+ public static final String STRUCT = "STRUCT";
+ public static final String VARIANT = "VARIANT";
+ public static final String IPV4 = "IPV4";
+ public static final String IPV6 = "IPV6";
+}
diff --git a/flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/catalog/doris/FieldSchema.java b/flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/catalog/doris/FieldSchema.java
new file mode 100644
index 000000000..a8d85e1f1
--- /dev/null
+++ b/flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/catalog/doris/FieldSchema.java
@@ -0,0 +1,90 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.flink.catalog.doris;
+
+public class FieldSchema {
+ private String name;
+ private String typeString;
+ private String defaultValue;
+ private String comment;
+
+ public FieldSchema() {}
+
+ public FieldSchema(String name, String typeString, String comment) {
+ this.name = name;
+ this.typeString = typeString;
+ this.comment = comment;
+ }
+
+ public FieldSchema(String name, String typeString, String defaultValue, String comment) {
+ this.name = name;
+ this.typeString = typeString;
+ this.defaultValue = defaultValue;
+ this.comment = comment;
+ }
+
+ public String getName() {
+ return name;
+ }
+
+ public void setName(String name) {
+ this.name = name;
+ }
+
+ public String getTypeString() {
+ return typeString;
+ }
+
+ public void setTypeString(String typeString) {
+ this.typeString = typeString;
+ }
+
+ public String getComment() {
+ return comment;
+ }
+
+ public void setComment(String comment) {
+ this.comment = comment;
+ }
+
+ public String getDefaultValue() {
+ return defaultValue;
+ }
+
+ public void setDefaultValue(String defaultValue) {
+ this.defaultValue = defaultValue;
+ }
+
+ @Override
+ public String toString() {
+ return "FieldSchema{"
+ + "name='"
+ + name
+ + '\''
+ + ", typeString='"
+ + typeString
+ + '\''
+ + ", defaultValue='"
+ + defaultValue
+ + '\''
+ + ", comment='"
+ + comment
+ + '\''
+ + '}';
+ }
+}
diff --git a/flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/catalog/doris/TableSchema.java b/flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/catalog/doris/TableSchema.java
new file mode 100644
index 000000000..f7617ab19
--- /dev/null
+++ b/flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/catalog/doris/TableSchema.java
@@ -0,0 +1,148 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.flink.catalog.doris;
+
+import org.apache.flink.api.java.tuple.Tuple2;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+public class TableSchema {
+ public static final String DORIS_TABLE_REGEX = "^[a-zA-Z][a-zA-Z0-9-_]*$";
+ private String database;
+ private String table;
+ private String tableComment;
+ private Map fields;
+ private List keys = new ArrayList<>();
+ private DataModel model = DataModel.DUPLICATE;
+ private List distributeKeys = new ArrayList<>();
+ private Map properties = new HashMap<>();
+ private Integer tableBuckets;
+
+ // Currently only supports auto partition, eg: DATE_TRUNC(column,interval)
+ private Tuple2 partitionInfo;
+
+ public String getDatabase() {
+ return database;
+ }
+
+ public String getTable() {
+ return table;
+ }
+
+ public String getTableComment() {
+ return tableComment;
+ }
+
+ public Map getFields() {
+ return fields;
+ }
+
+ public List getKeys() {
+ return keys;
+ }
+
+ public DataModel getModel() {
+ return model;
+ }
+
+ public List getDistributeKeys() {
+ return distributeKeys;
+ }
+
+ public Map getProperties() {
+ return properties;
+ }
+
+ public void setDatabase(String database) {
+ this.database = database;
+ }
+
+ public void setTable(String table) {
+ this.table = table;
+ }
+
+ public void setTableComment(String tableComment) {
+ this.tableComment = tableComment;
+ }
+
+ public void setFields(Map fields) {
+ this.fields = fields;
+ }
+
+ public void setKeys(List keys) {
+ this.keys = keys;
+ }
+
+ public void setModel(DataModel model) {
+ this.model = model;
+ }
+
+ public void setDistributeKeys(List distributeKeys) {
+ this.distributeKeys = distributeKeys;
+ }
+
+ public void setProperties(Map properties) {
+ this.properties = properties;
+ }
+
+ public void setTableBuckets(Integer tableBuckets) {
+ this.tableBuckets = tableBuckets;
+ }
+
+ public Integer getTableBuckets() {
+ return tableBuckets;
+ }
+
+ public Tuple2 getPartitionInfo() {
+ return partitionInfo;
+ }
+
+ public void setPartitionInfo(Tuple2 partitionInfo) {
+ this.partitionInfo = partitionInfo;
+ }
+
+ @Override
+ public String toString() {
+ return "TableSchema{"
+ + "database='"
+ + database
+ + '\''
+ + ", table='"
+ + table
+ + '\''
+ + ", tableComment='"
+ + tableComment
+ + '\''
+ + ", fields="
+ + fields
+ + ", keys="
+ + String.join(",", keys)
+ + ", model="
+ + model.name()
+ + ", distributeKeys="
+ + String.join(",", distributeKeys)
+ + ", properties="
+ + properties
+ + ", tableBuckets="
+ + tableBuckets
+ + '}';
+ }
+}
diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/cfg/ConfigurationOptions.java b/flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/cfg/ConfigurationOptions.java
similarity index 100%
rename from flink-doris-connector/src/main/java/org/apache/doris/flink/cfg/ConfigurationOptions.java
rename to flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/cfg/ConfigurationOptions.java
diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/cfg/DorisConnectionOptions.java b/flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/cfg/DorisConnectionOptions.java
similarity index 100%
rename from flink-doris-connector/src/main/java/org/apache/doris/flink/cfg/DorisConnectionOptions.java
rename to flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/cfg/DorisConnectionOptions.java
diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/cfg/DorisExecutionOptions.java b/flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/cfg/DorisExecutionOptions.java
similarity index 100%
rename from flink-doris-connector/src/main/java/org/apache/doris/flink/cfg/DorisExecutionOptions.java
rename to flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/cfg/DorisExecutionOptions.java
diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/cfg/DorisLookupOptions.java b/flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/cfg/DorisLookupOptions.java
similarity index 100%
rename from flink-doris-connector/src/main/java/org/apache/doris/flink/cfg/DorisLookupOptions.java
rename to flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/cfg/DorisLookupOptions.java
diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/cfg/DorisOptions.java b/flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/cfg/DorisOptions.java
similarity index 100%
rename from flink-doris-connector/src/main/java/org/apache/doris/flink/cfg/DorisOptions.java
rename to flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/cfg/DorisOptions.java
diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/cfg/DorisReadOptions.java b/flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/cfg/DorisReadOptions.java
similarity index 100%
rename from flink-doris-connector/src/main/java/org/apache/doris/flink/cfg/DorisReadOptions.java
rename to flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/cfg/DorisReadOptions.java
diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/cfg/DorisStreamOptions.java b/flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/cfg/DorisStreamOptions.java
similarity index 100%
rename from flink-doris-connector/src/main/java/org/apache/doris/flink/cfg/DorisStreamOptions.java
rename to flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/cfg/DorisStreamOptions.java
diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/connection/JdbcConnectionProvider.java b/flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/connection/JdbcConnectionProvider.java
similarity index 100%
rename from flink-doris-connector/src/main/java/org/apache/doris/flink/connection/JdbcConnectionProvider.java
rename to flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/connection/JdbcConnectionProvider.java
diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/connection/SimpleJdbcConnectionProvider.java b/flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/connection/SimpleJdbcConnectionProvider.java
similarity index 100%
rename from flink-doris-connector/src/main/java/org/apache/doris/flink/connection/SimpleJdbcConnectionProvider.java
rename to flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/connection/SimpleJdbcConnectionProvider.java
diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/deserialization/DorisDeserializationSchema.java b/flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/deserialization/DorisDeserializationSchema.java
similarity index 100%
rename from flink-doris-connector/src/main/java/org/apache/doris/flink/deserialization/DorisDeserializationSchema.java
rename to flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/deserialization/DorisDeserializationSchema.java
diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/deserialization/RowDataDeserializationSchema.java b/flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/deserialization/RowDataDeserializationSchema.java
similarity index 100%
rename from flink-doris-connector/src/main/java/org/apache/doris/flink/deserialization/RowDataDeserializationSchema.java
rename to flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/deserialization/RowDataDeserializationSchema.java
diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/deserialization/SimpleListDeserializationSchema.java b/flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/deserialization/SimpleListDeserializationSchema.java
similarity index 100%
rename from flink-doris-connector/src/main/java/org/apache/doris/flink/deserialization/SimpleListDeserializationSchema.java
rename to flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/deserialization/SimpleListDeserializationSchema.java
diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/deserialization/converter/DorisRowConverter.java b/flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/deserialization/converter/DorisRowConverter.java
similarity index 100%
rename from flink-doris-connector/src/main/java/org/apache/doris/flink/deserialization/converter/DorisRowConverter.java
rename to flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/deserialization/converter/DorisRowConverter.java
diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/exception/ConnectedFailedException.java b/flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/exception/ConnectedFailedException.java
similarity index 100%
rename from flink-doris-connector/src/main/java/org/apache/doris/flink/exception/ConnectedFailedException.java
rename to flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/exception/ConnectedFailedException.java
diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/exception/CopyLoadException.java b/flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/exception/CopyLoadException.java
similarity index 100%
rename from flink-doris-connector/src/main/java/org/apache/doris/flink/exception/CopyLoadException.java
rename to flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/exception/CopyLoadException.java
diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/exception/CreateTableException.java b/flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/exception/CreateTableException.java
similarity index 100%
rename from flink-doris-connector/src/main/java/org/apache/doris/flink/exception/CreateTableException.java
rename to flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/exception/CreateTableException.java
diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/exception/DorisBatchLoadException.java b/flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/exception/DorisBatchLoadException.java
similarity index 100%
rename from flink-doris-connector/src/main/java/org/apache/doris/flink/exception/DorisBatchLoadException.java
rename to flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/exception/DorisBatchLoadException.java
diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/exception/DorisException.java b/flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/exception/DorisException.java
similarity index 100%
rename from flink-doris-connector/src/main/java/org/apache/doris/flink/exception/DorisException.java
rename to flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/exception/DorisException.java
diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/exception/DorisInternalException.java b/flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/exception/DorisInternalException.java
similarity index 100%
rename from flink-doris-connector/src/main/java/org/apache/doris/flink/exception/DorisInternalException.java
rename to flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/exception/DorisInternalException.java
diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/exception/DorisRuntimeException.java b/flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/exception/DorisRuntimeException.java
similarity index 100%
rename from flink-doris-connector/src/main/java/org/apache/doris/flink/exception/DorisRuntimeException.java
rename to flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/exception/DorisRuntimeException.java
diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/exception/DorisSchemaChangeException.java b/flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/exception/DorisSchemaChangeException.java
similarity index 100%
rename from flink-doris-connector/src/main/java/org/apache/doris/flink/exception/DorisSchemaChangeException.java
rename to flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/exception/DorisSchemaChangeException.java
diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/exception/DorisSystemException.java b/flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/exception/DorisSystemException.java
similarity index 100%
rename from flink-doris-connector/src/main/java/org/apache/doris/flink/exception/DorisSystemException.java
rename to flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/exception/DorisSystemException.java
diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/exception/IllegalArgumentException.java b/flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/exception/IllegalArgumentException.java
similarity index 100%
rename from flink-doris-connector/src/main/java/org/apache/doris/flink/exception/IllegalArgumentException.java
rename to flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/exception/IllegalArgumentException.java
diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/exception/LabelAlreadyExistsException.java b/flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/exception/LabelAlreadyExistsException.java
similarity index 100%
rename from flink-doris-connector/src/main/java/org/apache/doris/flink/exception/LabelAlreadyExistsException.java
rename to flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/exception/LabelAlreadyExistsException.java
diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/exception/ShouldNeverHappenException.java b/flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/exception/ShouldNeverHappenException.java
similarity index 100%
rename from flink-doris-connector/src/main/java/org/apache/doris/flink/exception/ShouldNeverHappenException.java
rename to flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/exception/ShouldNeverHappenException.java
diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/exception/StreamLoadException.java b/flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/exception/StreamLoadException.java
similarity index 100%
rename from flink-doris-connector/src/main/java/org/apache/doris/flink/exception/StreamLoadException.java
rename to flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/exception/StreamLoadException.java
diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/lookup/DorisJdbcLookupReader.java b/flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/lookup/DorisJdbcLookupReader.java
similarity index 100%
rename from flink-doris-connector/src/main/java/org/apache/doris/flink/lookup/DorisJdbcLookupReader.java
rename to flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/lookup/DorisJdbcLookupReader.java
diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/lookup/DorisLookupReader.java b/flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/lookup/DorisLookupReader.java
similarity index 100%
rename from flink-doris-connector/src/main/java/org/apache/doris/flink/lookup/DorisLookupReader.java
rename to flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/lookup/DorisLookupReader.java
diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/lookup/ExecutionPool.java b/flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/lookup/ExecutionPool.java
similarity index 100%
rename from flink-doris-connector/src/main/java/org/apache/doris/flink/lookup/ExecutionPool.java
rename to flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/lookup/ExecutionPool.java
diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/lookup/Get.java b/flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/lookup/Get.java
similarity index 100%
rename from flink-doris-connector/src/main/java/org/apache/doris/flink/lookup/Get.java
rename to flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/lookup/Get.java
diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/lookup/GetAction.java b/flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/lookup/GetAction.java
similarity index 100%
rename from flink-doris-connector/src/main/java/org/apache/doris/flink/lookup/GetAction.java
rename to flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/lookup/GetAction.java
diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/lookup/LookupMetrics.java b/flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/lookup/LookupMetrics.java
similarity index 100%
rename from flink-doris-connector/src/main/java/org/apache/doris/flink/lookup/LookupMetrics.java
rename to flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/lookup/LookupMetrics.java
diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/lookup/LookupSchema.java b/flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/lookup/LookupSchema.java
similarity index 100%
rename from flink-doris-connector/src/main/java/org/apache/doris/flink/lookup/LookupSchema.java
rename to flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/lookup/LookupSchema.java
diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/lookup/Record.java b/flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/lookup/Record.java
similarity index 100%
rename from flink-doris-connector/src/main/java/org/apache/doris/flink/lookup/Record.java
rename to flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/lookup/Record.java
diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/lookup/RecordKey.java b/flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/lookup/RecordKey.java
similarity index 100%
rename from flink-doris-connector/src/main/java/org/apache/doris/flink/lookup/RecordKey.java
rename to flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/lookup/RecordKey.java
diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/lookup/Worker.java b/flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/lookup/Worker.java
similarity index 100%
rename from flink-doris-connector/src/main/java/org/apache/doris/flink/lookup/Worker.java
rename to flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/lookup/Worker.java
diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/rest/PartitionDefinition.java b/flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/rest/PartitionDefinition.java
similarity index 100%
rename from flink-doris-connector/src/main/java/org/apache/doris/flink/rest/PartitionDefinition.java
rename to flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/rest/PartitionDefinition.java
diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/rest/RestService.java b/flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/rest/RestService.java
similarity index 100%
rename from flink-doris-connector/src/main/java/org/apache/doris/flink/rest/RestService.java
rename to flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/rest/RestService.java
diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/rest/SchemaUtils.java b/flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/rest/SchemaUtils.java
similarity index 100%
rename from flink-doris-connector/src/main/java/org/apache/doris/flink/rest/SchemaUtils.java
rename to flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/rest/SchemaUtils.java
diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/rest/models/BackendV2.java b/flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/rest/models/BackendV2.java
similarity index 100%
rename from flink-doris-connector/src/main/java/org/apache/doris/flink/rest/models/BackendV2.java
rename to flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/rest/models/BackendV2.java
diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/rest/models/Field.java b/flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/rest/models/Field.java
similarity index 100%
rename from flink-doris-connector/src/main/java/org/apache/doris/flink/rest/models/Field.java
rename to flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/rest/models/Field.java
diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/rest/models/QueryPlan.java b/flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/rest/models/QueryPlan.java
similarity index 100%
rename from flink-doris-connector/src/main/java/org/apache/doris/flink/rest/models/QueryPlan.java
rename to flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/rest/models/QueryPlan.java
diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/rest/models/RespContent.java b/flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/rest/models/RespContent.java
similarity index 100%
rename from flink-doris-connector/src/main/java/org/apache/doris/flink/rest/models/RespContent.java
rename to flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/rest/models/RespContent.java
diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/rest/models/Schema.java b/flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/rest/models/Schema.java
similarity index 100%
rename from flink-doris-connector/src/main/java/org/apache/doris/flink/rest/models/Schema.java
rename to flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/rest/models/Schema.java
diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/rest/models/Tablet.java b/flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/rest/models/Tablet.java
similarity index 100%
rename from flink-doris-connector/src/main/java/org/apache/doris/flink/rest/models/Tablet.java
rename to flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/rest/models/Tablet.java
diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/serialization/Routing.java b/flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/serialization/Routing.java
similarity index 100%
rename from flink-doris-connector/src/main/java/org/apache/doris/flink/serialization/Routing.java
rename to flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/serialization/Routing.java
diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/serialization/RowBatch.java b/flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/serialization/RowBatch.java
similarity index 100%
rename from flink-doris-connector/src/main/java/org/apache/doris/flink/serialization/RowBatch.java
rename to flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/serialization/RowBatch.java
diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/sink/BackendUtil.java b/flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/sink/BackendUtil.java
similarity index 100%
rename from flink-doris-connector/src/main/java/org/apache/doris/flink/sink/BackendUtil.java
rename to flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/sink/BackendUtil.java
diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/sink/DorisAbstractCommittable.java b/flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/sink/DorisAbstractCommittable.java
similarity index 100%
rename from flink-doris-connector/src/main/java/org/apache/doris/flink/sink/DorisAbstractCommittable.java
rename to flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/sink/DorisAbstractCommittable.java
diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/sink/DorisCommittable.java b/flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/sink/DorisCommittable.java
similarity index 100%
rename from flink-doris-connector/src/main/java/org/apache/doris/flink/sink/DorisCommittable.java
rename to flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/sink/DorisCommittable.java
diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/sink/DorisCommittableSerializer.java b/flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/sink/DorisCommittableSerializer.java
similarity index 100%
rename from flink-doris-connector/src/main/java/org/apache/doris/flink/sink/DorisCommittableSerializer.java
rename to flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/sink/DorisCommittableSerializer.java
diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/sink/DorisSink.java b/flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/sink/DorisSink.java
similarity index 100%
rename from flink-doris-connector/src/main/java/org/apache/doris/flink/sink/DorisSink.java
rename to flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/sink/DorisSink.java
diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/sink/EscapeHandler.java b/flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/sink/EscapeHandler.java
similarity index 100%
rename from flink-doris-connector/src/main/java/org/apache/doris/flink/sink/EscapeHandler.java
rename to flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/sink/EscapeHandler.java
diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/sink/HttpGetWithEntity.java b/flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/sink/HttpGetWithEntity.java
similarity index 100%
rename from flink-doris-connector/src/main/java/org/apache/doris/flink/sink/HttpGetWithEntity.java
rename to flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/sink/HttpGetWithEntity.java
diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/sink/HttpPutBuilder.java b/flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/sink/HttpPutBuilder.java
similarity index 100%
rename from flink-doris-connector/src/main/java/org/apache/doris/flink/sink/HttpPutBuilder.java
rename to flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/sink/HttpPutBuilder.java
diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/sink/HttpUtil.java b/flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/sink/HttpUtil.java
similarity index 100%
rename from flink-doris-connector/src/main/java/org/apache/doris/flink/sink/HttpUtil.java
rename to flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/sink/HttpUtil.java
diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/sink/LoadStatus.java b/flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/sink/LoadStatus.java
similarity index 100%
rename from flink-doris-connector/src/main/java/org/apache/doris/flink/sink/LoadStatus.java
rename to flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/sink/LoadStatus.java
diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/sink/ResponseUtil.java b/flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/sink/ResponseUtil.java
similarity index 100%
rename from flink-doris-connector/src/main/java/org/apache/doris/flink/sink/ResponseUtil.java
rename to flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/sink/ResponseUtil.java
diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/sink/batch/BatchBufferHttpEntity.java b/flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/sink/batch/BatchBufferHttpEntity.java
similarity index 100%
rename from flink-doris-connector/src/main/java/org/apache/doris/flink/sink/batch/BatchBufferHttpEntity.java
rename to flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/sink/batch/BatchBufferHttpEntity.java
diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/sink/batch/BatchBufferStream.java b/flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/sink/batch/BatchBufferStream.java
similarity index 100%
rename from flink-doris-connector/src/main/java/org/apache/doris/flink/sink/batch/BatchBufferStream.java
rename to flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/sink/batch/BatchBufferStream.java
diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/sink/batch/BatchRecordBuffer.java b/flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/sink/batch/BatchRecordBuffer.java
similarity index 100%
rename from flink-doris-connector/src/main/java/org/apache/doris/flink/sink/batch/BatchRecordBuffer.java
rename to flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/sink/batch/BatchRecordBuffer.java
diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/sink/batch/DorisBatchSink.java b/flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/sink/batch/DorisBatchSink.java
similarity index 100%
rename from flink-doris-connector/src/main/java/org/apache/doris/flink/sink/batch/DorisBatchSink.java
rename to flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/sink/batch/DorisBatchSink.java
diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/sink/batch/DorisBatchStreamLoad.java b/flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/sink/batch/DorisBatchStreamLoad.java
similarity index 100%
rename from flink-doris-connector/src/main/java/org/apache/doris/flink/sink/batch/DorisBatchStreamLoad.java
rename to flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/sink/batch/DorisBatchStreamLoad.java
diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/sink/batch/DorisBatchWriter.java b/flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/sink/batch/DorisBatchWriter.java
similarity index 100%
rename from flink-doris-connector/src/main/java/org/apache/doris/flink/sink/batch/DorisBatchWriter.java
rename to flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/sink/batch/DorisBatchWriter.java
diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/sink/batch/RecordWithMeta.java b/flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/sink/batch/RecordWithMeta.java
similarity index 100%
rename from flink-doris-connector/src/main/java/org/apache/doris/flink/sink/batch/RecordWithMeta.java
rename to flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/sink/batch/RecordWithMeta.java
diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/sink/committer/DorisCommitter.java b/flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/sink/committer/DorisCommitter.java
similarity index 100%
rename from flink-doris-connector/src/main/java/org/apache/doris/flink/sink/committer/DorisCommitter.java
rename to flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/sink/committer/DorisCommitter.java
diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/sink/copy/BackoffAndRetryUtils.java b/flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/sink/copy/BackoffAndRetryUtils.java
similarity index 100%
rename from flink-doris-connector/src/main/java/org/apache/doris/flink/sink/copy/BackoffAndRetryUtils.java
rename to flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/sink/copy/BackoffAndRetryUtils.java
diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/sink/copy/BatchRecordBuffer.java b/flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/sink/copy/BatchRecordBuffer.java
similarity index 100%
rename from flink-doris-connector/src/main/java/org/apache/doris/flink/sink/copy/BatchRecordBuffer.java
rename to flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/sink/copy/BatchRecordBuffer.java
diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/sink/copy/BatchStageLoad.java b/flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/sink/copy/BatchStageLoad.java
similarity index 100%
rename from flink-doris-connector/src/main/java/org/apache/doris/flink/sink/copy/BatchStageLoad.java
rename to flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/sink/copy/BatchStageLoad.java
diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/sink/copy/CopyCommittableSerializer.java b/flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/sink/copy/CopyCommittableSerializer.java
similarity index 100%
rename from flink-doris-connector/src/main/java/org/apache/doris/flink/sink/copy/CopyCommittableSerializer.java
rename to flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/sink/copy/CopyCommittableSerializer.java
diff --git a/flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/sink/copy/CopySQLBuilder.java b/flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/sink/copy/CopySQLBuilder.java
new file mode 100644
index 000000000..8edfba3f1
--- /dev/null
+++ b/flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/sink/copy/CopySQLBuilder.java
@@ -0,0 +1,97 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.flink.sink.copy;
+
+import org.apache.doris.flink.catalog.doris.DorisSchemaFactory;
+import org.apache.doris.flink.cfg.DorisExecutionOptions;
+
+import java.util.Arrays;
+import java.util.List;
+import java.util.Map;
+import java.util.Properties;
+import java.util.StringJoiner;
+
+import static org.apache.doris.flink.sink.writer.LoadConstants.COLUMNS_KEY;
+import static org.apache.doris.flink.sink.writer.LoadConstants.FIELD_DELIMITER_KEY;
+import static org.apache.doris.flink.sink.writer.LoadConstants.FORMAT_KEY;
+import static org.apache.doris.flink.sink.writer.LoadConstants.JSON;
+import static org.apache.doris.flink.sink.writer.LoadConstants.LINE_DELIMITER_KEY;
+import static org.apache.doris.flink.sink.writer.LoadConstants.READ_JSON_BY_LINE;
+
+public class CopySQLBuilder {
+ private static final String COPY_SYNC = "copy.async";
+ private static final String COPY_DELETE = "copy.use_delete_sign";
+ private static final String STRIP_OUT_ARRAY = "strip_outer_array";
+ private final DorisExecutionOptions executionOptions;
+ private final String tableIdentifier;
+ private final List fileList;
+ private Properties properties;
+
+ public CopySQLBuilder(
+ String tableIdentifier, DorisExecutionOptions executionOptions, List fileList) {
+ this.tableIdentifier = tableIdentifier;
+ this.executionOptions = executionOptions;
+ this.fileList = fileList;
+ this.properties = executionOptions.getStreamLoadProp();
+ }
+
+ public String buildCopySQL() {
+ StringBuilder sb = new StringBuilder();
+ sb.append("COPY INTO ")
+ .append(DorisSchemaFactory.quoteTableIdentifier(tableIdentifier))
+ .append(" FROM @~('{")
+ .append(String.join(",", fileList))
+ .append("}') ")
+ .append("PROPERTIES (");
+
+ // copy into must be sync
+ properties.put(COPY_SYNC, false);
+ if (executionOptions.getDeletable()) {
+ properties.put(COPY_DELETE, true);
+ }
+
+ if (JSON.equals(properties.getProperty(FORMAT_KEY))) {
+ properties.put(STRIP_OUT_ARRAY, false);
+ }
+
+ properties.remove(READ_JSON_BY_LINE);
+ properties.remove(COLUMNS_KEY);
+ StringJoiner props = new StringJoiner(",");
+ for (Map.Entry entry : properties.entrySet()) {
+ String key = concatPropPrefix(String.valueOf(entry.getKey()));
+ String value = String.valueOf(entry.getValue());
+ String prop = String.format("'%s'='%s'", key, value);
+ props.add(prop);
+ }
+ sb.append(props).append(")");
+ return sb.toString();
+ }
+
+ static final List PREFIX_LIST =
+ Arrays.asList(FIELD_DELIMITER_KEY, LINE_DELIMITER_KEY, STRIP_OUT_ARRAY);
+
+ private String concatPropPrefix(String key) {
+ if (PREFIX_LIST.contains(key)) {
+ return "file." + key;
+ }
+ if (FORMAT_KEY.equals(key)) {
+ return "file.type";
+ }
+ return key;
+ }
+}
diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/sink/copy/DorisCopyCommittable.java b/flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/sink/copy/DorisCopyCommittable.java
similarity index 100%
rename from flink-doris-connector/src/main/java/org/apache/doris/flink/sink/copy/DorisCopyCommittable.java
rename to flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/sink/copy/DorisCopyCommittable.java
diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/sink/copy/DorisCopyCommitter.java b/flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/sink/copy/DorisCopyCommitter.java
similarity index 100%
rename from flink-doris-connector/src/main/java/org/apache/doris/flink/sink/copy/DorisCopyCommitter.java
rename to flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/sink/copy/DorisCopyCommitter.java
diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/sink/copy/DorisCopyWriter.java b/flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/sink/copy/DorisCopyWriter.java
similarity index 100%
rename from flink-doris-connector/src/main/java/org/apache/doris/flink/sink/copy/DorisCopyWriter.java
rename to flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/sink/copy/DorisCopyWriter.java
diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/sink/copy/HttpPostBuilder.java b/flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/sink/copy/HttpPostBuilder.java
similarity index 100%
rename from flink-doris-connector/src/main/java/org/apache/doris/flink/sink/copy/HttpPostBuilder.java
rename to flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/sink/copy/HttpPostBuilder.java
diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/sink/copy/models/BaseResponse.java b/flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/sink/copy/models/BaseResponse.java
similarity index 100%
rename from flink-doris-connector/src/main/java/org/apache/doris/flink/sink/copy/models/BaseResponse.java
rename to flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/sink/copy/models/BaseResponse.java
diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/sink/copy/models/CopyIntoResp.java b/flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/sink/copy/models/CopyIntoResp.java
similarity index 100%
rename from flink-doris-connector/src/main/java/org/apache/doris/flink/sink/copy/models/CopyIntoResp.java
rename to flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/sink/copy/models/CopyIntoResp.java
diff --git a/flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/sink/schema/SQLParserSchemaManager.java b/flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/sink/schema/SQLParserSchemaManager.java
new file mode 100644
index 000000000..626f17eca
--- /dev/null
+++ b/flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/sink/schema/SQLParserSchemaManager.java
@@ -0,0 +1,401 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.flink.sink.schema;
+
+import org.apache.flink.annotation.VisibleForTesting;
+import org.apache.flink.util.Preconditions;
+
+import net.sf.jsqlparser.JSQLParserException;
+import net.sf.jsqlparser.parser.CCJSqlParserUtil;
+import net.sf.jsqlparser.statement.Statement;
+import net.sf.jsqlparser.statement.alter.Alter;
+import net.sf.jsqlparser.statement.alter.AlterExpression;
+import net.sf.jsqlparser.statement.alter.AlterExpression.ColumnDataType;
+import net.sf.jsqlparser.statement.alter.AlterOperation;
+import net.sf.jsqlparser.statement.create.table.ColDataType;
+import net.sf.jsqlparser.statement.create.table.CreateTable;
+import net.sf.jsqlparser.statement.create.table.Index;
+import org.apache.commons.collections.CollectionUtils;
+import org.apache.doris.flink.catalog.doris.DorisSchemaFactory;
+import org.apache.doris.flink.catalog.doris.DorisType;
+import org.apache.doris.flink.catalog.doris.FieldSchema;
+import org.apache.doris.flink.catalog.doris.TableSchema;
+import org.apache.doris.flink.sink.writer.serializer.jsondebezium.JsonDebeziumChangeUtils;
+import org.apache.doris.flink.tools.cdc.DorisTableConfig;
+import org.apache.doris.flink.tools.cdc.SourceConnector;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.Serializable;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.HashSet;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Set;
+
+/** Use {@link CCJSqlParserUtil} to parse SQL statements. */
+public class SQLParserSchemaManager implements Serializable {
+ private static final Logger LOG = LoggerFactory.getLogger(SQLParserSchemaManager.class);
+ private static final String DEFAULT = "DEFAULT";
+ private static final String COMMENT = "COMMENT";
+ private static final String PRIMARY = "PRIMARY";
+ private static final String PRIMARY_KEY = "PRIMARY KEY";
+ private static final String UNIQUE = "UNIQUE";
+ private static final String DORIS_CURRENT_TIMESTAMP = "CURRENT_TIMESTAMP";
+ private static final List TYPE_MODIFIER =
+ Arrays.asList("UNSIGNED", "ZEROFILL", "PRECISION");
+ private static final Set sourceConnectorTimeValues =
+ new HashSet<>(
+ Arrays.asList(
+ "SYSDATE",
+ "SYSTIMESTAMP",
+ "CURRENT_TIMESTAMP",
+ "NOW()",
+ "CURRENT TIMESTAMP",
+ "GETDATE()"));
+
+ /**
+ * Doris' schema change only supports ADD, DROP, and RENAME operations. This method is only used
+ * to parse the above schema change operations.
+ */
+ public List parseAlterDDLs(
+ SourceConnector sourceConnector, String ddl, String dorisTable) {
+ List ddlList = new ArrayList<>();
+ try {
+ Statement statement = CCJSqlParserUtil.parse(ddl);
+ if (statement instanceof Alter) {
+ Alter alterStatement = (Alter) statement;
+ List alterExpressions = alterStatement.getAlterExpressions();
+ for (AlterExpression alterExpression : alterExpressions) {
+ AlterOperation operation = alterExpression.getOperation();
+ switch (operation) {
+ case DROP:
+ String dropColumnDDL =
+ processDropColumnOperation(alterExpression, dorisTable);
+ ddlList.add(dropColumnDDL);
+ break;
+ case ADD:
+ List addColumnDDL =
+ processAddColumnOperation(
+ sourceConnector, alterExpression, dorisTable);
+ ddlList.addAll(addColumnDDL);
+ break;
+ case CHANGE:
+ String changeColumnDDL =
+ processChangeColumnOperation(alterExpression, dorisTable);
+ ddlList.add(changeColumnDDL);
+ break;
+ case RENAME:
+ String renameColumnDDL =
+ processRenameColumnOperation(alterExpression, dorisTable);
+ ddlList.add(renameColumnDDL);
+ break;
+ default:
+ LOG.warn(
+ "Unsupported alter ddl operations, operation={}, ddl={}",
+ operation.name(),
+ ddl);
+ }
+ }
+ } else {
+ LOG.warn("Unsupported ddl operations, ddl={}", ddl);
+ }
+ } catch (JSQLParserException e) {
+ LOG.warn("Failed to parse DDL SQL, SQL={}", ddl, e);
+ }
+ return ddlList;
+ }
+
+ public TableSchema parseCreateTableStatement(
+ SourceConnector sourceConnector,
+ String ddl,
+ String dorisTable,
+ DorisTableConfig dorisTableConfig) {
+ try {
+ Statement statement = CCJSqlParserUtil.parse(ddl);
+ if (statement instanceof CreateTable) {
+ CreateTable createTable = (CreateTable) statement;
+ Map columnFields = new LinkedHashMap<>();
+ List pkKeys = new ArrayList<>();
+ createTable
+ .getColumnDefinitions()
+ .forEach(
+ column -> {
+ String columnName = column.getColumnName();
+ List columnSpecs = column.getColumnSpecs();
+ FieldSchema fieldSchema =
+ getFieldSchema(
+ column.getColumnName(),
+ column.getColumnSpecs(),
+ column.getColDataType(),
+ sourceConnector);
+ columnFields.put(columnName, fieldSchema);
+ extractColumnPrimaryKey(columnName, columnSpecs, pkKeys);
+ });
+
+ List indexes = createTable.getIndexes();
+ extractIndexesPrimaryKey(indexes, pkKeys);
+ String[] dbTable = dorisTable.split("\\.");
+ Preconditions.checkArgument(dbTable.length == 2);
+
+ return DorisSchemaFactory.createTableSchema(
+ dbTable[0],
+ dbTable[1],
+ columnFields,
+ pkKeys,
+ dorisTableConfig,
+ extractTableComment(createTable.getTableOptionsStrings()));
+ } else {
+ LOG.warn(
+ "Unsupported statement type. ddl={}, sourceConnector={}, dorisTable={}",
+ ddl,
+ sourceConnector.getConnectorName(),
+ dorisTable);
+ }
+ } catch (JSQLParserException e) {
+ LOG.warn(
+ "Failed to parse create table statement. ddl={}, sourceConnector={}, dorisTable={}",
+ ddl,
+ sourceConnector.getConnectorName(),
+ dorisTable);
+ }
+ return null;
+ }
+
+ private String extractTypeModifier(List columnSpecs) {
+ if (CollectionUtils.isEmpty(columnSpecs)) {
+ return "";
+ }
+ StringBuilder builder = new StringBuilder();
+ for (String columnSpec : columnSpecs) {
+ String columnSpecUpperCase = columnSpec.toUpperCase(Locale.ROOT);
+ if (TYPE_MODIFIER.contains(columnSpecUpperCase)) {
+ builder.append(" ").append(columnSpecUpperCase);
+ }
+ }
+ return builder.toString();
+ }
+
+ private void extractIndexesPrimaryKey(List indexes, List pkKeys) {
+ if (CollectionUtils.isEmpty(indexes)) {
+ return;
+ }
+ indexes.stream()
+ .filter(
+ index ->
+ PRIMARY_KEY.equalsIgnoreCase(index.getType())
+ || UNIQUE.equalsIgnoreCase(index.getType()))
+ .flatMap(index -> index.getColumnsNames().stream())
+ .distinct()
+ .filter(
+ primaryKey ->
+ pkKeys.stream()
+ .noneMatch(pkKey -> pkKey.equalsIgnoreCase(primaryKey)))
+ .forEach(pkKeys::add);
+ }
+
+ private void extractColumnPrimaryKey(
+ String columnName, List columnSpecs, List pkKeys) {
+ if (CollectionUtils.isEmpty(columnSpecs)) {
+ return;
+ }
+ for (String columnSpec : columnSpecs) {
+ if (PRIMARY.equalsIgnoreCase(columnSpec)) {
+ pkKeys.add(columnName);
+ }
+ }
+ }
+
+ private String extractTableComment(List tableOptionsStrings) {
+ if (CollectionUtils.isEmpty(tableOptionsStrings)) {
+ return null;
+ }
+
+ for (int i = 0; i < tableOptionsStrings.size(); i++) {
+ String columnSpec = tableOptionsStrings.get(i);
+ // If you encounter a COMMENT and the next element is an equal sign (=)
+ if (COMMENT.equalsIgnoreCase(columnSpec)
+ && i + 1 < tableOptionsStrings.size()
+ && "=".equals(tableOptionsStrings.get(i + 1))) {
+ tableOptionsStrings.remove(i + 1);
+ break;
+ }
+ }
+
+ return extractAdjacentString(tableOptionsStrings, COMMENT);
+ }
+
+ private String parseDataType(
+ ColDataType colDataType, String typeModifier, SourceConnector sourceConnector) {
+ String dataType = colDataType.getDataType();
+ int length = 0;
+ int scale = 0;
+ if (CollectionUtils.isNotEmpty(colDataType.getArgumentsStringList())) {
+ List argumentsStringList = colDataType.getArgumentsStringList();
+ length = Integer.parseInt(argumentsStringList.get(0));
+ if (argumentsStringList.size() == 2) {
+ scale = Integer.parseInt(argumentsStringList.get(1));
+ }
+ }
+ return JsonDebeziumChangeUtils.buildDorisTypeName(
+ sourceConnector, dataType + typeModifier, length, scale);
+ }
+
+ private String processDropColumnOperation(AlterExpression alterExpression, String dorisTable) {
+ String dropColumnDDL =
+ SchemaChangeHelper.buildDropColumnDDL(dorisTable, alterExpression.getColumnName());
+ LOG.info("Parsed drop column DDL SQL is: {}", dropColumnDDL);
+ return dropColumnDDL;
+ }
+
+ private List processAddColumnOperation(
+ SourceConnector sourceConnector, AlterExpression alterExpression, String dorisTable) {
+ List colDataTypeList = alterExpression.getColDataTypeList();
+ List addColumnList = new ArrayList<>();
+ for (ColumnDataType columnDataType : colDataTypeList) {
+ FieldSchema fieldSchema =
+ getFieldSchema(
+ columnDataType.getColumnName(),
+ columnDataType.getColumnSpecs(),
+ columnDataType.getColDataType(),
+ sourceConnector);
+ String addColumnDDL = SchemaChangeHelper.buildAddColumnDDL(dorisTable, fieldSchema);
+ LOG.info("Parsed add column DDL SQL is: {}", addColumnDDL);
+ addColumnList.add(addColumnDDL);
+ }
+ return addColumnList;
+ }
+
+ private FieldSchema getFieldSchema(
+ String columnName,
+ List columnSpecs,
+ ColDataType colDataType,
+ SourceConnector sourceConnector) {
+ String typeModifier = extractTypeModifier(columnSpecs);
+ String datatype = parseDataType(colDataType, typeModifier, sourceConnector);
+
+ String defaultValue = extractDefaultValue(datatype, columnSpecs);
+ String comment = extractComment(columnSpecs);
+ return new FieldSchema(columnName, datatype, defaultValue, comment);
+ }
+
+ private String processChangeColumnOperation(
+ AlterExpression alterExpression, String dorisTable) {
+ String columnNewName = alterExpression.getColDataTypeList().get(0).getColumnName();
+ String columnOldName = alterExpression.getColumnOldName();
+ String renameColumnDDL =
+ SchemaChangeHelper.buildRenameColumnDDL(dorisTable, columnOldName, columnNewName);
+ LOG.warn(
+ "Note: Only rename column names are supported in doris. "
+ + "Therefore, the change syntax used here only supports the use of rename."
+ + " Parsed change column DDL SQL is: {}",
+ renameColumnDDL);
+ return renameColumnDDL;
+ }
+
+ private String processRenameColumnOperation(
+ AlterExpression alterExpression, String dorisTable) {
+ String columnNewName = alterExpression.getColumnName();
+ String columnOldName = alterExpression.getColumnOldName();
+ String renameColumnDDL =
+ SchemaChangeHelper.buildRenameColumnDDL(dorisTable, columnOldName, columnNewName);
+ LOG.info("Parsed rename column DDL SQL is: {}", renameColumnDDL);
+ return renameColumnDDL;
+ }
+
+ @VisibleForTesting
+ public String extractDefaultValue(String dateType, List columnSpecs) {
+ if (CollectionUtils.isEmpty(columnSpecs)) {
+ return null;
+ }
+ String adjacentDefaultValue = extractAdjacentString(columnSpecs, DEFAULT);
+ return parseDorisDefaultValue(dateType, adjacentDefaultValue);
+ }
+
+ private String parseDorisDefaultValue(String dateType, String defaultValue) {
+ if (Objects.isNull(defaultValue)) {
+ return null;
+ }
+ // In doris, DATETIME supports specifying the current time by default through
+ // CURRENT_TIMESTAMP.
+ if ((dateType.startsWith(DorisType.DATETIME) || dateType.startsWith(DorisType.DATETIME_V2))
+ && sourceConnectorTimeValues.contains(defaultValue.toUpperCase(Locale.ROOT))) {
+ return DORIS_CURRENT_TIMESTAMP;
+ }
+ return defaultValue;
+ }
+
+ private String extractAdjacentString(List columnSpecs, String key) {
+ int columnSpecsSize = columnSpecs.size();
+ for (int i = 0; i < columnSpecsSize; i++) {
+ String columnSpec = columnSpecs.get(i);
+ if (key.equalsIgnoreCase(columnSpec) && i < columnSpecsSize - 1) {
+ String adjacentString = columnSpecs.get(i + 1);
+ if (!(DEFAULT.equalsIgnoreCase(adjacentString))
+ && !(COMMENT.equalsIgnoreCase(adjacentString))) {
+ return removeQuotes(adjacentString);
+ }
+ LOG.warn(
+ "Failed to extract adjacent string value. columnSpecs={}, key={}",
+ String.join(",", columnSpecs),
+ key);
+ }
+ }
+ return null;
+ }
+
+ @VisibleForTesting
+ public String extractComment(List columnSpecs) {
+ if (CollectionUtils.isEmpty(columnSpecs)) {
+ return null;
+ }
+ return extractAdjacentString(columnSpecs, COMMENT);
+ }
+
+ private String removeQuotes(String content) {
+ content = removeContinuousChar(content, '\'');
+ content = removeContinuousChar(content, '\"');
+ return content;
+ }
+
+ /**
+ * remove the continuous char in the string from both sides.
+ *
+ * @param str the input string, target the char to be removed
+ * @return the string without continuous chars from both sides
+ */
+ @VisibleForTesting
+ public String removeContinuousChar(String str, char target) {
+ if (str == null || str.length() < 2) {
+ return str;
+ }
+ int start = 0;
+ int end = str.length() - 1;
+ while (start <= end && str.charAt(start) == target) {
+ start++;
+ }
+ while (end >= start && str.charAt(end) == target) {
+ end--;
+ }
+ return str.substring(start, end + 1);
+ }
+}
diff --git a/flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/sink/schema/SchemaChangeHelper.java b/flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/sink/schema/SchemaChangeHelper.java
new file mode 100644
index 000000000..aacd6f7f6
--- /dev/null
+++ b/flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/sink/schema/SchemaChangeHelper.java
@@ -0,0 +1,218 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.flink.sink.schema;
+
+import org.apache.commons.collections.CollectionUtils;
+import org.apache.commons.compress.utils.Lists;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.doris.flink.catalog.doris.DorisSchemaFactory;
+import org.apache.doris.flink.catalog.doris.FieldSchema;
+
+import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
+
+public class SchemaChangeHelper {
+ public static final String DEFAULT_DATABASE = "information_schema";
+
+ private static final List dropFieldSchemas = Lists.newArrayList();
+ private static final List addFieldSchemas = Lists.newArrayList();
+ // Used to determine whether the doris table supports ddl
+ private static final List ddlSchemas = Lists.newArrayList();
+ private static final String ADD_DDL = "ALTER TABLE %s ADD COLUMN %s %s";
+ private static final String DROP_DDL = "ALTER TABLE %s DROP COLUMN %s";
+ private static final String RENAME_DDL = "ALTER TABLE %s RENAME COLUMN %s %s";
+ private static final String CHECK_COLUMN_EXISTS =
+ "SELECT COLUMN_NAME FROM information_schema.`COLUMNS` WHERE TABLE_SCHEMA = '%s' AND TABLE_NAME = '%s' AND COLUMN_NAME = '%s'";
+ private static final String CHECK_DATABASE_EXISTS =
+ "SELECT `SCHEMA_NAME` FROM `INFORMATION_SCHEMA`.`SCHEMATA` WHERE SCHEMA_NAME = '%s'";
+ private static final String CREATE_DATABASE_DDL = "CREATE DATABASE IF NOT EXISTS %s";
+ private static final String MODIFY_TYPE_DDL = "ALTER TABLE %s MODIFY COLUMN %s %s";
+ private static final String MODIFY_COMMENT_DDL = "ALTER TABLE %s MODIFY COLUMN %s COMMENT '%s'";
+ private static final String SHOW_FULL_COLUMN_DDL = "SHOW FULL COLUMNS FROM `%s`.`%s`";
+
+ public static void compareSchema(
+ Map updateFiledSchemaMap,
+ Map originFieldSchemaMap) {
+ dropFieldSchemas.clear();
+ addFieldSchemas.clear();
+ for (Entry updateFieldSchema : updateFiledSchemaMap.entrySet()) {
+ String columName = updateFieldSchema.getKey();
+ if (!originFieldSchemaMap.containsKey(columName)) {
+ addFieldSchemas.add(updateFieldSchema.getValue());
+ originFieldSchemaMap.put(columName, updateFieldSchema.getValue());
+ }
+ }
+ for (Entry originFieldSchema : originFieldSchemaMap.entrySet()) {
+ String columName = originFieldSchema.getKey();
+ if (!updateFiledSchemaMap.containsKey(columName)) {
+ dropFieldSchemas.add(columName);
+ }
+ }
+ if (CollectionUtils.isNotEmpty(dropFieldSchemas)) {
+ dropFieldSchemas.forEach(originFieldSchemaMap::remove);
+ }
+ }
+
+ public static List generateRenameDDLSql(
+ String table,
+ String oldColumnName,
+ String newColumnName,
+ Map originFieldSchemaMap) {
+ ddlSchemas.clear();
+ List ddlList = Lists.newArrayList();
+ FieldSchema fieldSchema = null;
+ for (Entry originFieldSchema : originFieldSchemaMap.entrySet()) {
+ if (originFieldSchema.getKey().equals(oldColumnName)) {
+ fieldSchema = originFieldSchema.getValue();
+ ddlList.add(buildRenameColumnDDL(table, oldColumnName, newColumnName));
+ ddlSchemas.add(new DDLSchema(oldColumnName, false));
+ }
+ }
+ originFieldSchemaMap.remove(oldColumnName);
+ originFieldSchemaMap.put(newColumnName, fieldSchema);
+ return ddlList;
+ }
+
+ public static List generateDDLSql(String table) {
+ ddlSchemas.clear();
+ List ddlList = Lists.newArrayList();
+ for (FieldSchema fieldSchema : addFieldSchemas) {
+ ddlList.add(buildAddColumnDDL(table, fieldSchema));
+ ddlSchemas.add(new DDLSchema(fieldSchema.getName(), false));
+ }
+ for (String columName : dropFieldSchemas) {
+ ddlList.add(buildDropColumnDDL(table, columName));
+ ddlSchemas.add(new DDLSchema(columName, true));
+ }
+
+ dropFieldSchemas.clear();
+ addFieldSchemas.clear();
+ return ddlList;
+ }
+
+ public static String buildAddColumnDDL(String tableIdentifier, FieldSchema fieldSchema) {
+ String name = fieldSchema.getName();
+ String type = fieldSchema.getTypeString();
+ String defaultValue = fieldSchema.getDefaultValue();
+ String comment = fieldSchema.getComment();
+ StringBuilder addDDL =
+ new StringBuilder(
+ String.format(
+ ADD_DDL,
+ DorisSchemaFactory.quoteTableIdentifier(tableIdentifier),
+ DorisSchemaFactory.identifier(name),
+ type));
+ if (defaultValue != null) {
+ addDDL.append(" DEFAULT ").append(DorisSchemaFactory.quoteDefaultValue(defaultValue));
+ }
+ commentColumn(addDDL, comment);
+ return addDDL.toString();
+ }
+
+ public static String buildDropColumnDDL(String tableIdentifier, String columName) {
+ return String.format(
+ DROP_DDL,
+ DorisSchemaFactory.quoteTableIdentifier(tableIdentifier),
+ DorisSchemaFactory.identifier(columName));
+ }
+
+ public static String buildRenameColumnDDL(
+ String tableIdentifier, String oldColumnName, String newColumnName) {
+ return String.format(
+ RENAME_DDL,
+ DorisSchemaFactory.quoteTableIdentifier(tableIdentifier),
+ DorisSchemaFactory.identifier(oldColumnName),
+ DorisSchemaFactory.identifier(newColumnName));
+ }
+
+ public static String buildColumnExistsQuery(String database, String table, String column) {
+ return String.format(CHECK_COLUMN_EXISTS, database, table, column);
+ }
+
+ public static String buildDatabaseExistsQuery(String database) {
+ return String.format(CHECK_DATABASE_EXISTS, database);
+ }
+
+ public static String buildCreateDatabaseDDL(String database) {
+ return String.format(CREATE_DATABASE_DDL, DorisSchemaFactory.identifier(database));
+ }
+
+ public static String buildModifyColumnCommentDDL(
+ String tableIdentifier, String columnName, String newComment) {
+ return String.format(
+ MODIFY_COMMENT_DDL,
+ DorisSchemaFactory.quoteTableIdentifier(tableIdentifier),
+ DorisSchemaFactory.identifier(columnName),
+ DorisSchemaFactory.quoteComment(newComment));
+ }
+
+ public static String buildModifyColumnDataTypeDDL(
+ String tableIdentifier, FieldSchema fieldSchema) {
+ String columnName = fieldSchema.getName();
+ String dataType = fieldSchema.getTypeString();
+ String comment = fieldSchema.getComment();
+ String defaultValue = fieldSchema.getDefaultValue();
+ StringBuilder modifyDDL =
+ new StringBuilder(
+ String.format(
+ MODIFY_TYPE_DDL,
+ DorisSchemaFactory.quoteTableIdentifier(tableIdentifier),
+ DorisSchemaFactory.identifier(columnName),
+ dataType));
+ if (defaultValue != null) {
+ modifyDDL
+ .append(" DEFAULT ")
+ .append(DorisSchemaFactory.quoteDefaultValue(defaultValue));
+ }
+ commentColumn(modifyDDL, comment);
+ return modifyDDL.toString();
+ }
+
+ private static void commentColumn(StringBuilder ddl, String comment) {
+ if (StringUtils.isNotEmpty(comment)) {
+ ddl.append(" COMMENT '").append(DorisSchemaFactory.quoteComment(comment)).append("'");
+ }
+ }
+
+ public static String buildShowFullColumnDDL(String database, String table) {
+ return String.format(SHOW_FULL_COLUMN_DDL, database, table);
+ }
+
+ public static List getDdlSchemas() {
+ return ddlSchemas;
+ }
+
+ public static class DDLSchema {
+ private final String columnName;
+ private final boolean isDropColumn;
+
+ public DDLSchema(String columnName, boolean isDropColumn) {
+ this.columnName = columnName;
+ this.isDropColumn = isDropColumn;
+ }
+
+ public String getColumnName() {
+ return columnName;
+ }
+
+ public boolean isDropColumn() {
+ return isDropColumn;
+ }
+ }
+}
diff --git a/flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/sink/schema/SchemaChangeManager.java b/flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/sink/schema/SchemaChangeManager.java
new file mode 100644
index 000000000..50ec1d34a
--- /dev/null
+++ b/flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/sink/schema/SchemaChangeManager.java
@@ -0,0 +1,366 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.flink.sink.schema;
+
+import org.apache.flink.util.CollectionUtil;
+import org.apache.flink.util.StringUtils;
+
+import com.fasterxml.jackson.core.JsonProcessingException;
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.node.NullNode;
+import org.apache.commons.codec.binary.Base64;
+import org.apache.doris.flink.catalog.doris.DorisSystem;
+import org.apache.doris.flink.catalog.doris.FieldSchema;
+import org.apache.doris.flink.catalog.doris.TableSchema;
+import org.apache.doris.flink.cfg.DorisOptions;
+import org.apache.doris.flink.exception.DorisSchemaChangeException;
+import org.apache.doris.flink.exception.IllegalArgumentException;
+import org.apache.doris.flink.rest.RestService;
+import org.apache.doris.flink.rest.models.Field;
+import org.apache.doris.flink.rest.models.Schema;
+import org.apache.doris.flink.sink.HttpGetWithEntity;
+import org.apache.http.HttpHeaders;
+import org.apache.http.client.methods.CloseableHttpResponse;
+import org.apache.http.client.methods.HttpPost;
+import org.apache.http.client.methods.HttpUriRequest;
+import org.apache.http.entity.StringEntity;
+import org.apache.http.impl.client.CloseableHttpClient;
+import org.apache.http.impl.client.HttpClients;
+import org.apache.http.util.EntityUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.io.Serializable;
+import java.nio.charset.StandardCharsets;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Optional;
+
+public class SchemaChangeManager implements Serializable {
+ private static final long serialVersionUID = 1L;
+ private static final Logger LOG = LoggerFactory.getLogger(SchemaChangeManager.class);
+ private static final String CHECK_SCHEMA_CHANGE_API =
+ "http://%s/api/enable_light_schema_change/%s/%s";
+ private static final String SCHEMA_CHANGE_API = "http://%s/api/query/default_cluster/%s";
+ private ObjectMapper objectMapper = new ObjectMapper();
+ private DorisOptions dorisOptions;
+ private String charsetEncoding = "UTF-8";
+
+ public SchemaChangeManager(DorisOptions dorisOptions) {
+ this.dorisOptions = dorisOptions;
+ }
+
+ public SchemaChangeManager(DorisOptions dorisOptions, String charsetEncoding) {
+ this.dorisOptions = dorisOptions;
+ this.charsetEncoding = charsetEncoding;
+ }
+
+ public boolean createTable(TableSchema table) throws IOException, IllegalArgumentException {
+ // auto create database if not exists
+ if (!checkDatabaseExists(table.getDatabase())) {
+ execute(
+ SchemaChangeHelper.buildCreateDatabaseDDL(table.getDatabase()),
+ SchemaChangeHelper.DEFAULT_DATABASE);
+ }
+ String createTableDDL = DorisSystem.buildCreateTableDDL(table);
+ return execute(createTableDDL, table.getDatabase());
+ }
+
+ public boolean addColumn(String database, String table, FieldSchema field)
+ throws IOException, IllegalArgumentException {
+ if (checkColumnExists(database, table, field.getName())) {
+ LOG.warn(
+ "The column {} already exists in table {}, no need to add it again",
+ field.getName(),
+ table);
+ return true;
+ }
+ String tableIdentifier = getTableIdentifier(database, table);
+ String addColumnDDL = SchemaChangeHelper.buildAddColumnDDL(tableIdentifier, field);
+ return schemaChange(
+ database, table, buildRequestParam(false, field.getName()), addColumnDDL);
+ }
+
+ public boolean dropColumn(String database, String table, String columnName)
+ throws IOException, IllegalArgumentException {
+ if (!checkColumnExists(database, table, columnName)) {
+ LOG.warn("The column {} not exists in table {}, no need to drop", columnName, table);
+ return true;
+ }
+ String tableIdentifier = getTableIdentifier(database, table);
+ String dropColumnDDL = SchemaChangeHelper.buildDropColumnDDL(tableIdentifier, columnName);
+ return schemaChange(database, table, buildRequestParam(true, columnName), dropColumnDDL);
+ }
+
+ public boolean renameColumn(
+ String database, String table, String oldColumnName, String newColumnName)
+ throws IOException, IllegalArgumentException {
+ String tableIdentifier = getTableIdentifier(database, table);
+ String renameColumnDDL =
+ SchemaChangeHelper.buildRenameColumnDDL(
+ tableIdentifier, oldColumnName, newColumnName);
+ return schemaChange(
+ database, table, buildRequestParam(true, oldColumnName), renameColumnDDL);
+ }
+
+ public boolean modifyColumnDataType(String database, String table, FieldSchema field)
+ throws IOException, IllegalArgumentException {
+ if (!checkColumnExists(database, table, field.getName())) {
+ LOG.warn(
+ "The column {} is not exists in table {}, can not modify it's type",
+ field.getName(),
+ table);
+ return false;
+ }
+
+ String ddl = SchemaChangeHelper.buildShowFullColumnDDL(database, table);
+ String defaultValue = getDefaultValue(ddl, database, field.getName());
+ if (!StringUtils.isNullOrWhitespaceOnly(field.getDefaultValue())) {
+ // Can not change default value
+ if (!field.getDefaultValue().equals(defaultValue)) {
+ LOG.warn(
+ "Column:{} can not change default value from {} to {}, fallback it",
+ field.getName(),
+ defaultValue,
+ field.getDefaultValue());
+ field.setDefaultValue(defaultValue);
+ }
+ } else {
+ // If user does not give a default value, need fill it from
+ // original table schema to avoid change type failed if default value exists
+ field.setDefaultValue(defaultValue);
+ }
+
+ // If user does not give a comment, need fill it from
+ // original table schema to avoid miss comment
+ if (StringUtils.isNullOrWhitespaceOnly(field.getComment())) {
+ Schema tableSchema = getTableSchema(database, table);
+ Optional originalField =
+ tableSchema.getProperties().stream()
+ .filter(column -> column.getName().equals(field.getName()))
+ .findAny();
+ originalField.ifPresent(oldField -> field.setComment(oldField.getComment()));
+ }
+ String tableIdentifier = getTableIdentifier(database, table);
+ String modifyColumnDDL =
+ SchemaChangeHelper.buildModifyColumnDataTypeDDL(tableIdentifier, field);
+ return schemaChange(
+ database, table, buildRequestParam(false, field.getName()), modifyColumnDDL);
+ }
+
+ public boolean modifyColumnComment(
+ String database, String table, String columnName, String newComment)
+ throws IOException, IllegalArgumentException {
+ if (!checkColumnExists(database, table, columnName)) {
+ LOG.warn(
+ "The column {} is not exists in table {}, can not modify it's comment",
+ columnName,
+ table);
+ return false;
+ }
+ String tableIdentifier = getTableIdentifier(database, table);
+ String modifyColumnCommentDDL =
+ SchemaChangeHelper.buildModifyColumnCommentDDL(
+ tableIdentifier, columnName, newComment);
+ return schemaChange(
+ database, table, buildRequestParam(false, columnName), modifyColumnCommentDDL);
+ }
+
+ public Schema getTableSchema(String database, String table) {
+ return RestService.getSchema(dorisOptions, database, table, LOG);
+ }
+
+ public boolean schemaChange(
+ String database, String table, Map params, String sql)
+ throws IOException, IllegalArgumentException {
+ if (checkSchemaChange(database, table, params)) {
+ return execute(sql, database);
+ }
+ return false;
+ }
+
+ public static Map buildRequestParam(boolean dropColumn, String columnName) {
+ Map params = new HashMap<>();
+ params.put("isDropColumn", dropColumn);
+ params.put("columnName", columnName);
+ return params;
+ }
+
+ /** check ddl can do light schema change. */
+ public boolean checkSchemaChange(String database, String table, Map params)
+ throws IOException, IllegalArgumentException {
+ if (CollectionUtil.isNullOrEmpty(params)) {
+ return false;
+ }
+ String requestUrl =
+ String.format(
+ CHECK_SCHEMA_CHANGE_API,
+ RestService.randomEndpoint(dorisOptions.getFenodes(), LOG),
+ database,
+ table);
+ HttpGetWithEntity httpGet = new HttpGetWithEntity(requestUrl);
+ httpGet.setHeader(HttpHeaders.AUTHORIZATION, authHeader());
+ httpGet.setEntity(
+ new StringEntity(objectMapper.writeValueAsString(params), charsetEncoding));
+ String responseEntity = handleResponse(httpGet);
+ return handleSchemaChange(responseEntity);
+ }
+
+ private boolean handleSchemaChange(String responseEntity) throws JsonProcessingException {
+ Map responseMap = objectMapper.readValue(responseEntity, Map.class);
+ String code = responseMap.getOrDefault("code", "-1").toString();
+ if (code.equals("0")) {
+ return true;
+ } else {
+ throw new DorisSchemaChangeException(
+ "Failed to schemaChange, response: " + responseEntity);
+ }
+ }
+
+ private String getDefaultValue(String ddl, String database, String column)
+ throws IOException, IllegalArgumentException {
+ String responseEntity = executeThenReturnResponse(ddl, database);
+ JsonNode responseNode = objectMapper.readTree(responseEntity);
+ String code = responseNode.get("code").asText("-1");
+ if (code.equals("0")) {
+ JsonNode data = responseNode.get("data").get("data");
+ for (JsonNode node : data) {
+ if (node.get(0).asText().equals(column)) {
+ JsonNode defaultValueNode = node.get(5);
+ return (defaultValueNode instanceof NullNode)
+ ? null
+ : defaultValueNode.asText();
+ }
+ }
+ return null;
+ } else {
+ throw new DorisSchemaChangeException(
+ "Failed to get default value, response: " + responseEntity);
+ }
+ }
+
+ /** execute sql in doris. */
+ private String executeThenReturnResponse(String ddl, String database)
+ throws IOException, IllegalArgumentException {
+ if (StringUtils.isNullOrWhitespaceOnly(ddl)) {
+ throw new IllegalArgumentException("ddl can not be null or empty string!");
+ }
+ LOG.info("Execute SQL: {}", ddl);
+ HttpPost httpPost = buildHttpPost(ddl, database);
+ return handleResponse(httpPost);
+ }
+
+ public boolean execute(String ddl, String database)
+ throws IOException, IllegalArgumentException {
+ String responseEntity = executeThenReturnResponse(ddl, database);
+ return handleSchemaChange(responseEntity);
+ }
+
+ public HttpPost buildHttpPost(String ddl, String database)
+ throws IllegalArgumentException, IOException {
+ Map param = new HashMap<>();
+ param.put("stmt", ddl);
+ String requestUrl =
+ String.format(
+ SCHEMA_CHANGE_API,
+ RestService.randomEndpoint(dorisOptions.getFenodes(), LOG),
+ database);
+ HttpPost httpPost = new HttpPost(requestUrl);
+ httpPost.setHeader(HttpHeaders.AUTHORIZATION, authHeader());
+ httpPost.setHeader(
+ HttpHeaders.CONTENT_TYPE,
+ String.format("application/json;charset=%s", charsetEncoding));
+ httpPost.setEntity(
+ new StringEntity(objectMapper.writeValueAsString(param), charsetEncoding));
+ return httpPost;
+ }
+
+ private String handleResponse(HttpUriRequest request) {
+ try (CloseableHttpClient httpclient = HttpClients.createDefault()) {
+ CloseableHttpResponse response = httpclient.execute(request);
+ final int statusCode = response.getStatusLine().getStatusCode();
+ final String reasonPhrase = response.getStatusLine().getReasonPhrase();
+ if (statusCode == 200 && response.getEntity() != null) {
+ return EntityUtils.toString(response.getEntity());
+ } else {
+ throw new DorisSchemaChangeException(
+ "Failed to schemaChange, status: "
+ + statusCode
+ + ", reason: "
+ + reasonPhrase);
+ }
+ } catch (Exception e) {
+ LOG.error("SchemaChange request error,", e);
+ throw new DorisSchemaChangeException(
+ "SchemaChange request error with " + e.getMessage());
+ }
+ }
+
+ /** When processing a column, determine whether it exists and be idempotent. */
+ public boolean checkColumnExists(String database, String table, String columnName)
+ throws IllegalArgumentException, IOException {
+ String existsQuery = SchemaChangeHelper.buildColumnExistsQuery(database, table, columnName);
+ return sendHttpPostRequest(existsQuery, database);
+ }
+
+ private boolean checkDatabaseExists(String database)
+ throws IllegalArgumentException, IOException {
+ String existsQuery = SchemaChangeHelper.buildDatabaseExistsQuery(database);
+ return sendHttpPostRequest(existsQuery, SchemaChangeHelper.DEFAULT_DATABASE);
+ }
+
+ private boolean sendHttpPostRequest(String sql, String database)
+ throws IOException, IllegalArgumentException {
+ HttpPost httpPost = buildHttpPost(sql, database);
+ try (CloseableHttpClient httpclient = HttpClients.createDefault()) {
+ CloseableHttpResponse response = httpclient.execute(httpPost);
+ final int statusCode = response.getStatusLine().getStatusCode();
+ if (statusCode == 200 && response.getEntity() != null) {
+ String loadResult = EntityUtils.toString(response.getEntity());
+ JsonNode responseNode = objectMapper.readTree(loadResult);
+ String code = responseNode.get("code").asText("-1");
+ if (code.equals("0")) {
+ JsonNode data = responseNode.get("data").get("data");
+ if (!data.isEmpty()) {
+ return true;
+ }
+ }
+ }
+ } catch (Exception e) {
+ LOG.error(
+ "send http post request error {}, default return false, SQL:{}",
+ e.getMessage(),
+ sql);
+ }
+ return false;
+ }
+
+ private String authHeader() {
+ return "Basic "
+ + new String(
+ Base64.encodeBase64(
+ (dorisOptions.getUsername() + ":" + dorisOptions.getPassword())
+ .getBytes(StandardCharsets.UTF_8)));
+ }
+
+ private static String getTableIdentifier(String database, String table) {
+ return String.format("%s.%s", database, table);
+ }
+}
diff --git a/flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/sink/schema/SchemaChangeMode.java b/flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/sink/schema/SchemaChangeMode.java
new file mode 100644
index 000000000..e55a4d311
--- /dev/null
+++ b/flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/sink/schema/SchemaChangeMode.java
@@ -0,0 +1,33 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.flink.sink.schema;
+
+public enum SchemaChangeMode {
+ DEBEZIUM_STRUCTURE("debezium_structure"),
+ SQL_PARSER("sql_parser");
+
+ private final String name;
+
+ SchemaChangeMode(String name) {
+ this.name = name;
+ }
+
+ public String getName() {
+ return name;
+ }
+}
diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/sink/util/DeleteOperation.java b/flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/sink/util/DeleteOperation.java
similarity index 100%
rename from flink-doris-connector/src/main/java/org/apache/doris/flink/sink/util/DeleteOperation.java
rename to flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/sink/util/DeleteOperation.java
diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/sink/writer/CacheRecordBuffer.java b/flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/sink/writer/CacheRecordBuffer.java
similarity index 100%
rename from flink-doris-connector/src/main/java/org/apache/doris/flink/sink/writer/CacheRecordBuffer.java
rename to flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/sink/writer/CacheRecordBuffer.java
diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/sink/writer/ChangeEvent.java b/flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/sink/writer/ChangeEvent.java
similarity index 100%
rename from flink-doris-connector/src/main/java/org/apache/doris/flink/sink/writer/ChangeEvent.java
rename to flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/sink/writer/ChangeEvent.java
diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/sink/writer/DorisAbstractWriter.java b/flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/sink/writer/DorisAbstractWriter.java
similarity index 100%
rename from flink-doris-connector/src/main/java/org/apache/doris/flink/sink/writer/DorisAbstractWriter.java
rename to flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/sink/writer/DorisAbstractWriter.java
diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/sink/writer/DorisStreamLoad.java b/flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/sink/writer/DorisStreamLoad.java
similarity index 100%
rename from flink-doris-connector/src/main/java/org/apache/doris/flink/sink/writer/DorisStreamLoad.java
rename to flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/sink/writer/DorisStreamLoad.java
diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/sink/writer/DorisWriteMetrics.java b/flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/sink/writer/DorisWriteMetrics.java
similarity index 100%
rename from flink-doris-connector/src/main/java/org/apache/doris/flink/sink/writer/DorisWriteMetrics.java
rename to flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/sink/writer/DorisWriteMetrics.java
diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/sink/writer/DorisWriter.java b/flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/sink/writer/DorisWriter.java
similarity index 100%
rename from flink-doris-connector/src/main/java/org/apache/doris/flink/sink/writer/DorisWriter.java
rename to flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/sink/writer/DorisWriter.java
diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/sink/writer/DorisWriterState.java b/flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/sink/writer/DorisWriterState.java
similarity index 100%
rename from flink-doris-connector/src/main/java/org/apache/doris/flink/sink/writer/DorisWriterState.java
rename to flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/sink/writer/DorisWriterState.java
diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/sink/writer/DorisWriterStateSerializer.java b/flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/sink/writer/DorisWriterStateSerializer.java
similarity index 100%
rename from flink-doris-connector/src/main/java/org/apache/doris/flink/sink/writer/DorisWriterStateSerializer.java
rename to flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/sink/writer/DorisWriterStateSerializer.java
diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/sink/writer/EventType.java b/flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/sink/writer/EventType.java
similarity index 100%
rename from flink-doris-connector/src/main/java/org/apache/doris/flink/sink/writer/EventType.java
rename to flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/sink/writer/EventType.java
diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/sink/writer/LabelGenerator.java b/flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/sink/writer/LabelGenerator.java
similarity index 100%
rename from flink-doris-connector/src/main/java/org/apache/doris/flink/sink/writer/LabelGenerator.java
rename to flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/sink/writer/LabelGenerator.java
diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/sink/writer/LoadConstants.java b/flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/sink/writer/LoadConstants.java
similarity index 100%
rename from flink-doris-connector/src/main/java/org/apache/doris/flink/sink/writer/LoadConstants.java
rename to flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/sink/writer/LoadConstants.java
diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/sink/writer/RecordBuffer.java b/flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/sink/writer/RecordBuffer.java
similarity index 100%
rename from flink-doris-connector/src/main/java/org/apache/doris/flink/sink/writer/RecordBuffer.java
rename to flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/sink/writer/RecordBuffer.java
diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/sink/writer/RecordStream.java b/flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/sink/writer/RecordStream.java
similarity index 100%
rename from flink-doris-connector/src/main/java/org/apache/doris/flink/sink/writer/RecordStream.java
rename to flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/sink/writer/RecordStream.java
diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/sink/writer/WriteMode.java b/flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/sink/writer/WriteMode.java
similarity index 100%
rename from flink-doris-connector/src/main/java/org/apache/doris/flink/sink/writer/WriteMode.java
rename to flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/sink/writer/WriteMode.java
diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/sink/writer/serializer/DorisRecord.java b/flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/sink/writer/serializer/DorisRecord.java
similarity index 100%
rename from flink-doris-connector/src/main/java/org/apache/doris/flink/sink/writer/serializer/DorisRecord.java
rename to flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/sink/writer/serializer/DorisRecord.java
diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/sink/writer/serializer/DorisRecordSerializer.java b/flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/sink/writer/serializer/DorisRecordSerializer.java
similarity index 100%
rename from flink-doris-connector/src/main/java/org/apache/doris/flink/sink/writer/serializer/DorisRecordSerializer.java
rename to flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/sink/writer/serializer/DorisRecordSerializer.java
diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/sink/writer/serializer/JsonDebeziumSchemaSerializer.java b/flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/sink/writer/serializer/JsonDebeziumSchemaSerializer.java
similarity index 100%
rename from flink-doris-connector/src/main/java/org/apache/doris/flink/sink/writer/serializer/JsonDebeziumSchemaSerializer.java
rename to flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/sink/writer/serializer/JsonDebeziumSchemaSerializer.java
diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/sink/writer/serializer/RecordWithMetaSerializer.java b/flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/sink/writer/serializer/RecordWithMetaSerializer.java
similarity index 100%
rename from flink-doris-connector/src/main/java/org/apache/doris/flink/sink/writer/serializer/RecordWithMetaSerializer.java
rename to flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/sink/writer/serializer/RecordWithMetaSerializer.java
diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/sink/writer/serializer/RowDataSerializer.java b/flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/sink/writer/serializer/RowDataSerializer.java
similarity index 100%
rename from flink-doris-connector/src/main/java/org/apache/doris/flink/sink/writer/serializer/RowDataSerializer.java
rename to flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/sink/writer/serializer/RowDataSerializer.java
diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/sink/writer/serializer/RowSerializer.java b/flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/sink/writer/serializer/RowSerializer.java
similarity index 100%
rename from flink-doris-connector/src/main/java/org/apache/doris/flink/sink/writer/serializer/RowSerializer.java
rename to flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/sink/writer/serializer/RowSerializer.java
diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/sink/writer/serializer/SimpleStringSerializer.java b/flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/sink/writer/serializer/SimpleStringSerializer.java
similarity index 100%
rename from flink-doris-connector/src/main/java/org/apache/doris/flink/sink/writer/serializer/SimpleStringSerializer.java
rename to flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/sink/writer/serializer/SimpleStringSerializer.java
diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/sink/writer/serializer/jsondebezium/CdcDataChange.java b/flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/sink/writer/serializer/jsondebezium/CdcDataChange.java
similarity index 100%
rename from flink-doris-connector/src/main/java/org/apache/doris/flink/sink/writer/serializer/jsondebezium/CdcDataChange.java
rename to flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/sink/writer/serializer/jsondebezium/CdcDataChange.java
diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/sink/writer/serializer/jsondebezium/CdcSchemaChange.java b/flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/sink/writer/serializer/jsondebezium/CdcSchemaChange.java
similarity index 100%
rename from flink-doris-connector/src/main/java/org/apache/doris/flink/sink/writer/serializer/jsondebezium/CdcSchemaChange.java
rename to flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/sink/writer/serializer/jsondebezium/CdcSchemaChange.java
diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/sink/writer/serializer/jsondebezium/JsonDebeziumChangeContext.java b/flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/sink/writer/serializer/jsondebezium/JsonDebeziumChangeContext.java
similarity index 100%
rename from flink-doris-connector/src/main/java/org/apache/doris/flink/sink/writer/serializer/jsondebezium/JsonDebeziumChangeContext.java
rename to flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/sink/writer/serializer/jsondebezium/JsonDebeziumChangeContext.java
diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/sink/writer/serializer/jsondebezium/JsonDebeziumChangeUtils.java b/flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/sink/writer/serializer/jsondebezium/JsonDebeziumChangeUtils.java
similarity index 100%
rename from flink-doris-connector/src/main/java/org/apache/doris/flink/sink/writer/serializer/jsondebezium/JsonDebeziumChangeUtils.java
rename to flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/sink/writer/serializer/jsondebezium/JsonDebeziumChangeUtils.java
diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/sink/writer/serializer/jsondebezium/JsonDebeziumDataChange.java b/flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/sink/writer/serializer/jsondebezium/JsonDebeziumDataChange.java
similarity index 100%
rename from flink-doris-connector/src/main/java/org/apache/doris/flink/sink/writer/serializer/jsondebezium/JsonDebeziumDataChange.java
rename to flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/sink/writer/serializer/jsondebezium/JsonDebeziumDataChange.java
diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/sink/writer/serializer/jsondebezium/JsonDebeziumSchemaChange.java b/flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/sink/writer/serializer/jsondebezium/JsonDebeziumSchemaChange.java
similarity index 100%
rename from flink-doris-connector/src/main/java/org/apache/doris/flink/sink/writer/serializer/jsondebezium/JsonDebeziumSchemaChange.java
rename to flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/sink/writer/serializer/jsondebezium/JsonDebeziumSchemaChange.java
diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/sink/writer/serializer/jsondebezium/JsonDebeziumSchemaChangeImpl.java b/flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/sink/writer/serializer/jsondebezium/JsonDebeziumSchemaChangeImpl.java
similarity index 100%
rename from flink-doris-connector/src/main/java/org/apache/doris/flink/sink/writer/serializer/jsondebezium/JsonDebeziumSchemaChangeImpl.java
rename to flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/sink/writer/serializer/jsondebezium/JsonDebeziumSchemaChangeImpl.java
diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/sink/writer/serializer/jsondebezium/JsonDebeziumSchemaChangeImplV2.java b/flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/sink/writer/serializer/jsondebezium/JsonDebeziumSchemaChangeImplV2.java
similarity index 100%
rename from flink-doris-connector/src/main/java/org/apache/doris/flink/sink/writer/serializer/jsondebezium/JsonDebeziumSchemaChangeImplV2.java
rename to flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/sink/writer/serializer/jsondebezium/JsonDebeziumSchemaChangeImplV2.java
diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/sink/writer/serializer/jsondebezium/SQLParserSchemaChange.java b/flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/sink/writer/serializer/jsondebezium/SQLParserSchemaChange.java
similarity index 100%
rename from flink-doris-connector/src/main/java/org/apache/doris/flink/sink/writer/serializer/jsondebezium/SQLParserSchemaChange.java
rename to flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/sink/writer/serializer/jsondebezium/SQLParserSchemaChange.java
diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/source/DorisSource.java b/flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/source/DorisSource.java
similarity index 100%
rename from flink-doris-connector/src/main/java/org/apache/doris/flink/source/DorisSource.java
rename to flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/source/DorisSource.java
diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/source/assigners/DorisSplitAssigner.java b/flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/source/assigners/DorisSplitAssigner.java
similarity index 100%
rename from flink-doris-connector/src/main/java/org/apache/doris/flink/source/assigners/DorisSplitAssigner.java
rename to flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/source/assigners/DorisSplitAssigner.java
diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/source/assigners/SimpleSplitAssigner.java b/flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/source/assigners/SimpleSplitAssigner.java
similarity index 100%
rename from flink-doris-connector/src/main/java/org/apache/doris/flink/source/assigners/SimpleSplitAssigner.java
rename to flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/source/assigners/SimpleSplitAssigner.java
diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/source/enumerator/DorisSourceEnumerator.java b/flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/source/enumerator/DorisSourceEnumerator.java
similarity index 100%
rename from flink-doris-connector/src/main/java/org/apache/doris/flink/source/enumerator/DorisSourceEnumerator.java
rename to flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/source/enumerator/DorisSourceEnumerator.java
diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/source/enumerator/PendingSplitsCheckpoint.java b/flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/source/enumerator/PendingSplitsCheckpoint.java
similarity index 100%
rename from flink-doris-connector/src/main/java/org/apache/doris/flink/source/enumerator/PendingSplitsCheckpoint.java
rename to flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/source/enumerator/PendingSplitsCheckpoint.java
diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/source/enumerator/PendingSplitsCheckpointSerializer.java b/flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/source/enumerator/PendingSplitsCheckpointSerializer.java
similarity index 100%
rename from flink-doris-connector/src/main/java/org/apache/doris/flink/source/enumerator/PendingSplitsCheckpointSerializer.java
rename to flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/source/enumerator/PendingSplitsCheckpointSerializer.java
diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/source/reader/DorisFlightValueReader.java b/flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/source/reader/DorisFlightValueReader.java
similarity index 100%
rename from flink-doris-connector/src/main/java/org/apache/doris/flink/source/reader/DorisFlightValueReader.java
rename to flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/source/reader/DorisFlightValueReader.java
diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/source/reader/DorisRecordEmitter.java b/flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/source/reader/DorisRecordEmitter.java
similarity index 100%
rename from flink-doris-connector/src/main/java/org/apache/doris/flink/source/reader/DorisRecordEmitter.java
rename to flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/source/reader/DorisRecordEmitter.java
diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/source/reader/DorisSourceReader.java b/flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/source/reader/DorisSourceReader.java
similarity index 100%
rename from flink-doris-connector/src/main/java/org/apache/doris/flink/source/reader/DorisSourceReader.java
rename to flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/source/reader/DorisSourceReader.java
diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/source/reader/DorisSourceSplitReader.java b/flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/source/reader/DorisSourceSplitReader.java
similarity index 100%
rename from flink-doris-connector/src/main/java/org/apache/doris/flink/source/reader/DorisSourceSplitReader.java
rename to flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/source/reader/DorisSourceSplitReader.java
diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/source/reader/DorisValueReader.java b/flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/source/reader/DorisValueReader.java
similarity index 100%
rename from flink-doris-connector/src/main/java/org/apache/doris/flink/source/reader/DorisValueReader.java
rename to flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/source/reader/DorisValueReader.java
diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/source/reader/ValueReader.java b/flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/source/reader/ValueReader.java
similarity index 100%
rename from flink-doris-connector/src/main/java/org/apache/doris/flink/source/reader/ValueReader.java
rename to flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/source/reader/ValueReader.java
diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/source/split/DorisSourceSplit.java b/flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/source/split/DorisSourceSplit.java
similarity index 100%
rename from flink-doris-connector/src/main/java/org/apache/doris/flink/source/split/DorisSourceSplit.java
rename to flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/source/split/DorisSourceSplit.java
diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/source/split/DorisSourceSplitSerializer.java b/flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/source/split/DorisSourceSplitSerializer.java
similarity index 100%
rename from flink-doris-connector/src/main/java/org/apache/doris/flink/source/split/DorisSourceSplitSerializer.java
rename to flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/source/split/DorisSourceSplitSerializer.java
diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/source/split/DorisSourceSplitState.java b/flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/source/split/DorisSourceSplitState.java
similarity index 100%
rename from flink-doris-connector/src/main/java/org/apache/doris/flink/source/split/DorisSourceSplitState.java
rename to flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/source/split/DorisSourceSplitState.java
diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/source/split/DorisSplitRecords.java b/flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/source/split/DorisSplitRecords.java
similarity index 100%
rename from flink-doris-connector/src/main/java/org/apache/doris/flink/source/split/DorisSplitRecords.java
rename to flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/source/split/DorisSplitRecords.java
diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/table/DorisConfigOptions.java b/flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/table/DorisConfigOptions.java
similarity index 100%
rename from flink-doris-connector/src/main/java/org/apache/doris/flink/table/DorisConfigOptions.java
rename to flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/table/DorisConfigOptions.java
diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/table/DorisDynamicTableFactory.java b/flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/table/DorisDynamicTableFactory.java
similarity index 100%
rename from flink-doris-connector/src/main/java/org/apache/doris/flink/table/DorisDynamicTableFactory.java
rename to flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/table/DorisDynamicTableFactory.java
diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/table/DorisDynamicTableSink.java b/flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/table/DorisDynamicTableSink.java
similarity index 100%
rename from flink-doris-connector/src/main/java/org/apache/doris/flink/table/DorisDynamicTableSink.java
rename to flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/table/DorisDynamicTableSink.java
diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/table/DorisDynamicTableSource.java b/flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/table/DorisDynamicTableSource.java
similarity index 100%
rename from flink-doris-connector/src/main/java/org/apache/doris/flink/table/DorisDynamicTableSource.java
rename to flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/table/DorisDynamicTableSource.java
diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/table/DorisExpressionVisitor.java b/flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/table/DorisExpressionVisitor.java
similarity index 100%
rename from flink-doris-connector/src/main/java/org/apache/doris/flink/table/DorisExpressionVisitor.java
rename to flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/table/DorisExpressionVisitor.java
diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/table/DorisRowDataAsyncLookupFunction.java b/flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/table/DorisRowDataAsyncLookupFunction.java
similarity index 100%
rename from flink-doris-connector/src/main/java/org/apache/doris/flink/table/DorisRowDataAsyncLookupFunction.java
rename to flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/table/DorisRowDataAsyncLookupFunction.java
diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/table/DorisRowDataInputFormat.java b/flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/table/DorisRowDataInputFormat.java
similarity index 100%
rename from flink-doris-connector/src/main/java/org/apache/doris/flink/table/DorisRowDataInputFormat.java
rename to flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/table/DorisRowDataInputFormat.java
diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/table/DorisRowDataJdbcLookupFunction.java b/flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/table/DorisRowDataJdbcLookupFunction.java
similarity index 100%
rename from flink-doris-connector/src/main/java/org/apache/doris/flink/table/DorisRowDataJdbcLookupFunction.java
rename to flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/table/DorisRowDataJdbcLookupFunction.java
diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/table/DorisTableInputSplit.java b/flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/table/DorisTableInputSplit.java
similarity index 100%
rename from flink-doris-connector/src/main/java/org/apache/doris/flink/table/DorisTableInputSplit.java
rename to flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/table/DorisTableInputSplit.java
diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/tools/cdc/DorisTableConfig.java b/flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/tools/cdc/DorisTableConfig.java
similarity index 100%
rename from flink-doris-connector/src/main/java/org/apache/doris/flink/tools/cdc/DorisTableConfig.java
rename to flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/tools/cdc/DorisTableConfig.java
diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/tools/cdc/SourceConnector.java b/flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/tools/cdc/SourceConnector.java
similarity index 100%
rename from flink-doris-connector/src/main/java/org/apache/doris/flink/tools/cdc/SourceConnector.java
rename to flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/tools/cdc/SourceConnector.java
diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/tools/cdc/SourceSchema.java b/flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/tools/cdc/SourceSchema.java
similarity index 100%
rename from flink-doris-connector/src/main/java/org/apache/doris/flink/tools/cdc/SourceSchema.java
rename to flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/tools/cdc/SourceSchema.java
diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/tools/cdc/converter/TableNameConverter.java b/flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/tools/cdc/converter/TableNameConverter.java
similarity index 100%
rename from flink-doris-connector/src/main/java/org/apache/doris/flink/tools/cdc/converter/TableNameConverter.java
rename to flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/tools/cdc/converter/TableNameConverter.java
diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/tools/cdc/db2/Db2Type.java b/flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/tools/cdc/db2/Db2Type.java
similarity index 99%
rename from flink-doris-connector/src/main/java/org/apache/doris/flink/tools/cdc/db2/Db2Type.java
rename to flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/tools/cdc/db2/Db2Type.java
index 1255d1e7c..8668256d2 100644
--- a/flink-doris-connector/src/main/java/org/apache/doris/flink/tools/cdc/db2/Db2Type.java
+++ b/flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/tools/cdc/db2/Db2Type.java
@@ -17,9 +17,8 @@
package org.apache.doris.flink.tools.cdc.db2;
-import org.apache.flink.util.Preconditions;
-
import org.apache.doris.flink.catalog.doris.DorisType;
+import org.apache.flink.util.Preconditions;
public class Db2Type {
private static final String BOOLEAN = "BOOLEAN";
diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/tools/cdc/mysql/MysqlType.java b/flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/tools/cdc/mysql/MysqlType.java
similarity index 99%
rename from flink-doris-connector/src/main/java/org/apache/doris/flink/tools/cdc/mysql/MysqlType.java
rename to flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/tools/cdc/mysql/MysqlType.java
index 49afe3751..3bbc2a2eb 100644
--- a/flink-doris-connector/src/main/java/org/apache/doris/flink/tools/cdc/mysql/MysqlType.java
+++ b/flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/tools/cdc/mysql/MysqlType.java
@@ -17,12 +17,10 @@
package org.apache.doris.flink.tools.cdc.mysql;
-import org.apache.flink.table.types.logical.TimestampType;
-import org.apache.flink.util.Preconditions;
-
import org.apache.doris.flink.catalog.doris.DorisType;
-
import static org.apache.doris.flink.catalog.DorisTypeMapper.MAX_SUPPORTED_DATE_TIME_PRECISION;
+import org.apache.flink.table.types.logical.TimestampType;
+import org.apache.flink.util.Preconditions;
public class MysqlType {
diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/tools/cdc/oracle/OracleType.java b/flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/tools/cdc/oracle/OracleType.java
similarity index 99%
rename from flink-doris-connector/src/main/java/org/apache/doris/flink/tools/cdc/oracle/OracleType.java
rename to flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/tools/cdc/oracle/OracleType.java
index 304f21fd0..dfd3babf5 100644
--- a/flink-doris-connector/src/main/java/org/apache/doris/flink/tools/cdc/oracle/OracleType.java
+++ b/flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/tools/cdc/oracle/OracleType.java
@@ -17,9 +17,8 @@
package org.apache.doris.flink.tools.cdc.oracle;
-import org.apache.flink.util.Preconditions;
-
import org.apache.doris.flink.catalog.doris.DorisType;
+import org.apache.flink.util.Preconditions;
public class OracleType {
private static final String VARCHAR2 = "VARCHAR2";
diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/tools/cdc/postgres/PostgresType.java b/flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/tools/cdc/postgres/PostgresType.java
similarity index 99%
rename from flink-doris-connector/src/main/java/org/apache/doris/flink/tools/cdc/postgres/PostgresType.java
rename to flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/tools/cdc/postgres/PostgresType.java
index ddffb6d9f..9127e54aa 100644
--- a/flink-doris-connector/src/main/java/org/apache/doris/flink/tools/cdc/postgres/PostgresType.java
+++ b/flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/tools/cdc/postgres/PostgresType.java
@@ -17,9 +17,8 @@
package org.apache.doris.flink.tools.cdc.postgres;
-import org.apache.flink.util.Preconditions;
-
import org.apache.doris.flink.catalog.doris.DorisType;
+import org.apache.flink.util.Preconditions;
public class PostgresType {
private static final String INT2 = "int2";
diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/tools/cdc/sqlserver/SqlServerType.java b/flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/tools/cdc/sqlserver/SqlServerType.java
similarity index 100%
rename from flink-doris-connector/src/main/java/org/apache/doris/flink/tools/cdc/sqlserver/SqlServerType.java
rename to flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/tools/cdc/sqlserver/SqlServerType.java
diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/util/ErrorMessages.java b/flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/util/ErrorMessages.java
similarity index 100%
rename from flink-doris-connector/src/main/java/org/apache/doris/flink/util/ErrorMessages.java
rename to flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/util/ErrorMessages.java
diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/util/FastDateUtil.java b/flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/util/FastDateUtil.java
similarity index 100%
rename from flink-doris-connector/src/main/java/org/apache/doris/flink/util/FastDateUtil.java
rename to flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/util/FastDateUtil.java
diff --git a/flink-doris-connector/src/main/java/org/apache/doris/flink/util/IPUtils.java b/flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/util/IPUtils.java
similarity index 100%
rename from flink-doris-connector/src/main/java/org/apache/doris/flink/util/IPUtils.java
rename to flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/util/IPUtils.java
diff --git a/flink-doris-connector/src/main/java/org/apache/flink/table/runtime/arrow/serializers/ArrowSerializer.java b/flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/flink/table/runtime/arrow/serializers/ArrowSerializer.java
similarity index 100%
rename from flink-doris-connector/src/main/java/org/apache/flink/table/runtime/arrow/serializers/ArrowSerializer.java
rename to flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/flink/table/runtime/arrow/serializers/ArrowSerializer.java
diff --git a/flink-doris-connector/flink-doris-connector-base/src/main/resources/META-INF/services/org.apache.flink.table.factories.Factory b/flink-doris-connector/flink-doris-connector-base/src/main/resources/META-INF/services/org.apache.flink.table.factories.Factory
new file mode 100644
index 000000000..5863c8bf2
--- /dev/null
+++ b/flink-doris-connector/flink-doris-connector-base/src/main/resources/META-INF/services/org.apache.flink.table.factories.Factory
@@ -0,0 +1,19 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+org.apache.doris.flink.table.DorisDynamicTableFactory
+org.apache.doris.flink.catalog.DorisCatalogFactory
\ No newline at end of file
diff --git a/flink-doris-connector/flink-doris-connector-base/src/main/resources/log4j2.properties b/flink-doris-connector/flink-doris-connector-base/src/main/resources/log4j2.properties
new file mode 100644
index 000000000..591598b95
--- /dev/null
+++ b/flink-doris-connector/flink-doris-connector-base/src/main/resources/log4j2.properties
@@ -0,0 +1,25 @@
+################################################################################
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+################################################################################
+
+rootLogger.level = INFO
+rootLogger.appenderRef.console.ref = ConsoleAppender
+
+appender.console.name = ConsoleAppender
+appender.console.type = CONSOLE
+appender.console.layout.type = PatternLayout
+appender.console.layout.pattern = %d{HH:mm:ss,SSS} %-5p %-60c [%t] %x - %m%n
diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/backend/BackendClientTest.java b/flink-doris-connector/flink-doris-connector-base/src/test/java/org/apache/doris/flink/backend/BackendClientTest.java
similarity index 100%
rename from flink-doris-connector/src/test/java/org/apache/doris/flink/backend/BackendClientTest.java
rename to flink-doris-connector/flink-doris-connector-base/src/test/java/org/apache/doris/flink/backend/BackendClientTest.java
diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/catalog/DorisCatalogFactoryTest.java b/flink-doris-connector/flink-doris-connector-base/src/test/java/org/apache/doris/flink/catalog/DorisCatalogFactoryTest.java
similarity index 100%
rename from flink-doris-connector/src/test/java/org/apache/doris/flink/catalog/DorisCatalogFactoryTest.java
rename to flink-doris-connector/flink-doris-connector-base/src/test/java/org/apache/doris/flink/catalog/DorisCatalogFactoryTest.java
diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/catalog/DorisCatalogITCase.java b/flink-doris-connector/flink-doris-connector-base/src/test/java/org/apache/doris/flink/catalog/DorisCatalogITCase.java
similarity index 100%
rename from flink-doris-connector/src/test/java/org/apache/doris/flink/catalog/DorisCatalogITCase.java
rename to flink-doris-connector/flink-doris-connector-base/src/test/java/org/apache/doris/flink/catalog/DorisCatalogITCase.java
diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/catalog/DorisCatalogOptionsTest.java b/flink-doris-connector/flink-doris-connector-base/src/test/java/org/apache/doris/flink/catalog/DorisCatalogOptionsTest.java
similarity index 100%
rename from flink-doris-connector/src/test/java/org/apache/doris/flink/catalog/DorisCatalogOptionsTest.java
rename to flink-doris-connector/flink-doris-connector-base/src/test/java/org/apache/doris/flink/catalog/DorisCatalogOptionsTest.java
diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/catalog/DorisCatalogTest.java b/flink-doris-connector/flink-doris-connector-base/src/test/java/org/apache/doris/flink/catalog/DorisCatalogTest.java
similarity index 100%
rename from flink-doris-connector/src/test/java/org/apache/doris/flink/catalog/DorisCatalogTest.java
rename to flink-doris-connector/flink-doris-connector-base/src/test/java/org/apache/doris/flink/catalog/DorisCatalogTest.java
diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/catalog/DorisTypeMapperTest.java b/flink-doris-connector/flink-doris-connector-base/src/test/java/org/apache/doris/flink/catalog/DorisTypeMapperTest.java
similarity index 100%
rename from flink-doris-connector/src/test/java/org/apache/doris/flink/catalog/DorisTypeMapperTest.java
rename to flink-doris-connector/flink-doris-connector-base/src/test/java/org/apache/doris/flink/catalog/DorisTypeMapperTest.java
diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/catalog/doris/DorisSchemaFactoryTest.java b/flink-doris-connector/flink-doris-connector-base/src/test/java/org/apache/doris/flink/catalog/doris/DorisSchemaFactoryTest.java
similarity index 100%
rename from flink-doris-connector/src/test/java/org/apache/doris/flink/catalog/doris/DorisSchemaFactoryTest.java
rename to flink-doris-connector/flink-doris-connector-base/src/test/java/org/apache/doris/flink/catalog/doris/DorisSchemaFactoryTest.java
diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/catalog/doris/DorisSystemTest.java b/flink-doris-connector/flink-doris-connector-base/src/test/java/org/apache/doris/flink/catalog/doris/DorisSystemTest.java
similarity index 100%
rename from flink-doris-connector/src/test/java/org/apache/doris/flink/catalog/doris/DorisSystemTest.java
rename to flink-doris-connector/flink-doris-connector-base/src/test/java/org/apache/doris/flink/catalog/doris/DorisSystemTest.java
diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/cfg/DorisExecutionOptionsTest.java b/flink-doris-connector/flink-doris-connector-base/src/test/java/org/apache/doris/flink/cfg/DorisExecutionOptionsTest.java
similarity index 100%
rename from flink-doris-connector/src/test/java/org/apache/doris/flink/cfg/DorisExecutionOptionsTest.java
rename to flink-doris-connector/flink-doris-connector-base/src/test/java/org/apache/doris/flink/cfg/DorisExecutionOptionsTest.java
diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/cfg/DorisLookupOptionsTest.java b/flink-doris-connector/flink-doris-connector-base/src/test/java/org/apache/doris/flink/cfg/DorisLookupOptionsTest.java
similarity index 100%
rename from flink-doris-connector/src/test/java/org/apache/doris/flink/cfg/DorisLookupOptionsTest.java
rename to flink-doris-connector/flink-doris-connector-base/src/test/java/org/apache/doris/flink/cfg/DorisLookupOptionsTest.java
diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/cfg/DorisOptionsTest.java b/flink-doris-connector/flink-doris-connector-base/src/test/java/org/apache/doris/flink/cfg/DorisOptionsTest.java
similarity index 100%
rename from flink-doris-connector/src/test/java/org/apache/doris/flink/cfg/DorisOptionsTest.java
rename to flink-doris-connector/flink-doris-connector-base/src/test/java/org/apache/doris/flink/cfg/DorisOptionsTest.java
diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/cfg/DorisReadOptionsTest.java b/flink-doris-connector/flink-doris-connector-base/src/test/java/org/apache/doris/flink/cfg/DorisReadOptionsTest.java
similarity index 100%
rename from flink-doris-connector/src/test/java/org/apache/doris/flink/cfg/DorisReadOptionsTest.java
rename to flink-doris-connector/flink-doris-connector-base/src/test/java/org/apache/doris/flink/cfg/DorisReadOptionsTest.java
diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/connection/SimpleJdbcConnectionProviderTest.java b/flink-doris-connector/flink-doris-connector-base/src/test/java/org/apache/doris/flink/connection/SimpleJdbcConnectionProviderTest.java
similarity index 100%
rename from flink-doris-connector/src/test/java/org/apache/doris/flink/connection/SimpleJdbcConnectionProviderTest.java
rename to flink-doris-connector/flink-doris-connector-base/src/test/java/org/apache/doris/flink/connection/SimpleJdbcConnectionProviderTest.java
diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/container/AbstractContainerTestBase.java b/flink-doris-connector/flink-doris-connector-base/src/test/java/org/apache/doris/flink/container/AbstractContainerTestBase.java
similarity index 100%
rename from flink-doris-connector/src/test/java/org/apache/doris/flink/container/AbstractContainerTestBase.java
rename to flink-doris-connector/flink-doris-connector-base/src/test/java/org/apache/doris/flink/container/AbstractContainerTestBase.java
diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/container/AbstractITCaseService.java b/flink-doris-connector/flink-doris-connector-base/src/test/java/org/apache/doris/flink/container/AbstractITCaseService.java
similarity index 100%
rename from flink-doris-connector/src/test/java/org/apache/doris/flink/container/AbstractITCaseService.java
rename to flink-doris-connector/flink-doris-connector-base/src/test/java/org/apache/doris/flink/container/AbstractITCaseService.java
diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/container/ContainerUtils.java b/flink-doris-connector/flink-doris-connector-base/src/test/java/org/apache/doris/flink/container/ContainerUtils.java
similarity index 100%
rename from flink-doris-connector/src/test/java/org/apache/doris/flink/container/ContainerUtils.java
rename to flink-doris-connector/flink-doris-connector-base/src/test/java/org/apache/doris/flink/container/ContainerUtils.java
diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/container/e2e/Doris2DorisE2ECase.java b/flink-doris-connector/flink-doris-connector-base/src/test/java/org/apache/doris/flink/container/e2e/Doris2DorisE2ECase.java
similarity index 100%
rename from flink-doris-connector/src/test/java/org/apache/doris/flink/container/e2e/Doris2DorisE2ECase.java
rename to flink-doris-connector/flink-doris-connector-base/src/test/java/org/apache/doris/flink/container/e2e/Doris2DorisE2ECase.java
diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/container/instance/ContainerService.java b/flink-doris-connector/flink-doris-connector-base/src/test/java/org/apache/doris/flink/container/instance/ContainerService.java
similarity index 100%
rename from flink-doris-connector/src/test/java/org/apache/doris/flink/container/instance/ContainerService.java
rename to flink-doris-connector/flink-doris-connector-base/src/test/java/org/apache/doris/flink/container/instance/ContainerService.java
diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/container/instance/DorisContainer.java b/flink-doris-connector/flink-doris-connector-base/src/test/java/org/apache/doris/flink/container/instance/DorisContainer.java
similarity index 100%
rename from flink-doris-connector/src/test/java/org/apache/doris/flink/container/instance/DorisContainer.java
rename to flink-doris-connector/flink-doris-connector-base/src/test/java/org/apache/doris/flink/container/instance/DorisContainer.java
diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/container/instance/DorisCustomerContainer.java b/flink-doris-connector/flink-doris-connector-base/src/test/java/org/apache/doris/flink/container/instance/DorisCustomerContainer.java
similarity index 100%
rename from flink-doris-connector/src/test/java/org/apache/doris/flink/container/instance/DorisCustomerContainer.java
rename to flink-doris-connector/flink-doris-connector-base/src/test/java/org/apache/doris/flink/container/instance/DorisCustomerContainer.java
diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/container/instance/MySQLContainer.java b/flink-doris-connector/flink-doris-connector-base/src/test/java/org/apache/doris/flink/container/instance/MySQLContainer.java
similarity index 100%
rename from flink-doris-connector/src/test/java/org/apache/doris/flink/container/instance/MySQLContainer.java
rename to flink-doris-connector/flink-doris-connector-base/src/test/java/org/apache/doris/flink/container/instance/MySQLContainer.java
diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/deserialization/RowDataDeserializationSchemaTest.java b/flink-doris-connector/flink-doris-connector-base/src/test/java/org/apache/doris/flink/deserialization/RowDataDeserializationSchemaTest.java
similarity index 100%
rename from flink-doris-connector/src/test/java/org/apache/doris/flink/deserialization/RowDataDeserializationSchemaTest.java
rename to flink-doris-connector/flink-doris-connector-base/src/test/java/org/apache/doris/flink/deserialization/RowDataDeserializationSchemaTest.java
diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/deserialization/convert/DorisRowConverterTest.java b/flink-doris-connector/flink-doris-connector-base/src/test/java/org/apache/doris/flink/deserialization/convert/DorisRowConverterTest.java
similarity index 100%
rename from flink-doris-connector/src/test/java/org/apache/doris/flink/deserialization/convert/DorisRowConverterTest.java
rename to flink-doris-connector/flink-doris-connector-base/src/test/java/org/apache/doris/flink/deserialization/convert/DorisRowConverterTest.java
diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/example/CatalogExample.java b/flink-doris-connector/flink-doris-connector-base/src/test/java/org/apache/doris/flink/example/CatalogExample.java
similarity index 100%
rename from flink-doris-connector/src/test/java/org/apache/doris/flink/example/CatalogExample.java
rename to flink-doris-connector/flink-doris-connector-base/src/test/java/org/apache/doris/flink/example/CatalogExample.java
diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/example/DataGen2DorisExample.java b/flink-doris-connector/flink-doris-connector-base/src/test/java/org/apache/doris/flink/example/DataGen2DorisExample.java
similarity index 100%
rename from flink-doris-connector/src/test/java/org/apache/doris/flink/example/DataGen2DorisExample.java
rename to flink-doris-connector/flink-doris-connector-base/src/test/java/org/apache/doris/flink/example/DataGen2DorisExample.java
diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/example/DorisDateAndTimestampSqlTest.java b/flink-doris-connector/flink-doris-connector-base/src/test/java/org/apache/doris/flink/example/DorisDateAndTimestampSqlTest.java
similarity index 100%
rename from flink-doris-connector/src/test/java/org/apache/doris/flink/example/DorisDateAndTimestampSqlTest.java
rename to flink-doris-connector/flink-doris-connector-base/src/test/java/org/apache/doris/flink/example/DorisDateAndTimestampSqlTest.java
diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/example/DorisIntranetAccessSinkExample.java b/flink-doris-connector/flink-doris-connector-base/src/test/java/org/apache/doris/flink/example/DorisIntranetAccessSinkExample.java
similarity index 100%
rename from flink-doris-connector/src/test/java/org/apache/doris/flink/example/DorisIntranetAccessSinkExample.java
rename to flink-doris-connector/flink-doris-connector-base/src/test/java/org/apache/doris/flink/example/DorisIntranetAccessSinkExample.java
diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/example/DorisSinkArraySQLExample.java b/flink-doris-connector/flink-doris-connector-base/src/test/java/org/apache/doris/flink/example/DorisSinkArraySQLExample.java
similarity index 100%
rename from flink-doris-connector/src/test/java/org/apache/doris/flink/example/DorisSinkArraySQLExample.java
rename to flink-doris-connector/flink-doris-connector-base/src/test/java/org/apache/doris/flink/example/DorisSinkArraySQLExample.java
diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/example/DorisSinkBatchExample.java b/flink-doris-connector/flink-doris-connector-base/src/test/java/org/apache/doris/flink/example/DorisSinkBatchExample.java
similarity index 100%
rename from flink-doris-connector/src/test/java/org/apache/doris/flink/example/DorisSinkBatchExample.java
rename to flink-doris-connector/flink-doris-connector-base/src/test/java/org/apache/doris/flink/example/DorisSinkBatchExample.java
diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/example/DorisSinkExample.java b/flink-doris-connector/flink-doris-connector-base/src/test/java/org/apache/doris/flink/example/DorisSinkExample.java
similarity index 100%
rename from flink-doris-connector/src/test/java/org/apache/doris/flink/example/DorisSinkExample.java
rename to flink-doris-connector/flink-doris-connector-base/src/test/java/org/apache/doris/flink/example/DorisSinkExample.java
diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/example/DorisSinkExampleRowData.java b/flink-doris-connector/flink-doris-connector-base/src/test/java/org/apache/doris/flink/example/DorisSinkExampleRowData.java
similarity index 100%
rename from flink-doris-connector/src/test/java/org/apache/doris/flink/example/DorisSinkExampleRowData.java
rename to flink-doris-connector/flink-doris-connector-base/src/test/java/org/apache/doris/flink/example/DorisSinkExampleRowData.java
diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/example/DorisSinkMultiTableExample.java b/flink-doris-connector/flink-doris-connector-base/src/test/java/org/apache/doris/flink/example/DorisSinkMultiTableExample.java
similarity index 100%
rename from flink-doris-connector/src/test/java/org/apache/doris/flink/example/DorisSinkMultiTableExample.java
rename to flink-doris-connector/flink-doris-connector-base/src/test/java/org/apache/doris/flink/example/DorisSinkMultiTableExample.java
diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/example/DorisSinkSQLExample.java b/flink-doris-connector/flink-doris-connector-base/src/test/java/org/apache/doris/flink/example/DorisSinkSQLExample.java
similarity index 100%
rename from flink-doris-connector/src/test/java/org/apache/doris/flink/example/DorisSinkSQLExample.java
rename to flink-doris-connector/flink-doris-connector-base/src/test/java/org/apache/doris/flink/example/DorisSinkSQLExample.java
diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/example/DorisSinkStreamMultiTableExample.java b/flink-doris-connector/flink-doris-connector-base/src/test/java/org/apache/doris/flink/example/DorisSinkStreamMultiTableExample.java
similarity index 100%
rename from flink-doris-connector/src/test/java/org/apache/doris/flink/example/DorisSinkStreamMultiTableExample.java
rename to flink-doris-connector/flink-doris-connector-base/src/test/java/org/apache/doris/flink/example/DorisSinkStreamMultiTableExample.java
diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/example/DorisSourceExample.java b/flink-doris-connector/flink-doris-connector-base/src/test/java/org/apache/doris/flink/example/DorisSourceExample.java
similarity index 100%
rename from flink-doris-connector/src/test/java/org/apache/doris/flink/example/DorisSourceExample.java
rename to flink-doris-connector/flink-doris-connector-base/src/test/java/org/apache/doris/flink/example/DorisSourceExample.java
diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/example/DorisSourceSinkExample.java b/flink-doris-connector/flink-doris-connector-base/src/test/java/org/apache/doris/flink/example/DorisSourceSinkExample.java
similarity index 100%
rename from flink-doris-connector/src/test/java/org/apache/doris/flink/example/DorisSourceSinkExample.java
rename to flink-doris-connector/flink-doris-connector-base/src/test/java/org/apache/doris/flink/example/DorisSourceSinkExample.java
diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/example/LookupJoinCdcExample.java b/flink-doris-connector/flink-doris-connector-base/src/test/java/org/apache/doris/flink/example/LookupJoinCdcExample.java
similarity index 100%
rename from flink-doris-connector/src/test/java/org/apache/doris/flink/example/LookupJoinCdcExample.java
rename to flink-doris-connector/flink-doris-connector-base/src/test/java/org/apache/doris/flink/example/LookupJoinCdcExample.java
diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/example/LookupJoinExample.java b/flink-doris-connector/flink-doris-connector-base/src/test/java/org/apache/doris/flink/example/LookupJoinExample.java
similarity index 100%
rename from flink-doris-connector/src/test/java/org/apache/doris/flink/example/LookupJoinExample.java
rename to flink-doris-connector/flink-doris-connector-base/src/test/java/org/apache/doris/flink/example/LookupJoinExample.java
diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/lookup/DorisLookupTableITCase.java b/flink-doris-connector/flink-doris-connector-base/src/test/java/org/apache/doris/flink/lookup/DorisLookupTableITCase.java
similarity index 100%
rename from flink-doris-connector/src/test/java/org/apache/doris/flink/lookup/DorisLookupTableITCase.java
rename to flink-doris-connector/flink-doris-connector-base/src/test/java/org/apache/doris/flink/lookup/DorisLookupTableITCase.java
diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/lookup/RecordTest.java b/flink-doris-connector/flink-doris-connector-base/src/test/java/org/apache/doris/flink/lookup/RecordTest.java
similarity index 100%
rename from flink-doris-connector/src/test/java/org/apache/doris/flink/lookup/RecordTest.java
rename to flink-doris-connector/flink-doris-connector-base/src/test/java/org/apache/doris/flink/lookup/RecordTest.java
diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/rest/SchemaUtilsTest.java b/flink-doris-connector/flink-doris-connector-base/src/test/java/org/apache/doris/flink/rest/SchemaUtilsTest.java
similarity index 100%
rename from flink-doris-connector/src/test/java/org/apache/doris/flink/rest/SchemaUtilsTest.java
rename to flink-doris-connector/flink-doris-connector-base/src/test/java/org/apache/doris/flink/rest/SchemaUtilsTest.java
diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/rest/TestPartitionDefinition.java b/flink-doris-connector/flink-doris-connector-base/src/test/java/org/apache/doris/flink/rest/TestPartitionDefinition.java
similarity index 100%
rename from flink-doris-connector/src/test/java/org/apache/doris/flink/rest/TestPartitionDefinition.java
rename to flink-doris-connector/flink-doris-connector-base/src/test/java/org/apache/doris/flink/rest/TestPartitionDefinition.java
diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/rest/TestRestService.java b/flink-doris-connector/flink-doris-connector-base/src/test/java/org/apache/doris/flink/rest/TestRestService.java
similarity index 100%
rename from flink-doris-connector/src/test/java/org/apache/doris/flink/rest/TestRestService.java
rename to flink-doris-connector/flink-doris-connector-base/src/test/java/org/apache/doris/flink/rest/TestRestService.java
diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/rest/models/TestSchema.java b/flink-doris-connector/flink-doris-connector-base/src/test/java/org/apache/doris/flink/rest/models/TestSchema.java
similarity index 100%
rename from flink-doris-connector/src/test/java/org/apache/doris/flink/rest/models/TestSchema.java
rename to flink-doris-connector/flink-doris-connector-base/src/test/java/org/apache/doris/flink/rest/models/TestSchema.java
diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/serialization/TestRouting.java b/flink-doris-connector/flink-doris-connector-base/src/test/java/org/apache/doris/flink/serialization/TestRouting.java
similarity index 100%
rename from flink-doris-connector/src/test/java/org/apache/doris/flink/serialization/TestRouting.java
rename to flink-doris-connector/flink-doris-connector-base/src/test/java/org/apache/doris/flink/serialization/TestRouting.java
diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/serialization/TestRowBatch.java b/flink-doris-connector/flink-doris-connector-base/src/test/java/org/apache/doris/flink/serialization/TestRowBatch.java
similarity index 100%
rename from flink-doris-connector/src/test/java/org/apache/doris/flink/serialization/TestRowBatch.java
rename to flink-doris-connector/flink-doris-connector-base/src/test/java/org/apache/doris/flink/serialization/TestRowBatch.java
diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/sink/DorisSinkFailoverITCase.java b/flink-doris-connector/flink-doris-connector-base/src/test/java/org/apache/doris/flink/sink/DorisSinkFailoverITCase.java
similarity index 100%
rename from flink-doris-connector/src/test/java/org/apache/doris/flink/sink/DorisSinkFailoverITCase.java
rename to flink-doris-connector/flink-doris-connector-base/src/test/java/org/apache/doris/flink/sink/DorisSinkFailoverITCase.java
diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/sink/DorisSinkITCase.java b/flink-doris-connector/flink-doris-connector-base/src/test/java/org/apache/doris/flink/sink/DorisSinkITCase.java
similarity index 100%
rename from flink-doris-connector/src/test/java/org/apache/doris/flink/sink/DorisSinkITCase.java
rename to flink-doris-connector/flink-doris-connector-base/src/test/java/org/apache/doris/flink/sink/DorisSinkITCase.java
diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/sink/DorisSinkMultiTblFailoverITCase.java b/flink-doris-connector/flink-doris-connector-base/src/test/java/org/apache/doris/flink/sink/DorisSinkMultiTblFailoverITCase.java
similarity index 100%
rename from flink-doris-connector/src/test/java/org/apache/doris/flink/sink/DorisSinkMultiTblFailoverITCase.java
rename to flink-doris-connector/flink-doris-connector-base/src/test/java/org/apache/doris/flink/sink/DorisSinkMultiTblFailoverITCase.java
diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/sink/DorisSinkTest.java b/flink-doris-connector/flink-doris-connector-base/src/test/java/org/apache/doris/flink/sink/DorisSinkTest.java
similarity index 100%
rename from flink-doris-connector/src/test/java/org/apache/doris/flink/sink/DorisSinkTest.java
rename to flink-doris-connector/flink-doris-connector-base/src/test/java/org/apache/doris/flink/sink/DorisSinkTest.java
diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/sink/HttpEntityMock.java b/flink-doris-connector/flink-doris-connector-base/src/test/java/org/apache/doris/flink/sink/HttpEntityMock.java
similarity index 100%
rename from flink-doris-connector/src/test/java/org/apache/doris/flink/sink/HttpEntityMock.java
rename to flink-doris-connector/flink-doris-connector-base/src/test/java/org/apache/doris/flink/sink/HttpEntityMock.java
diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/sink/HttpTestUtil.java b/flink-doris-connector/flink-doris-connector-base/src/test/java/org/apache/doris/flink/sink/HttpTestUtil.java
similarity index 100%
rename from flink-doris-connector/src/test/java/org/apache/doris/flink/sink/HttpTestUtil.java
rename to flink-doris-connector/flink-doris-connector-base/src/test/java/org/apache/doris/flink/sink/HttpTestUtil.java
diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/sink/OptionUtils.java b/flink-doris-connector/flink-doris-connector-base/src/test/java/org/apache/doris/flink/sink/OptionUtils.java
similarity index 100%
rename from flink-doris-connector/src/test/java/org/apache/doris/flink/sink/OptionUtils.java
rename to flink-doris-connector/flink-doris-connector-base/src/test/java/org/apache/doris/flink/sink/OptionUtils.java
diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/sink/TestBackendUtil.java b/flink-doris-connector/flink-doris-connector-base/src/test/java/org/apache/doris/flink/sink/TestBackendUtil.java
similarity index 100%
rename from flink-doris-connector/src/test/java/org/apache/doris/flink/sink/TestBackendUtil.java
rename to flink-doris-connector/flink-doris-connector-base/src/test/java/org/apache/doris/flink/sink/TestBackendUtil.java
diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/sink/TestDorisCommittable.java b/flink-doris-connector/flink-doris-connector-base/src/test/java/org/apache/doris/flink/sink/TestDorisCommittable.java
similarity index 100%
rename from flink-doris-connector/src/test/java/org/apache/doris/flink/sink/TestDorisCommittable.java
rename to flink-doris-connector/flink-doris-connector-base/src/test/java/org/apache/doris/flink/sink/TestDorisCommittable.java
diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/sink/TestDorisCommittableSerializer.java b/flink-doris-connector/flink-doris-connector-base/src/test/java/org/apache/doris/flink/sink/TestDorisCommittableSerializer.java
similarity index 100%
rename from flink-doris-connector/src/test/java/org/apache/doris/flink/sink/TestDorisCommittableSerializer.java
rename to flink-doris-connector/flink-doris-connector-base/src/test/java/org/apache/doris/flink/sink/TestDorisCommittableSerializer.java
diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/sink/TestEscapeHandler.java b/flink-doris-connector/flink-doris-connector-base/src/test/java/org/apache/doris/flink/sink/TestEscapeHandler.java
similarity index 100%
rename from flink-doris-connector/src/test/java/org/apache/doris/flink/sink/TestEscapeHandler.java
rename to flink-doris-connector/flink-doris-connector-base/src/test/java/org/apache/doris/flink/sink/TestEscapeHandler.java
diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/sink/TestResponseUtil.java b/flink-doris-connector/flink-doris-connector-base/src/test/java/org/apache/doris/flink/sink/TestResponseUtil.java
similarity index 100%
rename from flink-doris-connector/src/test/java/org/apache/doris/flink/sink/TestResponseUtil.java
rename to flink-doris-connector/flink-doris-connector-base/src/test/java/org/apache/doris/flink/sink/TestResponseUtil.java
diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/sink/TestUtil.java b/flink-doris-connector/flink-doris-connector-base/src/test/java/org/apache/doris/flink/sink/TestUtil.java
similarity index 100%
rename from flink-doris-connector/src/test/java/org/apache/doris/flink/sink/TestUtil.java
rename to flink-doris-connector/flink-doris-connector-base/src/test/java/org/apache/doris/flink/sink/TestUtil.java
diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/sink/batch/TestBatchBufferHttpEntity.java b/flink-doris-connector/flink-doris-connector-base/src/test/java/org/apache/doris/flink/sink/batch/TestBatchBufferHttpEntity.java
similarity index 100%
rename from flink-doris-connector/src/test/java/org/apache/doris/flink/sink/batch/TestBatchBufferHttpEntity.java
rename to flink-doris-connector/flink-doris-connector-base/src/test/java/org/apache/doris/flink/sink/batch/TestBatchBufferHttpEntity.java
diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/sink/batch/TestBatchBufferStream.java b/flink-doris-connector/flink-doris-connector-base/src/test/java/org/apache/doris/flink/sink/batch/TestBatchBufferStream.java
similarity index 100%
rename from flink-doris-connector/src/test/java/org/apache/doris/flink/sink/batch/TestBatchBufferStream.java
rename to flink-doris-connector/flink-doris-connector-base/src/test/java/org/apache/doris/flink/sink/batch/TestBatchBufferStream.java
diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/sink/batch/TestDorisBatchSink.java b/flink-doris-connector/flink-doris-connector-base/src/test/java/org/apache/doris/flink/sink/batch/TestDorisBatchSink.java
similarity index 100%
rename from flink-doris-connector/src/test/java/org/apache/doris/flink/sink/batch/TestDorisBatchSink.java
rename to flink-doris-connector/flink-doris-connector-base/src/test/java/org/apache/doris/flink/sink/batch/TestDorisBatchSink.java
diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/sink/batch/TestDorisBatchStreamLoad.java b/flink-doris-connector/flink-doris-connector-base/src/test/java/org/apache/doris/flink/sink/batch/TestDorisBatchStreamLoad.java
similarity index 100%
rename from flink-doris-connector/src/test/java/org/apache/doris/flink/sink/batch/TestDorisBatchStreamLoad.java
rename to flink-doris-connector/flink-doris-connector-base/src/test/java/org/apache/doris/flink/sink/batch/TestDorisBatchStreamLoad.java
diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/sink/batch/TestDorisBatchWriter.java b/flink-doris-connector/flink-doris-connector-base/src/test/java/org/apache/doris/flink/sink/batch/TestDorisBatchWriter.java
similarity index 100%
rename from flink-doris-connector/src/test/java/org/apache/doris/flink/sink/batch/TestDorisBatchWriter.java
rename to flink-doris-connector/flink-doris-connector-base/src/test/java/org/apache/doris/flink/sink/batch/TestDorisBatchWriter.java
diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/sink/batch/TestRecordWithMeta.java b/flink-doris-connector/flink-doris-connector-base/src/test/java/org/apache/doris/flink/sink/batch/TestRecordWithMeta.java
similarity index 100%
rename from flink-doris-connector/src/test/java/org/apache/doris/flink/sink/batch/TestRecordWithMeta.java
rename to flink-doris-connector/flink-doris-connector-base/src/test/java/org/apache/doris/flink/sink/batch/TestRecordWithMeta.java
diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/sink/committer/MockCommitRequest.java b/flink-doris-connector/flink-doris-connector-base/src/test/java/org/apache/doris/flink/sink/committer/MockCommitRequest.java
similarity index 100%
rename from flink-doris-connector/src/test/java/org/apache/doris/flink/sink/committer/MockCommitRequest.java
rename to flink-doris-connector/flink-doris-connector-base/src/test/java/org/apache/doris/flink/sink/committer/MockCommitRequest.java
diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/sink/committer/TestDorisCommitter.java b/flink-doris-connector/flink-doris-connector-base/src/test/java/org/apache/doris/flink/sink/committer/TestDorisCommitter.java
similarity index 100%
rename from flink-doris-connector/src/test/java/org/apache/doris/flink/sink/committer/TestDorisCommitter.java
rename to flink-doris-connector/flink-doris-connector-base/src/test/java/org/apache/doris/flink/sink/committer/TestDorisCommitter.java
diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/sink/copy/TestBatchRecordBuffer.java b/flink-doris-connector/flink-doris-connector-base/src/test/java/org/apache/doris/flink/sink/copy/TestBatchRecordBuffer.java
similarity index 100%
rename from flink-doris-connector/src/test/java/org/apache/doris/flink/sink/copy/TestBatchRecordBuffer.java
rename to flink-doris-connector/flink-doris-connector-base/src/test/java/org/apache/doris/flink/sink/copy/TestBatchRecordBuffer.java
diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/sink/copy/TestCopyCommittableSerializer.java b/flink-doris-connector/flink-doris-connector-base/src/test/java/org/apache/doris/flink/sink/copy/TestCopyCommittableSerializer.java
similarity index 100%
rename from flink-doris-connector/src/test/java/org/apache/doris/flink/sink/copy/TestCopyCommittableSerializer.java
rename to flink-doris-connector/flink-doris-connector-base/src/test/java/org/apache/doris/flink/sink/copy/TestCopyCommittableSerializer.java
diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/sink/copy/TestDorisCopyCommitter.java b/flink-doris-connector/flink-doris-connector-base/src/test/java/org/apache/doris/flink/sink/copy/TestDorisCopyCommitter.java
similarity index 100%
rename from flink-doris-connector/src/test/java/org/apache/doris/flink/sink/copy/TestDorisCopyCommitter.java
rename to flink-doris-connector/flink-doris-connector-base/src/test/java/org/apache/doris/flink/sink/copy/TestDorisCopyCommitter.java
diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/sink/copy/TestDorisCopyWriter.java b/flink-doris-connector/flink-doris-connector-base/src/test/java/org/apache/doris/flink/sink/copy/TestDorisCopyWriter.java
similarity index 100%
rename from flink-doris-connector/src/test/java/org/apache/doris/flink/sink/copy/TestDorisCopyWriter.java
rename to flink-doris-connector/flink-doris-connector-base/src/test/java/org/apache/doris/flink/sink/copy/TestDorisCopyWriter.java
diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/sink/schema/SQLParserSchemaManagerTest.java b/flink-doris-connector/flink-doris-connector-base/src/test/java/org/apache/doris/flink/sink/schema/SQLParserSchemaManagerTest.java
similarity index 100%
rename from flink-doris-connector/src/test/java/org/apache/doris/flink/sink/schema/SQLParserSchemaManagerTest.java
rename to flink-doris-connector/flink-doris-connector-base/src/test/java/org/apache/doris/flink/sink/schema/SQLParserSchemaManagerTest.java
diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/sink/schema/SchemaChangeHelperTest.java b/flink-doris-connector/flink-doris-connector-base/src/test/java/org/apache/doris/flink/sink/schema/SchemaChangeHelperTest.java
similarity index 100%
rename from flink-doris-connector/src/test/java/org/apache/doris/flink/sink/schema/SchemaChangeHelperTest.java
rename to flink-doris-connector/flink-doris-connector-base/src/test/java/org/apache/doris/flink/sink/schema/SchemaChangeHelperTest.java
diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/sink/schema/SchemaManagerITCase.java b/flink-doris-connector/flink-doris-connector-base/src/test/java/org/apache/doris/flink/sink/schema/SchemaManagerITCase.java
similarity index 100%
rename from flink-doris-connector/src/test/java/org/apache/doris/flink/sink/schema/SchemaManagerITCase.java
rename to flink-doris-connector/flink-doris-connector-base/src/test/java/org/apache/doris/flink/sink/schema/SchemaManagerITCase.java
diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/sink/schema/SchemaManagerTest.java b/flink-doris-connector/flink-doris-connector-base/src/test/java/org/apache/doris/flink/sink/schema/SchemaManagerTest.java
similarity index 100%
rename from flink-doris-connector/src/test/java/org/apache/doris/flink/sink/schema/SchemaManagerTest.java
rename to flink-doris-connector/flink-doris-connector-base/src/test/java/org/apache/doris/flink/sink/schema/SchemaManagerTest.java
diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/sink/writer/TestCacheRecordBuffer.java b/flink-doris-connector/flink-doris-connector-base/src/test/java/org/apache/doris/flink/sink/writer/TestCacheRecordBuffer.java
similarity index 100%
rename from flink-doris-connector/src/test/java/org/apache/doris/flink/sink/writer/TestCacheRecordBuffer.java
rename to flink-doris-connector/flink-doris-connector-base/src/test/java/org/apache/doris/flink/sink/writer/TestCacheRecordBuffer.java
diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/sink/writer/TestDorisStreamLoad.java b/flink-doris-connector/flink-doris-connector-base/src/test/java/org/apache/doris/flink/sink/writer/TestDorisStreamLoad.java
similarity index 99%
rename from flink-doris-connector/src/test/java/org/apache/doris/flink/sink/writer/TestDorisStreamLoad.java
rename to flink-doris-connector/flink-doris-connector-base/src/test/java/org/apache/doris/flink/sink/writer/TestDorisStreamLoad.java
index b1fed97a6..1c8516a6c 100644
--- a/flink-doris-connector/src/test/java/org/apache/doris/flink/sink/writer/TestDorisStreamLoad.java
+++ b/flink-doris-connector/flink-doris-connector-base/src/test/java/org/apache/doris/flink/sink/writer/TestDorisStreamLoad.java
@@ -26,23 +26,21 @@
import org.apache.doris.flink.sink.OptionUtils;
import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.impl.client.CloseableHttpClient;
+import static org.hamcrest.core.StringStartsWith.startsWith;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.ExpectedException;
-
-import java.io.IOException;
-import java.nio.charset.StandardCharsets;
-import java.util.Properties;
-
-import static org.hamcrest.core.StringStartsWith.startsWith;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.ArgumentMatchers.anyLong;
import static org.mockito.Mockito.doNothing;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.spy;
import static org.mockito.Mockito.when;
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.util.Properties;
/** test for DorisStreamLoad. */
public class TestDorisStreamLoad {
@@ -57,6 +55,8 @@ public void setUp() throws Exception {
dorisOptions = OptionUtils.buildDorisOptions();
readOptions = OptionUtils.buildDorisReadOptions();
executionOptions = OptionUtils.buildExecutionOptional();
+ // clear thread interrupted status before each test
+ Thread.interrupted();
}
@Test
diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/sink/writer/TestDorisWriter.java b/flink-doris-connector/flink-doris-connector-base/src/test/java/org/apache/doris/flink/sink/writer/TestDorisWriter.java
similarity index 99%
rename from flink-doris-connector/src/test/java/org/apache/doris/flink/sink/writer/TestDorisWriter.java
rename to flink-doris-connector/flink-doris-connector-base/src/test/java/org/apache/doris/flink/sink/writer/TestDorisWriter.java
index c1f59c747..dc2826fe1 100644
--- a/flink-doris-connector/src/test/java/org/apache/doris/flink/sink/writer/TestDorisWriter.java
+++ b/flink-doris-connector/flink-doris-connector-base/src/test/java/org/apache/doris/flink/sink/writer/TestDorisWriter.java
@@ -17,11 +17,6 @@
package org.apache.doris.flink.sink.writer;
-import org.apache.flink.api.connector.sink2.Sink;
-import org.apache.flink.metrics.Counter;
-import org.apache.flink.metrics.Histogram;
-import org.apache.flink.metrics.groups.SinkWriterMetricGroup;
-
import org.apache.doris.flink.cfg.DorisExecutionOptions;
import org.apache.doris.flink.cfg.DorisOptions;
import org.apache.doris.flink.cfg.DorisReadOptions;
@@ -31,6 +26,10 @@
import org.apache.doris.flink.sink.HttpTestUtil;
import org.apache.doris.flink.sink.OptionUtils;
import org.apache.doris.flink.sink.writer.serializer.SimpleStringSerializer;
+import org.apache.flink.api.connector.sink2.Sink;
+import org.apache.flink.metrics.Counter;
+import org.apache.flink.metrics.Histogram;
+import org.apache.flink.metrics.groups.SinkWriterMetricGroup;
import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpUriRequest;
import org.apache.http.impl.client.CloseableHttpClient;
@@ -38,8 +37,12 @@
import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.ArgumentMatchers.anyString;
import org.mockito.MockedStatic;
-
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.mockStatic;
+import static org.mockito.Mockito.when;
import java.io.IOException;
import java.util.Collection;
import java.util.Collections;
@@ -48,12 +51,6 @@
import java.util.OptionalLong;
import java.util.concurrent.ConcurrentHashMap;
-import static org.mockito.ArgumentMatchers.any;
-import static org.mockito.ArgumentMatchers.anyString;
-import static org.mockito.Mockito.mock;
-import static org.mockito.Mockito.mockStatic;
-import static org.mockito.Mockito.when;
-
/** test for DorisWriter. */
public class TestDorisWriter {
DorisOptions dorisOptions;
@@ -68,6 +65,8 @@ public void setUp() {
executionOptions = OptionUtils.buildExecutionOptional();
backendUtilMockedStatic = mockStatic(BackendUtil.class);
backendUtilMockedStatic.when(() -> BackendUtil.tryHttpConnection(any())).thenReturn(true);
+ // clear thread interrupted status before each test
+ Thread.interrupted();
}
@Test
diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/sink/writer/TestDorisWriterStateSerializer.java b/flink-doris-connector/flink-doris-connector-base/src/test/java/org/apache/doris/flink/sink/writer/TestDorisWriterStateSerializer.java
similarity index 100%
rename from flink-doris-connector/src/test/java/org/apache/doris/flink/sink/writer/TestDorisWriterStateSerializer.java
rename to flink-doris-connector/flink-doris-connector-base/src/test/java/org/apache/doris/flink/sink/writer/TestDorisWriterStateSerializer.java
diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/sink/writer/TestJsonDebeziumSchemaSerializer.java b/flink-doris-connector/flink-doris-connector-base/src/test/java/org/apache/doris/flink/sink/writer/TestJsonDebeziumSchemaSerializer.java
similarity index 100%
rename from flink-doris-connector/src/test/java/org/apache/doris/flink/sink/writer/TestJsonDebeziumSchemaSerializer.java
rename to flink-doris-connector/flink-doris-connector-base/src/test/java/org/apache/doris/flink/sink/writer/TestJsonDebeziumSchemaSerializer.java
diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/sink/writer/TestLabelGenerator.java b/flink-doris-connector/flink-doris-connector-base/src/test/java/org/apache/doris/flink/sink/writer/TestLabelGenerator.java
similarity index 100%
rename from flink-doris-connector/src/test/java/org/apache/doris/flink/sink/writer/TestLabelGenerator.java
rename to flink-doris-connector/flink-doris-connector-base/src/test/java/org/apache/doris/flink/sink/writer/TestLabelGenerator.java
diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/sink/writer/TestRecordBuffer.java b/flink-doris-connector/flink-doris-connector-base/src/test/java/org/apache/doris/flink/sink/writer/TestRecordBuffer.java
similarity index 100%
rename from flink-doris-connector/src/test/java/org/apache/doris/flink/sink/writer/TestRecordBuffer.java
rename to flink-doris-connector/flink-doris-connector-base/src/test/java/org/apache/doris/flink/sink/writer/TestRecordBuffer.java
diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/sink/writer/TestRecordWithMetaSerializer.java b/flink-doris-connector/flink-doris-connector-base/src/test/java/org/apache/doris/flink/sink/writer/TestRecordWithMetaSerializer.java
similarity index 100%
rename from flink-doris-connector/src/test/java/org/apache/doris/flink/sink/writer/TestRecordWithMetaSerializer.java
rename to flink-doris-connector/flink-doris-connector-base/src/test/java/org/apache/doris/flink/sink/writer/TestRecordWithMetaSerializer.java
diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/sink/writer/TestRowDataSerializer.java b/flink-doris-connector/flink-doris-connector-base/src/test/java/org/apache/doris/flink/sink/writer/TestRowDataSerializer.java
similarity index 99%
rename from flink-doris-connector/src/test/java/org/apache/doris/flink/sink/writer/TestRowDataSerializer.java
rename to flink-doris-connector/flink-doris-connector-base/src/test/java/org/apache/doris/flink/sink/writer/TestRowDataSerializer.java
index ac10053dd..9626ae7e9 100644
--- a/flink-doris-connector/src/test/java/org/apache/doris/flink/sink/writer/TestRowDataSerializer.java
+++ b/flink-doris-connector/flink-doris-connector-base/src/test/java/org/apache/doris/flink/sink/writer/TestRowDataSerializer.java
@@ -17,6 +17,9 @@
package org.apache.doris.flink.sink.writer;
+import org.apache.doris.flink.sink.writer.serializer.RowDataSerializer;
+import com.fasterxml.jackson.core.type.TypeReference;
+import com.fasterxml.jackson.databind.ObjectMapper;
import org.apache.flink.table.api.DataTypes;
import org.apache.flink.table.data.GenericRowData;
import org.apache.flink.table.data.RowData;
@@ -27,14 +30,9 @@
import org.apache.flink.table.types.logical.RowType;
import org.apache.flink.table.types.utils.TypeConversions;
import org.apache.flink.types.RowKind;
-
-import com.fasterxml.jackson.core.type.TypeReference;
-import com.fasterxml.jackson.databind.ObjectMapper;
-import org.apache.doris.flink.sink.writer.serializer.RowDataSerializer;
import org.junit.Assert;
import org.junit.BeforeClass;
import org.junit.Test;
-
import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
@@ -56,6 +54,7 @@ public static void setUp() {
rowData.setRowKind(RowKind.INSERT);
dataTypes = new DataType[] {DataTypes.INT(), DataTypes.STRING(), DataTypes.DOUBLE()};
fieldNames = new String[] {"id", "name", "weight"};
+ Thread.interrupted();
}
@Test
diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/sink/writer/TestRowSerializer.java b/flink-doris-connector/flink-doris-connector-base/src/test/java/org/apache/doris/flink/sink/writer/TestRowSerializer.java
similarity index 100%
rename from flink-doris-connector/src/test/java/org/apache/doris/flink/sink/writer/TestRowSerializer.java
rename to flink-doris-connector/flink-doris-connector-base/src/test/java/org/apache/doris/flink/sink/writer/TestRowSerializer.java
diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/sink/writer/serializer/jsondebezium/TestJsonDebeziumChangeBase.java b/flink-doris-connector/flink-doris-connector-base/src/test/java/org/apache/doris/flink/sink/writer/serializer/jsondebezium/TestJsonDebeziumChangeBase.java
similarity index 100%
rename from flink-doris-connector/src/test/java/org/apache/doris/flink/sink/writer/serializer/jsondebezium/TestJsonDebeziumChangeBase.java
rename to flink-doris-connector/flink-doris-connector-base/src/test/java/org/apache/doris/flink/sink/writer/serializer/jsondebezium/TestJsonDebeziumChangeBase.java
diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/sink/writer/serializer/jsondebezium/TestJsonDebeziumDataChange.java b/flink-doris-connector/flink-doris-connector-base/src/test/java/org/apache/doris/flink/sink/writer/serializer/jsondebezium/TestJsonDebeziumDataChange.java
similarity index 100%
rename from flink-doris-connector/src/test/java/org/apache/doris/flink/sink/writer/serializer/jsondebezium/TestJsonDebeziumDataChange.java
rename to flink-doris-connector/flink-doris-connector-base/src/test/java/org/apache/doris/flink/sink/writer/serializer/jsondebezium/TestJsonDebeziumDataChange.java
diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/sink/writer/serializer/jsondebezium/TestJsonDebeziumSchemaChangeImpl.java b/flink-doris-connector/flink-doris-connector-base/src/test/java/org/apache/doris/flink/sink/writer/serializer/jsondebezium/TestJsonDebeziumSchemaChangeImpl.java
similarity index 100%
rename from flink-doris-connector/src/test/java/org/apache/doris/flink/sink/writer/serializer/jsondebezium/TestJsonDebeziumSchemaChangeImpl.java
rename to flink-doris-connector/flink-doris-connector-base/src/test/java/org/apache/doris/flink/sink/writer/serializer/jsondebezium/TestJsonDebeziumSchemaChangeImpl.java
diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/sink/writer/serializer/jsondebezium/TestJsonDebeziumSchemaChangeImplV2.java b/flink-doris-connector/flink-doris-connector-base/src/test/java/org/apache/doris/flink/sink/writer/serializer/jsondebezium/TestJsonDebeziumSchemaChangeImplV2.java
similarity index 100%
rename from flink-doris-connector/src/test/java/org/apache/doris/flink/sink/writer/serializer/jsondebezium/TestJsonDebeziumSchemaChangeImplV2.java
rename to flink-doris-connector/flink-doris-connector-base/src/test/java/org/apache/doris/flink/sink/writer/serializer/jsondebezium/TestJsonDebeziumSchemaChangeImplV2.java
diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/sink/writer/serializer/jsondebezium/TestSQLParserSchemaChange.java b/flink-doris-connector/flink-doris-connector-base/src/test/java/org/apache/doris/flink/sink/writer/serializer/jsondebezium/TestSQLParserSchemaChange.java
similarity index 100%
rename from flink-doris-connector/src/test/java/org/apache/doris/flink/sink/writer/serializer/jsondebezium/TestSQLParserSchemaChange.java
rename to flink-doris-connector/flink-doris-connector-base/src/test/java/org/apache/doris/flink/sink/writer/serializer/jsondebezium/TestSQLParserSchemaChange.java
diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/source/DorisSourceExampleTest.java b/flink-doris-connector/flink-doris-connector-base/src/test/java/org/apache/doris/flink/source/DorisSourceExampleTest.java
similarity index 100%
rename from flink-doris-connector/src/test/java/org/apache/doris/flink/source/DorisSourceExampleTest.java
rename to flink-doris-connector/flink-doris-connector-base/src/test/java/org/apache/doris/flink/source/DorisSourceExampleTest.java
diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/source/DorisSourceITCase.java b/flink-doris-connector/flink-doris-connector-base/src/test/java/org/apache/doris/flink/source/DorisSourceITCase.java
similarity index 96%
rename from flink-doris-connector/src/test/java/org/apache/doris/flink/source/DorisSourceITCase.java
rename to flink-doris-connector/flink-doris-connector-base/src/test/java/org/apache/doris/flink/source/DorisSourceITCase.java
index 31a94598e..ec9e98c6e 100644
--- a/flink-doris-connector/src/test/java/org/apache/doris/flink/source/DorisSourceITCase.java
+++ b/flink-doris-connector/flink-doris-connector-base/src/test/java/org/apache/doris/flink/source/DorisSourceITCase.java
@@ -17,6 +17,13 @@
package org.apache.doris.flink.source;
+import org.apache.doris.flink.catalog.doris.DataModel;
+import org.apache.doris.flink.cfg.DorisOptions;
+import org.apache.doris.flink.cfg.DorisReadOptions;
+import org.apache.doris.flink.container.AbstractITCaseService;
+import org.apache.doris.flink.container.ContainerUtils;
+import org.apache.doris.flink.deserialization.SimpleListDeserializationSchema;
+import org.apache.doris.flink.table.DorisConfigOptions;
import org.apache.flink.api.common.JobID;
import org.apache.flink.api.common.RuntimeExecutionMode;
import org.apache.flink.api.common.eventtime.WatermarkStrategy;
@@ -30,16 +37,6 @@
import org.apache.flink.test.util.MiniClusterWithClientResource;
import org.apache.flink.types.Row;
import org.apache.flink.util.CloseableIterator;
-
-import org.apache.doris.flink.catalog.doris.DataModel;
-import org.apache.doris.flink.cfg.DorisOptions;
-import org.apache.doris.flink.cfg.DorisReadOptions;
-import org.apache.doris.flink.cfg.DorisStreamOptions;
-import org.apache.doris.flink.container.AbstractITCaseService;
-import org.apache.doris.flink.container.ContainerUtils;
-import org.apache.doris.flink.datastream.DorisSourceFunction;
-import org.apache.doris.flink.deserialization.SimpleListDeserializationSchema;
-import org.apache.doris.flink.table.DorisConfigOptions;
import org.junit.Assert;
import org.junit.Rule;
import org.junit.Test;
@@ -47,14 +44,12 @@
import org.junit.runners.Parameterized;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-
import java.time.LocalDate;
import java.time.format.DateTimeFormatter;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Iterator;
import java.util.List;
-import java.util.Properties;
/** DorisSource ITCase. */
@RunWith(Parameterized.class)
@@ -62,7 +57,7 @@ public class DorisSourceITCase extends AbstractITCaseService {
private static final Logger LOG = LoggerFactory.getLogger(DorisSourceITCase.class);
private static final String DATABASE = "test_source";
private static final String TABLE_READ = "tbl_read";
- private static final String TABLE_READ_OLD_API = "tbl_read_old_api";
+ public static final String TABLE_READ_OLD_API = "tbl_read_old_api";
private static final String TABLE_READ_TBL = "tbl_read_tbl";
private static final String TABLE_READ_TBL_OLD_API = "tbl_read_tbl_old_api";
private static final String TABLE_READ_TBL_ALL_OPTIONS = "tbl_read_tbl_all_options";
@@ -142,32 +137,6 @@ public void testSource() throws Exception {
checkResultInAnyOrder("testSource", expected.toArray(), actual.toArray());
}
- @Test
- public void testOldSourceApi() throws Exception {
- initializeTable(TABLE_READ_OLD_API, DataModel.UNIQUE);
- StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
- env.setParallelism(DEFAULT_PARALLELISM);
- Properties properties = new Properties();
- properties.put("fenodes", getFenodes());
- properties.put("username", getDorisUsername());
- properties.put("password", getDorisPassword());
- properties.put("table.identifier", DATABASE + "." + TABLE_READ_OLD_API);
- DorisStreamOptions options = new DorisStreamOptions(properties);
-
- List actual = new ArrayList<>();
- try (CloseableIterator> iterator =
- env.addSource(
- new DorisSourceFunction(
- options, new SimpleListDeserializationSchema()))
- .executeAndCollect()) {
- while (iterator.hasNext()) {
- actual.add(iterator.next().toString());
- }
- }
- List expected = Arrays.asList("[doris, 18]", "[flink, 10]", "[apache, 12]");
- checkResultInAnyOrder("testOldSourceApi", expected.toArray(), actual.toArray());
- }
-
@Test
public void testTableSource() throws Exception {
initializeTable(TABLE_READ_TBL, DataModel.DUPLICATE);
@@ -816,7 +785,7 @@ private void checkResult(String testName, Object[] expected, Object[] actual) {
Assert.assertArrayEquals(expected, actual);
}
- private void checkResultInAnyOrder(String testName, Object[] expected, Object[] actual) {
+ public static void checkResultInAnyOrder(String testName, Object[] expected, Object[] actual) {
LOG.info(
"Checking DorisSourceITCase result. testName={}, actual={}, expected={}",
testName,
@@ -825,7 +794,7 @@ private void checkResultInAnyOrder(String testName, Object[] expected, Object[]
assertEqualsInAnyOrder(Arrays.asList(expected), Arrays.asList(actual));
}
- private void initializeTable(String table, DataModel dataModel) {
+ public static void initializeTable(String table, DataModel dataModel) {
String morProps =
!DataModel.UNIQUE_MOR.equals(dataModel)
? ""
diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/source/enumerator/DorisSourceEnumeratorTest.java b/flink-doris-connector/flink-doris-connector-base/src/test/java/org/apache/doris/flink/source/enumerator/DorisSourceEnumeratorTest.java
similarity index 100%
rename from flink-doris-connector/src/test/java/org/apache/doris/flink/source/enumerator/DorisSourceEnumeratorTest.java
rename to flink-doris-connector/flink-doris-connector-base/src/test/java/org/apache/doris/flink/source/enumerator/DorisSourceEnumeratorTest.java
diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/source/enumerator/PendingSplitsCheckpointSerializerTest.java b/flink-doris-connector/flink-doris-connector-base/src/test/java/org/apache/doris/flink/source/enumerator/PendingSplitsCheckpointSerializerTest.java
similarity index 100%
rename from flink-doris-connector/src/test/java/org/apache/doris/flink/source/enumerator/PendingSplitsCheckpointSerializerTest.java
rename to flink-doris-connector/flink-doris-connector-base/src/test/java/org/apache/doris/flink/source/enumerator/PendingSplitsCheckpointSerializerTest.java
diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/source/reader/DorisSourceReaderTest.java b/flink-doris-connector/flink-doris-connector-base/src/test/java/org/apache/doris/flink/source/reader/DorisSourceReaderTest.java
similarity index 100%
rename from flink-doris-connector/src/test/java/org/apache/doris/flink/source/reader/DorisSourceReaderTest.java
rename to flink-doris-connector/flink-doris-connector-base/src/test/java/org/apache/doris/flink/source/reader/DorisSourceReaderTest.java
diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/source/reader/TestingReaderContext.java b/flink-doris-connector/flink-doris-connector-base/src/test/java/org/apache/doris/flink/source/reader/TestingReaderContext.java
similarity index 100%
rename from flink-doris-connector/src/test/java/org/apache/doris/flink/source/reader/TestingReaderContext.java
rename to flink-doris-connector/flink-doris-connector-base/src/test/java/org/apache/doris/flink/source/reader/TestingReaderContext.java
diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/source/split/DorisSourceSplitSerializerTest.java b/flink-doris-connector/flink-doris-connector-base/src/test/java/org/apache/doris/flink/source/split/DorisSourceSplitSerializerTest.java
similarity index 100%
rename from flink-doris-connector/src/test/java/org/apache/doris/flink/source/split/DorisSourceSplitSerializerTest.java
rename to flink-doris-connector/flink-doris-connector-base/src/test/java/org/apache/doris/flink/source/split/DorisSourceSplitSerializerTest.java
diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/source/split/DorisSourceSplitTest.java b/flink-doris-connector/flink-doris-connector-base/src/test/java/org/apache/doris/flink/source/split/DorisSourceSplitTest.java
similarity index 100%
rename from flink-doris-connector/src/test/java/org/apache/doris/flink/source/split/DorisSourceSplitTest.java
rename to flink-doris-connector/flink-doris-connector-base/src/test/java/org/apache/doris/flink/source/split/DorisSourceSplitTest.java
diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/source/split/DorisSplitRecordsTest.java b/flink-doris-connector/flink-doris-connector-base/src/test/java/org/apache/doris/flink/source/split/DorisSplitRecordsTest.java
similarity index 100%
rename from flink-doris-connector/src/test/java/org/apache/doris/flink/source/split/DorisSplitRecordsTest.java
rename to flink-doris-connector/flink-doris-connector-base/src/test/java/org/apache/doris/flink/source/split/DorisSplitRecordsTest.java
diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/table/DorisDynamicTableFactoryTest.java b/flink-doris-connector/flink-doris-connector-base/src/test/java/org/apache/doris/flink/table/DorisDynamicTableFactoryTest.java
similarity index 100%
rename from flink-doris-connector/src/test/java/org/apache/doris/flink/table/DorisDynamicTableFactoryTest.java
rename to flink-doris-connector/flink-doris-connector-base/src/test/java/org/apache/doris/flink/table/DorisDynamicTableFactoryTest.java
diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/table/DorisDynamicTableSourceTest.java b/flink-doris-connector/flink-doris-connector-base/src/test/java/org/apache/doris/flink/table/DorisDynamicTableSourceTest.java
similarity index 100%
rename from flink-doris-connector/src/test/java/org/apache/doris/flink/table/DorisDynamicTableSourceTest.java
rename to flink-doris-connector/flink-doris-connector-base/src/test/java/org/apache/doris/flink/table/DorisDynamicTableSourceTest.java
diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/table/DorisRowDataJdbcLookupFunctionITCase.java b/flink-doris-connector/flink-doris-connector-base/src/test/java/org/apache/doris/flink/table/DorisRowDataJdbcLookupFunctionITCase.java
similarity index 100%
rename from flink-doris-connector/src/test/java/org/apache/doris/flink/table/DorisRowDataJdbcLookupFunctionITCase.java
rename to flink-doris-connector/flink-doris-connector-base/src/test/java/org/apache/doris/flink/table/DorisRowDataJdbcLookupFunctionITCase.java
diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/utils/FactoryMocks.java b/flink-doris-connector/flink-doris-connector-base/src/test/java/org/apache/doris/flink/utils/FactoryMocks.java
similarity index 100%
rename from flink-doris-connector/src/test/java/org/apache/doris/flink/utils/FactoryMocks.java
rename to flink-doris-connector/flink-doris-connector-base/src/test/java/org/apache/doris/flink/utils/FactoryMocks.java
diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/utils/FastDateUtilTest.java b/flink-doris-connector/flink-doris-connector-base/src/test/java/org/apache/doris/flink/utils/FastDateUtilTest.java
similarity index 100%
rename from flink-doris-connector/src/test/java/org/apache/doris/flink/utils/FastDateUtilTest.java
rename to flink-doris-connector/flink-doris-connector-base/src/test/java/org/apache/doris/flink/utils/FastDateUtilTest.java
diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/utils/MockMultiTableSource.java b/flink-doris-connector/flink-doris-connector-base/src/test/java/org/apache/doris/flink/utils/MockMultiTableSource.java
similarity index 100%
rename from flink-doris-connector/src/test/java/org/apache/doris/flink/utils/MockMultiTableSource.java
rename to flink-doris-connector/flink-doris-connector-base/src/test/java/org/apache/doris/flink/utils/MockMultiTableSource.java
diff --git a/flink-doris-connector/src/test/java/org/apache/doris/flink/utils/MockSource.java b/flink-doris-connector/flink-doris-connector-base/src/test/java/org/apache/doris/flink/utils/MockSource.java
similarity index 100%
rename from flink-doris-connector/src/test/java/org/apache/doris/flink/utils/MockSource.java
rename to flink-doris-connector/flink-doris-connector-base/src/test/java/org/apache/doris/flink/utils/MockSource.java
diff --git a/flink-doris-connector/src/test/resources/container/e2e/doris2doris/test_doris2doris_sink_test_tbl.sql b/flink-doris-connector/flink-doris-connector-base/src/test/resources/container/e2e/doris2doris/test_doris2doris_sink_test_tbl.sql
similarity index 100%
rename from flink-doris-connector/src/test/resources/container/e2e/doris2doris/test_doris2doris_sink_test_tbl.sql
rename to flink-doris-connector/flink-doris-connector-base/src/test/resources/container/e2e/doris2doris/test_doris2doris_sink_test_tbl.sql
diff --git a/flink-doris-connector/src/test/resources/container/e2e/doris2doris/test_doris2doris_source_test_tbl.sql b/flink-doris-connector/flink-doris-connector-base/src/test/resources/container/e2e/doris2doris/test_doris2doris_source_test_tbl.sql
similarity index 100%
rename from flink-doris-connector/src/test/resources/container/e2e/doris2doris/test_doris2doris_source_test_tbl.sql
rename to flink-doris-connector/flink-doris-connector-base/src/test/resources/container/e2e/doris2doris/test_doris2doris_source_test_tbl.sql
diff --git a/flink-doris-connector/src/test/resources/container/e2e/mysql2doris/testAutoAddTable.txt b/flink-doris-connector/flink-doris-connector-base/src/test/resources/container/e2e/mysql2doris/testAutoAddTable.txt
similarity index 100%
rename from flink-doris-connector/src/test/resources/container/e2e/mysql2doris/testAutoAddTable.txt
rename to flink-doris-connector/flink-doris-connector-base/src/test/resources/container/e2e/mysql2doris/testAutoAddTable.txt
diff --git a/flink-doris-connector/src/test/resources/container/e2e/mysql2doris/testAutoAddTable_init.sql b/flink-doris-connector/flink-doris-connector-base/src/test/resources/container/e2e/mysql2doris/testAutoAddTable_init.sql
similarity index 100%
rename from flink-doris-connector/src/test/resources/container/e2e/mysql2doris/testAutoAddTable_init.sql
rename to flink-doris-connector/flink-doris-connector-base/src/test/resources/container/e2e/mysql2doris/testAutoAddTable_init.sql
diff --git a/flink-doris-connector/src/test/resources/container/e2e/mysql2doris/testMySQL2Doris.txt b/flink-doris-connector/flink-doris-connector-base/src/test/resources/container/e2e/mysql2doris/testMySQL2Doris.txt
similarity index 100%
rename from flink-doris-connector/src/test/resources/container/e2e/mysql2doris/testMySQL2Doris.txt
rename to flink-doris-connector/flink-doris-connector-base/src/test/resources/container/e2e/mysql2doris/testMySQL2Doris.txt
diff --git a/flink-doris-connector/src/test/resources/container/e2e/mysql2doris/testMySQL2DorisByDefault.txt b/flink-doris-connector/flink-doris-connector-base/src/test/resources/container/e2e/mysql2doris/testMySQL2DorisByDefault.txt
similarity index 100%
rename from flink-doris-connector/src/test/resources/container/e2e/mysql2doris/testMySQL2DorisByDefault.txt
rename to flink-doris-connector/flink-doris-connector-base/src/test/resources/container/e2e/mysql2doris/testMySQL2DorisByDefault.txt
diff --git a/flink-doris-connector/src/test/resources/container/e2e/mysql2doris/testMySQL2DorisByDefault_init.sql b/flink-doris-connector/flink-doris-connector-base/src/test/resources/container/e2e/mysql2doris/testMySQL2DorisByDefault_init.sql
similarity index 100%
rename from flink-doris-connector/src/test/resources/container/e2e/mysql2doris/testMySQL2DorisByDefault_init.sql
rename to flink-doris-connector/flink-doris-connector-base/src/test/resources/container/e2e/mysql2doris/testMySQL2DorisByDefault_init.sql
diff --git a/flink-doris-connector/src/test/resources/container/e2e/mysql2doris/testMySQL2DorisCreateTable.txt b/flink-doris-connector/flink-doris-connector-base/src/test/resources/container/e2e/mysql2doris/testMySQL2DorisCreateTable.txt
similarity index 100%
rename from flink-doris-connector/src/test/resources/container/e2e/mysql2doris/testMySQL2DorisCreateTable.txt
rename to flink-doris-connector/flink-doris-connector-base/src/test/resources/container/e2e/mysql2doris/testMySQL2DorisCreateTable.txt
diff --git a/flink-doris-connector/src/test/resources/container/e2e/mysql2doris/testMySQL2DorisCreateTable_init.sql b/flink-doris-connector/flink-doris-connector-base/src/test/resources/container/e2e/mysql2doris/testMySQL2DorisCreateTable_init.sql
similarity index 100%
rename from flink-doris-connector/src/test/resources/container/e2e/mysql2doris/testMySQL2DorisCreateTable_init.sql
rename to flink-doris-connector/flink-doris-connector-base/src/test/resources/container/e2e/mysql2doris/testMySQL2DorisCreateTable_init.sql
diff --git a/flink-doris-connector/src/test/resources/container/e2e/mysql2doris/testMySQL2DorisEnableDelete.txt b/flink-doris-connector/flink-doris-connector-base/src/test/resources/container/e2e/mysql2doris/testMySQL2DorisEnableDelete.txt
similarity index 100%
rename from flink-doris-connector/src/test/resources/container/e2e/mysql2doris/testMySQL2DorisEnableDelete.txt
rename to flink-doris-connector/flink-doris-connector-base/src/test/resources/container/e2e/mysql2doris/testMySQL2DorisEnableDelete.txt
diff --git a/flink-doris-connector/src/test/resources/container/e2e/mysql2doris/testMySQL2DorisEnableDelete_init.sql b/flink-doris-connector/flink-doris-connector-base/src/test/resources/container/e2e/mysql2doris/testMySQL2DorisEnableDelete_init.sql
similarity index 100%
rename from flink-doris-connector/src/test/resources/container/e2e/mysql2doris/testMySQL2DorisEnableDelete_init.sql
rename to flink-doris-connector/flink-doris-connector-base/src/test/resources/container/e2e/mysql2doris/testMySQL2DorisEnableDelete_init.sql
diff --git a/flink-doris-connector/src/test/resources/container/e2e/mysql2doris/testMySQL2DorisMultiDb2One.txt b/flink-doris-connector/flink-doris-connector-base/src/test/resources/container/e2e/mysql2doris/testMySQL2DorisMultiDb2One.txt
similarity index 100%
rename from flink-doris-connector/src/test/resources/container/e2e/mysql2doris/testMySQL2DorisMultiDb2One.txt
rename to flink-doris-connector/flink-doris-connector-base/src/test/resources/container/e2e/mysql2doris/testMySQL2DorisMultiDb2One.txt
diff --git a/flink-doris-connector/src/test/resources/container/e2e/mysql2doris/testMySQL2DorisMultiDb2One_init.sql b/flink-doris-connector/flink-doris-connector-base/src/test/resources/container/e2e/mysql2doris/testMySQL2DorisMultiDb2One_init.sql
similarity index 100%
rename from flink-doris-connector/src/test/resources/container/e2e/mysql2doris/testMySQL2DorisMultiDb2One_init.sql
rename to flink-doris-connector/flink-doris-connector-base/src/test/resources/container/e2e/mysql2doris/testMySQL2DorisMultiDb2One_init.sql
diff --git a/flink-doris-connector/src/test/resources/container/e2e/mysql2doris/testMySQL2DorisMultiDbSync.txt b/flink-doris-connector/flink-doris-connector-base/src/test/resources/container/e2e/mysql2doris/testMySQL2DorisMultiDbSync.txt
similarity index 100%
rename from flink-doris-connector/src/test/resources/container/e2e/mysql2doris/testMySQL2DorisMultiDbSync.txt
rename to flink-doris-connector/flink-doris-connector-base/src/test/resources/container/e2e/mysql2doris/testMySQL2DorisMultiDbSync.txt
diff --git a/flink-doris-connector/src/test/resources/container/e2e/mysql2doris/testMySQL2DorisMultiDbSync_init.sql b/flink-doris-connector/flink-doris-connector-base/src/test/resources/container/e2e/mysql2doris/testMySQL2DorisMultiDbSync_init.sql
similarity index 100%
rename from flink-doris-connector/src/test/resources/container/e2e/mysql2doris/testMySQL2DorisMultiDbSync_init.sql
rename to flink-doris-connector/flink-doris-connector-base/src/test/resources/container/e2e/mysql2doris/testMySQL2DorisMultiDbSync_init.sql
diff --git a/flink-doris-connector/src/test/resources/container/e2e/mysql2doris/testMySQL2DorisSQLParse.txt b/flink-doris-connector/flink-doris-connector-base/src/test/resources/container/e2e/mysql2doris/testMySQL2DorisSQLParse.txt
similarity index 100%
rename from flink-doris-connector/src/test/resources/container/e2e/mysql2doris/testMySQL2DorisSQLParse.txt
rename to flink-doris-connector/flink-doris-connector-base/src/test/resources/container/e2e/mysql2doris/testMySQL2DorisSQLParse.txt
diff --git a/flink-doris-connector/src/test/resources/container/e2e/mysql2doris/testMySQL2DorisSQLParse_init.sql b/flink-doris-connector/flink-doris-connector-base/src/test/resources/container/e2e/mysql2doris/testMySQL2DorisSQLParse_init.sql
similarity index 100%
rename from flink-doris-connector/src/test/resources/container/e2e/mysql2doris/testMySQL2DorisSQLParse_init.sql
rename to flink-doris-connector/flink-doris-connector-base/src/test/resources/container/e2e/mysql2doris/testMySQL2DorisSQLParse_init.sql
diff --git a/flink-doris-connector/src/test/resources/container/e2e/mysql2doris/testMySQL2Doris_init.sql b/flink-doris-connector/flink-doris-connector-base/src/test/resources/container/e2e/mysql2doris/testMySQL2Doris_init.sql
similarity index 100%
rename from flink-doris-connector/src/test/resources/container/e2e/mysql2doris/testMySQL2Doris_init.sql
rename to flink-doris-connector/flink-doris-connector-base/src/test/resources/container/e2e/mysql2doris/testMySQL2Doris_init.sql
diff --git a/flink-doris-connector/src/test/resources/docker/doris/be.conf b/flink-doris-connector/flink-doris-connector-base/src/test/resources/docker/doris/be.conf
similarity index 100%
rename from flink-doris-connector/src/test/resources/docker/doris/be.conf
rename to flink-doris-connector/flink-doris-connector-base/src/test/resources/docker/doris/be.conf
diff --git a/flink-doris-connector/src/test/resources/docker/doris/fe.conf b/flink-doris-connector/flink-doris-connector-base/src/test/resources/docker/doris/fe.conf
similarity index 100%
rename from flink-doris-connector/src/test/resources/docker/doris/fe.conf
rename to flink-doris-connector/flink-doris-connector-base/src/test/resources/docker/doris/fe.conf
diff --git a/flink-doris-connector/src/test/resources/log4j2-test.properties b/flink-doris-connector/flink-doris-connector-base/src/test/resources/log4j2-test.properties
similarity index 100%
rename from flink-doris-connector/src/test/resources/log4j2-test.properties
rename to flink-doris-connector/flink-doris-connector-base/src/test/resources/log4j2-test.properties
diff --git a/flink-doris-connector/pom.xml b/flink-doris-connector/pom.xml
index c584ebbf2..077586e63 100644
--- a/flink-doris-connector/pom.xml
+++ b/flink-doris-connector/pom.xml
@@ -26,9 +26,10 @@ under the License.
23org.apache.doris
- flink-doris-connector-${flink.major.version}
+ flink-doris-connector-parent${revision}
- Flink Doris Connector
+ pom
+ Flink Doris Connector Parenthttps://doris.apache.org/
@@ -67,10 +68,8 @@ under the License.
- 25.1.0-SNAPSHOT
- 1.18.0
- 1.18
- 3.2.1
+ 26.0.0-SNAPSHOT
+ 3.5.0flink-python0.16.03.10.1
@@ -81,7 +80,7 @@ under the License.
1.0.18.142.17
- 2.4.2
+ 2.27.14.5.131.152.13.3
@@ -99,404 +98,221 @@ under the License.
0.12.0
-
-
- org.apache.doris
- thrift-service
- ${thrift-service.version}
-
-
- org.apache.flink
- flink-table-planner-loader
- ${flink.version}
- provided
-
-
-
- org.apache.flink
- flink-connector-base
- ${flink.version}
- provided
-
-
-
- org.apache.flink
- flink-table-api-java-bridge
- ${flink.version}
- provided
-
-
-
- org.apache.flink
- flink-table-runtime
- ${flink.version}
- provided
-
-
-
- org.apache.flink
- ${flink.python.id}
- ${flink.version}
- provided
-
-
-
- org.apache.thrift
- libthrift
- ${libthrift.version}
-
-
- httpclient
- org.apache.httpcomponents
-
-
- httpcore
- org.apache.httpcomponents
-
-
-
-
-
- org.apache.httpcomponents
- httpclient
- ${httpcomponents.version}
-
-
- commons-codec
- commons-codec
-
-
-
-
-
- commons-codec
- commons-codec
- ${commons-codec.version}
-
-
- org.apache.arrow.adbc
- adbc-driver-flight-sql
- ${adbc.version}
-
-
- org.apache.arrow
- flight-sql-jdbc-core
-
-
- org.apache.arrow
- flight-sql-jdbc-driver
-
-
- org.apache.arrow
- arrow-vector
-
-
- org.apache.arrow.adbc
- adbc-driver-manager
-
-
-
-
- org.apache.arrow
- arrow-vector
- ${arrow.version}
-
-
- org.apache.arrow
- arrow-memory-netty
- ${arrow.version}
- runtime
-
-
- com.fasterxml.jackson.core
- jackson-annotations
-
-
- com.fasterxml.jackson.core
- jackson-core
-
-
- com.fasterxml.jackson.core
- jackson-databind
-
-
-
-
-
- com.fasterxml.jackson.core
- jackson-annotations
- ${fasterxml.version}
-
-
-
- com.fasterxml.jackson.core
- jackson-core
- ${fasterxml.version}
-
-
-
- com.fasterxml.jackson.core
- jackson-databind
- ${fasterxml.version}
-
-
- com.google.guava
- guava
- ${guava.version}
-
-
- org.apache.logging.log4j
- log4j-slf4j-impl
- ${log4j.version}
- test
-
-
- org.apache.logging.log4j
- log4j-api
- ${log4j.version}
- test
-
-
- org.apache.logging.log4j
- log4j-core
- ${log4j.version}
- test
-
-
-
- org.apache.logging.log4j
- log4j-1.2-api
- ${log4j.version}
- test
-
-
-
- org.apache.flink
- flink-sql-connector-mysql-cdc
- ${flink.sql.cdc.version}
- provided
-
-
- flink-shaded-guava
- org.apache.flink
-
-
-
-
- org.apache.flink
- flink-sql-connector-oracle-cdc
- ${flink.sql.cdc.version}
- provided
-
-
- flink-shaded-guava
- org.apache.flink
-
-
-
-
- org.apache.flink
- flink-sql-connector-postgres-cdc
- ${flink.sql.cdc.version}
- provided
-
-
- flink-shaded-guava
- org.apache.flink
-
-
-
-
- org.apache.flink
- flink-sql-connector-sqlserver-cdc
- ${flink.sql.cdc.version}
- provided
-
-
- flink-shaded-guava
- org.apache.flink
-
-
-
-
- org.apache.flink
- flink-sql-connector-db2-cdc
- ${flink.sql.cdc.version}
- provided
-
-
- flink-shaded-guava
- org.apache.flink
-
-
-
-
- org.apache.flink
- flink-sql-connector-mongodb-cdc
-
- ${flink.sql.cdc.version}
- provided
-
-
- flink-shaded-guava
- org.apache.flink
-
-
-
-
-
- mysql
- mysql-connector-java
- ${mysql.driver.version}
- test
-
-
- com.oracle.ojdbc
- ojdbc8
- ${ojdbc.version}
- provided
-
-
-
- org.apache.flink
- flink-runtime-web
- ${flink.version}
- test
-
-
-
- org.hamcrest
- hamcrest-core
- ${hamcrest.version}
- test
-
-
-
- org.mockito
- mockito-core
- ${mockito.version}
- test
-
-
- org.mockito
- mockito-inline
- ${mockito.version}
- test
-
-
-
- junit
- junit
- ${junit.version}
- test
-
-
- org.testcontainers
- testcontainers
- ${testcontainers.version}
- test
-
-
- org.testcontainers
- mysql
- ${testcontainers.version}
- test
-
-
- org.apache.flink
- flink-table-common
- ${flink.version}
- test-jar
- test
-
-
- org.apache.flink
- flink-test-utils
- ${flink.version}
- test
-
-
- org.apache.flink
- flink-connector-test-utils
- ${flink.version}
- test
-
-
- com.github.jsqlparser
- jsqlparser
- ${jsqlparser.version}
-
-
+
+ flink-doris-connector-base
+ flink-doris-connector-1.20
+ flink-doris-connector-2.2
+
+
+
+
+
+ org.apache.doris
+ thrift-service
+ ${thrift-service.version}
+
+
+
+
+ org.apache.flink
+ flink-table-planner-loader
+ ${flink.version}
+ provided
+
+
+ org.apache.flink
+ flink-connector-base
+ ${flink.version}
+ provided
+
+
+ org.apache.flink
+ flink-table-api-java-bridge
+ ${flink.version}
+ provided
+
+
+ org.apache.flink
+ flink-table-runtime
+ ${flink.version}
+ provided
+
+
+ org.apache.flink
+ ${flink.python.id}
+ ${flink.version}
+ provided
+
+
+
+
+ org.apache.doris
+ flink-doris-connector-base
+ ${revision}
+
+
+
+ org.apache.thrift
+ libthrift
+ ${libthrift.version}
+
+
+ org.apache.httpcomponents
+ httpclient
+ ${httpcomponents.version}
+
+
+ commons-codec
+ commons-codec
+ ${commons-codec.version}
+
+
+ org.apache.arrow.adbc
+ adbc-driver-flight-sql
+ ${adbc.version}
+
+
+ org.apache.arrow
+ arrow-vector
+ ${arrow.version}
+
+
+ org.apache.arrow
+ arrow-memory-netty
+ ${arrow.version}
+
+
+
+ com.fasterxml.jackson.core
+ jackson-annotations
+ ${fasterxml.version}
+
+
+ com.fasterxml.jackson.core
+ jackson-core
+ ${fasterxml.version}
+
+
+ com.fasterxml.jackson.core
+ jackson-databind
+ ${fasterxml.version}
+
+
+ com.google.guava
+ guava
+ ${guava.version}
+
+
+ com.github.jsqlparser
+ jsqlparser
+ ${jsqlparser.version}
+
+
+
+
+ org.apache.logging.log4j
+ log4j-slf4j-impl
+ ${log4j.version}
+ test
+
+
+ org.apache.logging.log4j
+ log4j-api
+ ${log4j.version}
+ test
+
+
+ org.apache.logging.log4j
+ log4j-core
+ ${log4j.version}
+ test
+
+
+
+ org.apache.logging.log4j
+ log4j-1.2-api
+ ${log4j.version}
+ test
+
+
+ junit
+ junit
+ ${junit.version}
+ test
+
+
+ org.hamcrest
+ hamcrest-core
+ ${hamcrest.version}
+ test
+
+
+ org.mockito
+ mockito-core
+ ${mockito.version}
+ test
+
+
+ org.mockito
+ mockito-inline
+ ${mockito.version}
+ test
+
+
+ org.apache.flink
+ flink-runtime-web
+ ${flink.version}
+ test
+
+
+ org.testcontainers
+ testcontainers
+ ${testcontainers.version}
+ test
+
+
+ org.testcontainers
+ mysql
+ ${testcontainers.version}
+ test
+
+
+ org.apache.flink
+ flink-table-common
+ ${flink.version}
+ test-jar
+ test
+
+
+ org.apache.flink
+ flink-test-utils
+ ${flink.version}
+ test
+
+
+ org.apache.flink
+ flink-connector-test-utils
+ ${flink.version}
+ test
+
+
+ org.awaitility
+ awaitility
+ 4.2.0
+ test
+
+
+ mysql
+ mysql-connector-java
+ ${mysql.driver.version}
+ test
+
+
+
-
- org.apache.maven.plugins
- maven-gpg-plugin
-
-
- sign-artifacts
- verify
-
- sign
-
-
-
-
-
-
- org.apache.maven.plugins
- maven-shade-plugin
- 3.4.1
-
-
-
- org.apache.arrow
- org.apache.doris.shaded.org.apache.arrow
-
-
- io.netty
- org.apache.doris.shaded.io.netty
-
-
- com.fasterxml.jackson
- org.apache.doris.shaded.com.fasterxml.jackson
-
-
- org.apache.commons.codec
- org.apache.doris.shaded.org.apache.commons.codec
-
-
- com.google
- org.apache.doris.shaded.com.google
-
-
- org.apache.thrift
- org.apache.doris.shaded.org.apache.thrift
-
-
-
-
- *:*
-
- META-INF/*.SF
- META-INF/*.DSA
- META-INF/*.RSA
-
-
-
-
-
-
-
- shade
-
- package
-
-
-
-
org.apache.maven.pluginsmaven-compiler-plugin
@@ -506,157 +322,43 @@ under the License.
8
-
org.apache.maven.plugins
- maven-javadoc-plugin
- ${maven-javadoc-plugin.version}
+ maven-checkstyle-plugin
+ ${maven-checkstyle-plugin.version}true
- 8
- false
-
-
-
- attach-javadocs
-
- jar
-
-
-
-
-
-
- org.apache.maven.plugins
- maven-source-plugin
- ${maven-source-plugin.version}
-
- true
-
-
-
- compile
-
- jar
-
-
-
-
-
-
- org.codehaus.mojo
- license-maven-plugin
- 2.0.0
-
-
- add-third-party
-
- add-third-party
-
-
-
-
-
-
- org.apache.maven.plugins
- maven-release-plugin
- 3.0.0-M5
-
- ${releaseArgs}
+ ../../tools/maven/suppressions.xml
+ ../../tools/maven/checkstyle.xml
-
-
-
- org.apache.maven.plugins
- maven-surefire-plugin
- 3.0.0-M5
-
-
-
-
-
-
-
- org.apache.maven.plugins
- maven-shade-plugin
-
-
- com.diffplug.spotless
- spotless-maven-plugin
- ${spotless.version}
-
-
-
- 1.7
-
-
-
-
-
- org.apache.flink,org.apache.flink.shaded,,javax,java,scala,\#
-
-
-
-
-
-
-
-
- spotless-check
- validate
-
- check
-
-
-
-
-
- org.apache.maven.plugins
- maven-checkstyle-plugin
- ${maven-checkstyle-plugin.version}
-
-
- com.puppycrawl.tools
- checkstyle
-
- ${checkstyle.version}
-
-
-
-
- validate
- validate
-
- check
-
-
-
-
- true
- ../tools/maven/suppressions.xml
- true
- ../tools/maven/checkstyle.xml
- true
- true
-
-
-
- org.apache.maven.plugins
- maven-compiler-plugin
-
-
-
-
+
+
+ flink-1.20
+
+ true
+
+
+ 1.20.0
+ 1.20
+ 8
+ 8
+
+
+
+ flink-2.2
+
+ 2.2.0
+ 2.2
+ 17
+ 17
+
+ custom-env
@@ -677,8 +379,6 @@ under the License.
-
-
general-env
@@ -694,38 +394,5 @@ under the License.
-
-
- apache-release
-
-
-
- org.apache.maven.plugins
- maven-gpg-plugin
-
-
-
- org.apache.maven.plugins
- maven-javadoc-plugin
-
-
-
- org.apache.maven.plugins
- maven-source-plugin
-
-
-
- org.codehaus.mojo
- license-maven-plugin
-
-
-
- org.apache.maven.plugins
- maven-release-plugin
-
-
-
-
-
diff --git a/tools/releasing/deploy_staging_jars.sh b/tools/releasing/deploy_staging_jars.sh
index a66d2a1ce..3e8a99ef7 100644
--- a/tools/releasing/deploy_staging_jars.sh
+++ b/tools/releasing/deploy_staging_jars.sh
@@ -44,22 +44,25 @@ cd ${PROJECT_ROOT}/flink-doris-connector
echo "Deploying to repository.apache.org"
echo "Deploying Flink 1.15..."
-${MVN} clean deploy -Papache-release -DretryFailedDeploymentCount=10 -Dflink.version=1.15.0 -Dflink.major.version=1.15 -Dflink.python.id=flink-python_2.12 -DskipTests=true
+${MVN} clean deploy -Papache-release -DretryFailedDeploymentCount=10 -pl flink-doris-connector-1.20 -am -Dflink.version=1.15.0 -Dflink.major.version=1.15 -Dflink.python.id=flink-python_2.12 -DskipTests=true
echo "Deploying Flink 1.16..."
-${MVN} clean deploy -Papache-release -DretryFailedDeploymentCount=10 -Dflink.version=1.16.0 -Dflink.major.version=1.16 -DskipTests=true
+${MVN} clean deploy -Papache-release -DretryFailedDeploymentCount=10 -pl flink-doris-connector-1.20 -am -Dflink.version=1.16.0 -Dflink.major.version=1.16 -DskipTests=true
echo "Deploying Flink 1.17..."
-${MVN} clean deploy -Papache-release -DretryFailedDeploymentCount=10 -Dflink.version=1.17.0 -Dflink.major.version=1.17 -DskipTests=true
+${MVN} clean deploy -Papache-release -DretryFailedDeploymentCount=10 -pl flink-doris-connector-1.20 -am -Dflink.version=1.17.0 -Dflink.major.version=1.17 -DskipTests=true
echo "Deploying Flink 1.18..."
-${MVN} clean deploy -Papache-release -DretryFailedDeploymentCount=10 -Dflink.version=1.18.0 -Dflink.major.version=1.18 -DskipTests=true
+${MVN} clean deploy -Papache-release -DretryFailedDeploymentCount=10 -pl flink-doris-connector-1.20 -am -Dflink.version=1.18.0 -Dflink.major.version=1.18 -DskipTests=true
echo "Deploying Flink 1.19..."
-${MVN} clean deploy -Papache-release -DretryFailedDeploymentCount=10 -Dflink.version=1.19.0 -Dflink.major.version=1.19 -DskipTests=true
+${MVN} clean deploy -Papache-release -DretryFailedDeploymentCount=10 -pl flink-doris-connector-1.20 -am -Dflink.version=1.19.0 -Dflink.major.version=1.19 -DskipTests=true
echo "Deploying Flink 1.20..."
-${MVN} clean deploy -Papache-release -DretryFailedDeploymentCount=10 -Dflink.version=1.20.0 -Dflink.major.version=1.20 -DskipTests=true
+${MVN} clean deploy -Papache-release -DretryFailedDeploymentCount=10 -pl flink-doris-connector-1.20 -am -Dflink.version=1.20.0 -Dflink.major.version=1.20 -DskipTests=true
+
+echo "Deploying Flink 2.2..."
+${MVN} clean deploy -Papache-release -DretryFailedDeploymentCount=10 -pl flink-doris-connector-2.2 -am -DskipTests=true
echo "Deploy jar finished."
cd ${CURR_DIR}
\ No newline at end of file
From ea9e61c3a944a54c636871adbf67ce966ed68620 Mon Sep 17 00:00:00 2001
From: wudi <676366545@qq.com>
Date: Wed, 25 Feb 2026 10:23:34 +0800
Subject: [PATCH 02/22] checkstyle and fix ci
---
.github/workflows/run-e2ecase.yml | 2 +-
.github/workflows/run-itcase.yml | 4 +-
.../flink-doris-connector-1.20/pom.xml | 73 +-----
.../flink/example/CDCSchemaChangeExample.java | 12 +-
.../flink/example/DorisSourceDataStream.java | 6 +-
.../source/DorisSourceITCaseForOldApi.java | 11 +-
.../flink-doris-connector-base/pom.xml | 9 -
.../doris/flink/tools/cdc/db2/Db2Type.java | 3 +-
.../flink/tools/cdc/mysql/MysqlType.java | 6 +-
.../flink/tools/cdc/oracle/OracleType.java | 3 +-
.../tools/cdc/postgres/PostgresType.java | 3 +-
.../flink/example/DataGen2DorisExample.java | 2 +-
.../flink/example/DorisSourceExample.java | 30 +--
.../sink/writer/TestDorisStreamLoad.java | 10 +-
.../flink/sink/writer/TestDorisWriter.java | 21 +-
.../sink/writer/TestRowDataSerializer.java | 8 +-
.../doris/flink/source/DorisSourceITCase.java | 16 +-
flink-doris-connector/pom.xml | 244 +++++++++++++++++-
18 files changed, 321 insertions(+), 142 deletions(-)
diff --git a/.github/workflows/run-e2ecase.yml b/.github/workflows/run-e2ecase.yml
index 777e3db31..3e509902b 100644
--- a/.github/workflows/run-e2ecase.yml
+++ b/.github/workflows/run-e2ecase.yml
@@ -41,6 +41,6 @@ jobs:
- name: Run E2ECases
run: |
cd flink-doris-connector && mvn test \
- -pl flink-doris-connector-1.20 -am \
+ -Pflink-1.20 -pl flink-doris-connector-1.20 -am \
-Dtest="*E2ECase" \
-Dimage="apache/doris:doris-all-in-one-2.1.0"
diff --git a/.github/workflows/run-itcase.yml b/.github/workflows/run-itcase.yml
index 1d0a933a1..ef53b83e9 100644
--- a/.github/workflows/run-itcase.yml
+++ b/.github/workflows/run-itcase.yml
@@ -38,9 +38,9 @@ jobs:
distribution: adopt
java-version: '8'
- - name: Run ITCases (Flink 1.x module)
+ - name: Run ITCases (Flink 1.20 module)
run: |
- cd flink-doris-connector && mvn test \
+ cd flink-doris-connector && mvn test -Pflink-1.20 \
-pl flink-doris-connector-1.20 -am \
-Dtest="*ITCase" \
-Dimage="apache/doris:doris-all-in-one-2.1.0"
diff --git a/flink-doris-connector/flink-doris-connector-1.20/pom.xml b/flink-doris-connector/flink-doris-connector-1.20/pom.xml
index 0272cb4a8..b7f81b4ea 100644
--- a/flink-doris-connector/flink-doris-connector-1.20/pom.xml
+++ b/flink-doris-connector/flink-doris-connector-1.20/pom.xml
@@ -35,9 +35,6 @@ under the License.
1.20.01.20flink-python
-
- 8
- 8
@@ -137,7 +134,12 @@ under the License.
${ojdbc.version}provided
+
+ com.github.jsqlparser
+ jsqlparser
+
+
org.hamcresthamcrest-core
@@ -174,10 +176,6 @@ under the License.
org.apache.flinkflink-connector-test-utils
-
- com.github.jsqlparser
- jsqlparser
-
@@ -185,71 +183,12 @@ under the License.
org.apache.maven.pluginsmaven-compiler-plugin
+
88
-
- org.apache.maven.plugins
- maven-shade-plugin
- 3.4.1
-
-
-
- org.apache.arrow
- org.apache.doris.shaded.org.apache.arrow
-
-
- io.netty
- org.apache.doris.shaded.io.netty
-
-
- com.fasterxml.jackson
- org.apache.doris.shaded.com.fasterxml.jackson
-
-
- org.apache.commons.codec
- org.apache.doris.shaded.org.apache.commons.codec
-
-
- com.google
- org.apache.doris.shaded.com.google
-
-
- org.apache.thrift
- org.apache.doris.shaded.org.apache.thrift
-
-
-
-
- *:*
-
- META-INF/*.SF
- META-INF/*.DSA
- META-INF/*.RSA
-
-
-
-
-
-
-
- shade
-
- package
-
-
-
-
- org.apache.maven.plugins
- maven-checkstyle-plugin
-
- ../../tools/maven/suppressions.xml
- ../../tools/maven/checkstyle.xml
- true
-
- org.apache.maven.pluginsmaven-jar-plugin
diff --git a/flink-doris-connector/flink-doris-connector-1.20/src/test/java/org/apache/doris/flink/example/CDCSchemaChangeExample.java b/flink-doris-connector/flink-doris-connector-1.20/src/test/java/org/apache/doris/flink/example/CDCSchemaChangeExample.java
index c29c31b96..8fe017e1c 100644
--- a/flink-doris-connector/flink-doris-connector-1.20/src/test/java/org/apache/doris/flink/example/CDCSchemaChangeExample.java
+++ b/flink-doris-connector/flink-doris-connector-1.20/src/test/java/org/apache/doris/flink/example/CDCSchemaChangeExample.java
@@ -17,17 +17,19 @@
package org.apache.doris.flink.example;
+import org.apache.flink.api.common.eventtime.WatermarkStrategy;
+import org.apache.flink.cdc.connectors.mysql.source.MySqlSource;
+import org.apache.flink.cdc.connectors.shaded.org.apache.kafka.connect.json.JsonConverterConfig;
+import org.apache.flink.cdc.debezium.JsonDebeziumDeserializationSchema;
+import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
+
import org.apache.doris.flink.cfg.DorisExecutionOptions;
import org.apache.doris.flink.cfg.DorisOptions;
import org.apache.doris.flink.cfg.DorisReadOptions;
import org.apache.doris.flink.sink.DorisSink;
import org.apache.doris.flink.sink.writer.serializer.JsonDebeziumSchemaSerializer;
import org.apache.doris.flink.utils.DateToStringConverter;
-import org.apache.flink.api.common.eventtime.WatermarkStrategy;
-import org.apache.flink.cdc.connectors.mysql.source.MySqlSource;
-import org.apache.flink.cdc.connectors.shaded.org.apache.kafka.connect.json.JsonConverterConfig;
-import org.apache.flink.cdc.debezium.JsonDebeziumDeserializationSchema;
-import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
+
import java.util.HashMap;
import java.util.Map;
import java.util.Properties;
diff --git a/flink-doris-connector/flink-doris-connector-1.20/src/test/java/org/apache/doris/flink/example/DorisSourceDataStream.java b/flink-doris-connector/flink-doris-connector-1.20/src/test/java/org/apache/doris/flink/example/DorisSourceDataStream.java
index 5e973c090..ba57b9509 100644
--- a/flink-doris-connector/flink-doris-connector-1.20/src/test/java/org/apache/doris/flink/example/DorisSourceDataStream.java
+++ b/flink-doris-connector/flink-doris-connector-1.20/src/test/java/org/apache/doris/flink/example/DorisSourceDataStream.java
@@ -17,14 +17,16 @@
package org.apache.doris.flink.example;
+import org.apache.flink.api.common.eventtime.WatermarkStrategy;
+import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
+
import org.apache.doris.flink.cfg.DorisOptions;
import org.apache.doris.flink.cfg.DorisReadOptions;
import org.apache.doris.flink.cfg.DorisStreamOptions;
import org.apache.doris.flink.datastream.DorisSourceFunction;
import org.apache.doris.flink.deserialization.SimpleListDeserializationSchema;
import org.apache.doris.flink.source.DorisSource;
-import org.apache.flink.api.common.eventtime.WatermarkStrategy;
-import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
+
import java.util.List;
import java.util.Properties;
diff --git a/flink-doris-connector/flink-doris-connector-1.20/src/test/java/org/apache/doris/flink/source/DorisSourceITCaseForOldApi.java b/flink-doris-connector/flink-doris-connector-1.20/src/test/java/org/apache/doris/flink/source/DorisSourceITCaseForOldApi.java
index 838215799..fd625117e 100644
--- a/flink-doris-connector/flink-doris-connector-1.20/src/test/java/org/apache/doris/flink/source/DorisSourceITCaseForOldApi.java
+++ b/flink-doris-connector/flink-doris-connector-1.20/src/test/java/org/apache/doris/flink/source/DorisSourceITCaseForOldApi.java
@@ -1,20 +1,23 @@
package org.apache.doris.flink.source;
+import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
+import org.apache.flink.util.CloseableIterator;
+
import org.apache.doris.flink.catalog.doris.DataModel;
import org.apache.doris.flink.cfg.DorisStreamOptions;
import org.apache.doris.flink.container.AbstractITCaseService;
import org.apache.doris.flink.datastream.DorisSourceFunction;
import org.apache.doris.flink.deserialization.SimpleListDeserializationSchema;
-import static org.apache.doris.flink.source.DorisSourceITCase.checkResultInAnyOrder;
-import static org.apache.doris.flink.source.DorisSourceITCase.initializeTable;
-import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
-import org.apache.flink.util.CloseableIterator;
import org.junit.Test;
+
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import java.util.Properties;
+import static org.apache.doris.flink.source.DorisSourceITCase.checkResultInAnyOrder;
+import static org.apache.doris.flink.source.DorisSourceITCase.initializeTable;
+
public class DorisSourceITCaseForOldApi extends AbstractITCaseService {
private static final String DATABASE = "test_source";
diff --git a/flink-doris-connector/flink-doris-connector-base/pom.xml b/flink-doris-connector/flink-doris-connector-base/pom.xml
index cf0e48e23..1570a080a 100644
--- a/flink-doris-connector/flink-doris-connector-base/pom.xml
+++ b/flink-doris-connector/flink-doris-connector-base/pom.xml
@@ -222,15 +222,6 @@ under the License.
-
- org.apache.maven.plugins
- maven-checkstyle-plugin
-
- ../../tools/maven/suppressions.xml
- ../../tools/maven/checkstyle.xml
- true
-
-
diff --git a/flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/tools/cdc/db2/Db2Type.java b/flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/tools/cdc/db2/Db2Type.java
index 8668256d2..1255d1e7c 100644
--- a/flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/tools/cdc/db2/Db2Type.java
+++ b/flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/tools/cdc/db2/Db2Type.java
@@ -17,9 +17,10 @@
package org.apache.doris.flink.tools.cdc.db2;
-import org.apache.doris.flink.catalog.doris.DorisType;
import org.apache.flink.util.Preconditions;
+import org.apache.doris.flink.catalog.doris.DorisType;
+
public class Db2Type {
private static final String BOOLEAN = "BOOLEAN";
private static final String SMALLINT = "SMALLINT";
diff --git a/flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/tools/cdc/mysql/MysqlType.java b/flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/tools/cdc/mysql/MysqlType.java
index 3bbc2a2eb..49afe3751 100644
--- a/flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/tools/cdc/mysql/MysqlType.java
+++ b/flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/tools/cdc/mysql/MysqlType.java
@@ -17,11 +17,13 @@
package org.apache.doris.flink.tools.cdc.mysql;
-import org.apache.doris.flink.catalog.doris.DorisType;
-import static org.apache.doris.flink.catalog.DorisTypeMapper.MAX_SUPPORTED_DATE_TIME_PRECISION;
import org.apache.flink.table.types.logical.TimestampType;
import org.apache.flink.util.Preconditions;
+import org.apache.doris.flink.catalog.doris.DorisType;
+
+import static org.apache.doris.flink.catalog.DorisTypeMapper.MAX_SUPPORTED_DATE_TIME_PRECISION;
+
public class MysqlType {
// MySQL driver returns width of timestamp types instead of precision.
diff --git a/flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/tools/cdc/oracle/OracleType.java b/flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/tools/cdc/oracle/OracleType.java
index dfd3babf5..304f21fd0 100644
--- a/flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/tools/cdc/oracle/OracleType.java
+++ b/flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/tools/cdc/oracle/OracleType.java
@@ -17,9 +17,10 @@
package org.apache.doris.flink.tools.cdc.oracle;
-import org.apache.doris.flink.catalog.doris.DorisType;
import org.apache.flink.util.Preconditions;
+import org.apache.doris.flink.catalog.doris.DorisType;
+
public class OracleType {
private static final String VARCHAR2 = "VARCHAR2";
private static final String NVARCHAR2 = "NVARCHAR2";
diff --git a/flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/tools/cdc/postgres/PostgresType.java b/flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/tools/cdc/postgres/PostgresType.java
index 9127e54aa..ddffb6d9f 100644
--- a/flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/tools/cdc/postgres/PostgresType.java
+++ b/flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/tools/cdc/postgres/PostgresType.java
@@ -17,9 +17,10 @@
package org.apache.doris.flink.tools.cdc.postgres;
-import org.apache.doris.flink.catalog.doris.DorisType;
import org.apache.flink.util.Preconditions;
+import org.apache.doris.flink.catalog.doris.DorisType;
+
public class PostgresType {
private static final String INT2 = "int2";
private static final String SMALLSERIAL = "smallserial";
diff --git a/flink-doris-connector/flink-doris-connector-base/src/test/java/org/apache/doris/flink/example/DataGen2DorisExample.java b/flink-doris-connector/flink-doris-connector-base/src/test/java/org/apache/doris/flink/example/DataGen2DorisExample.java
index 6c5116c3e..dabacd9ea 100644
--- a/flink-doris-connector/flink-doris-connector-base/src/test/java/org/apache/doris/flink/example/DataGen2DorisExample.java
+++ b/flink-doris-connector/flink-doris-connector-base/src/test/java/org/apache/doris/flink/example/DataGen2DorisExample.java
@@ -51,7 +51,7 @@ public static void main(String[] args) {
+ " ) "
+ " WITH ("
+ " 'connector' = 'doris',"
- + " 'fenodes' = '127.0.0.1:8030',"
+ + " 'fenodes' = '10.16.10.6:28737',"
+ " 'table.identifier' = 'test.student',"
+ " 'username' = 'root',"
+ " 'password' = '',"
diff --git a/flink-doris-connector/flink-doris-connector-base/src/test/java/org/apache/doris/flink/example/DorisSourceExample.java b/flink-doris-connector/flink-doris-connector-base/src/test/java/org/apache/doris/flink/example/DorisSourceExample.java
index 90e461cfc..b7e2346f0 100644
--- a/flink-doris-connector/flink-doris-connector-base/src/test/java/org/apache/doris/flink/example/DorisSourceExample.java
+++ b/flink-doris-connector/flink-doris-connector-base/src/test/java/org/apache/doris/flink/example/DorisSourceExample.java
@@ -33,25 +33,17 @@ public static void main(String[] args) throws Exception {
// register a table in the catalog
tEnv.executeSql(
"CREATE TABLE doris_source ("
- + "bigint_1 BIGINT,"
- + "char_1 STRING,"
- + "date_1 STRING,"
- + "datetime_1 STRING,"
- + "decimal_1 DECIMAL(5,2),"
- + "double_1 DOUBLE,"
- + "float_1 FLOAT ,"
- + "int_1 INT ,"
- + "largeint_1 STRING, "
- + "smallint_1 SMALLINT, "
- + "tinyint_1 TINYINT, "
- + "varchar_1 STRING "
- + ") "
- + "WITH (\n"
- + " 'connector' = 'doris',\n"
- + " 'fenodes' = 'FE_IP:8030',\n"
- + " 'table.identifier' = 'db.table',\n"
- + " 'username' = 'root',\n"
- + " 'password' = ''\n"
+ + " id INT,"
+ + " name STRING,"
+ + " age INT"
+ + " ) "
+ + " WITH ("
+ + " 'connector' = 'doris',"
+ + " 'fenodes' = '10.16.10.6:28737',"
+ + " 'table.identifier' = 'test.student',"
+ + " 'username' = 'root',"
+ + " 'password' = '',"
+ + " 'sink.label-prefix' = 'doris_labe3l'"
+ ")");
// define a dynamic aggregating query
diff --git a/flink-doris-connector/flink-doris-connector-base/src/test/java/org/apache/doris/flink/sink/writer/TestDorisStreamLoad.java b/flink-doris-connector/flink-doris-connector-base/src/test/java/org/apache/doris/flink/sink/writer/TestDorisStreamLoad.java
index 1c8516a6c..cda8674ad 100644
--- a/flink-doris-connector/flink-doris-connector-base/src/test/java/org/apache/doris/flink/sink/writer/TestDorisStreamLoad.java
+++ b/flink-doris-connector/flink-doris-connector-base/src/test/java/org/apache/doris/flink/sink/writer/TestDorisStreamLoad.java
@@ -26,21 +26,23 @@
import org.apache.doris.flink.sink.OptionUtils;
import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.impl.client.CloseableHttpClient;
-import static org.hamcrest.core.StringStartsWith.startsWith;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.ExpectedException;
+
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.util.Properties;
+
+import static org.hamcrest.core.StringStartsWith.startsWith;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.ArgumentMatchers.anyLong;
import static org.mockito.Mockito.doNothing;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.spy;
import static org.mockito.Mockito.when;
-import java.io.IOException;
-import java.nio.charset.StandardCharsets;
-import java.util.Properties;
/** test for DorisStreamLoad. */
public class TestDorisStreamLoad {
diff --git a/flink-doris-connector/flink-doris-connector-base/src/test/java/org/apache/doris/flink/sink/writer/TestDorisWriter.java b/flink-doris-connector/flink-doris-connector-base/src/test/java/org/apache/doris/flink/sink/writer/TestDorisWriter.java
index dc2826fe1..bd251e4a3 100644
--- a/flink-doris-connector/flink-doris-connector-base/src/test/java/org/apache/doris/flink/sink/writer/TestDorisWriter.java
+++ b/flink-doris-connector/flink-doris-connector-base/src/test/java/org/apache/doris/flink/sink/writer/TestDorisWriter.java
@@ -17,6 +17,11 @@
package org.apache.doris.flink.sink.writer;
+import org.apache.flink.api.connector.sink2.Sink;
+import org.apache.flink.metrics.Counter;
+import org.apache.flink.metrics.Histogram;
+import org.apache.flink.metrics.groups.SinkWriterMetricGroup;
+
import org.apache.doris.flink.cfg.DorisExecutionOptions;
import org.apache.doris.flink.cfg.DorisOptions;
import org.apache.doris.flink.cfg.DorisReadOptions;
@@ -26,10 +31,6 @@
import org.apache.doris.flink.sink.HttpTestUtil;
import org.apache.doris.flink.sink.OptionUtils;
import org.apache.doris.flink.sink.writer.serializer.SimpleStringSerializer;
-import org.apache.flink.api.connector.sink2.Sink;
-import org.apache.flink.metrics.Counter;
-import org.apache.flink.metrics.Histogram;
-import org.apache.flink.metrics.groups.SinkWriterMetricGroup;
import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpUriRequest;
import org.apache.http.impl.client.CloseableHttpClient;
@@ -37,12 +38,8 @@
import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
-import static org.mockito.ArgumentMatchers.any;
-import static org.mockito.ArgumentMatchers.anyString;
import org.mockito.MockedStatic;
-import static org.mockito.Mockito.mock;
-import static org.mockito.Mockito.mockStatic;
-import static org.mockito.Mockito.when;
+
import java.io.IOException;
import java.util.Collection;
import java.util.Collections;
@@ -51,6 +48,12 @@
import java.util.OptionalLong;
import java.util.concurrent.ConcurrentHashMap;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.ArgumentMatchers.anyString;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.mockStatic;
+import static org.mockito.Mockito.when;
+
/** test for DorisWriter. */
public class TestDorisWriter {
DorisOptions dorisOptions;
diff --git a/flink-doris-connector/flink-doris-connector-base/src/test/java/org/apache/doris/flink/sink/writer/TestRowDataSerializer.java b/flink-doris-connector/flink-doris-connector-base/src/test/java/org/apache/doris/flink/sink/writer/TestRowDataSerializer.java
index 9626ae7e9..3c68a6820 100644
--- a/flink-doris-connector/flink-doris-connector-base/src/test/java/org/apache/doris/flink/sink/writer/TestRowDataSerializer.java
+++ b/flink-doris-connector/flink-doris-connector-base/src/test/java/org/apache/doris/flink/sink/writer/TestRowDataSerializer.java
@@ -17,9 +17,6 @@
package org.apache.doris.flink.sink.writer;
-import org.apache.doris.flink.sink.writer.serializer.RowDataSerializer;
-import com.fasterxml.jackson.core.type.TypeReference;
-import com.fasterxml.jackson.databind.ObjectMapper;
import org.apache.flink.table.api.DataTypes;
import org.apache.flink.table.data.GenericRowData;
import org.apache.flink.table.data.RowData;
@@ -30,9 +27,14 @@
import org.apache.flink.table.types.logical.RowType;
import org.apache.flink.table.types.utils.TypeConversions;
import org.apache.flink.types.RowKind;
+
+import com.fasterxml.jackson.core.type.TypeReference;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import org.apache.doris.flink.sink.writer.serializer.RowDataSerializer;
import org.junit.Assert;
import org.junit.BeforeClass;
import org.junit.Test;
+
import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
diff --git a/flink-doris-connector/flink-doris-connector-base/src/test/java/org/apache/doris/flink/source/DorisSourceITCase.java b/flink-doris-connector/flink-doris-connector-base/src/test/java/org/apache/doris/flink/source/DorisSourceITCase.java
index ec9e98c6e..7a43bf806 100644
--- a/flink-doris-connector/flink-doris-connector-base/src/test/java/org/apache/doris/flink/source/DorisSourceITCase.java
+++ b/flink-doris-connector/flink-doris-connector-base/src/test/java/org/apache/doris/flink/source/DorisSourceITCase.java
@@ -17,13 +17,6 @@
package org.apache.doris.flink.source;
-import org.apache.doris.flink.catalog.doris.DataModel;
-import org.apache.doris.flink.cfg.DorisOptions;
-import org.apache.doris.flink.cfg.DorisReadOptions;
-import org.apache.doris.flink.container.AbstractITCaseService;
-import org.apache.doris.flink.container.ContainerUtils;
-import org.apache.doris.flink.deserialization.SimpleListDeserializationSchema;
-import org.apache.doris.flink.table.DorisConfigOptions;
import org.apache.flink.api.common.JobID;
import org.apache.flink.api.common.RuntimeExecutionMode;
import org.apache.flink.api.common.eventtime.WatermarkStrategy;
@@ -37,6 +30,14 @@
import org.apache.flink.test.util.MiniClusterWithClientResource;
import org.apache.flink.types.Row;
import org.apache.flink.util.CloseableIterator;
+
+import org.apache.doris.flink.catalog.doris.DataModel;
+import org.apache.doris.flink.cfg.DorisOptions;
+import org.apache.doris.flink.cfg.DorisReadOptions;
+import org.apache.doris.flink.container.AbstractITCaseService;
+import org.apache.doris.flink.container.ContainerUtils;
+import org.apache.doris.flink.deserialization.SimpleListDeserializationSchema;
+import org.apache.doris.flink.table.DorisConfigOptions;
import org.junit.Assert;
import org.junit.Rule;
import org.junit.Test;
@@ -44,6 +45,7 @@
import org.junit.runners.Parameterized;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+
import java.time.LocalDate;
import java.time.format.DateTimeFormatter;
import java.util.ArrayList;
diff --git a/flink-doris-connector/pom.xml b/flink-doris-connector/pom.xml
index 077586e63..5c3d2eafd 100644
--- a/flink-doris-connector/pom.xml
+++ b/flink-doris-connector/pom.xml
@@ -313,6 +313,72 @@ under the License.
+
+ org.apache.maven.plugins
+ maven-gpg-plugin
+
+
+ sign-artifacts
+ verify
+
+ sign
+
+
+
+
+
+
+ org.apache.maven.plugins
+ maven-shade-plugin
+ 3.4.1
+
+
+
+ org.apache.arrow
+ org.apache.doris.shaded.org.apache.arrow
+
+
+ io.netty
+ org.apache.doris.shaded.io.netty
+
+
+ com.fasterxml.jackson
+ org.apache.doris.shaded.com.fasterxml.jackson
+
+
+ org.apache.commons.codec
+ org.apache.doris.shaded.org.apache.commons.codec
+
+
+ com.google
+ org.apache.doris.shaded.com.google
+
+
+ org.apache.thrift
+ org.apache.doris.shaded.org.apache.thrift
+
+
+
+
+ *:*
+
+ META-INF/*.SF
+ META-INF/*.DSA
+ META-INF/*.RSA
+
+
+
+
+
+
+
+ shade
+
+ package
+
+
+
+
org.apache.maven.pluginsmaven-compiler-plugin
@@ -322,18 +388,152 @@ under the License.
8
+
org.apache.maven.plugins
- maven-checkstyle-plugin
- ${maven-checkstyle-plugin.version}
+ maven-javadoc-plugin
+ ${maven-javadoc-plugin.version}true
- ../../tools/maven/suppressions.xml
- ../../tools/maven/checkstyle.xml
+ 8
+ false
+
+
+
+ attach-javadocs
+
+ jar
+
+
+
+
+
+
+ org.apache.maven.plugins
+ maven-source-plugin
+ ${maven-source-plugin.version}
+
+ true
+
+
+
+ compile
+
+ jar
+
+
+
+
+
+
+ org.codehaus.mojo
+ license-maven-plugin
+ 2.0.0
+
+
+ add-third-party
+
+ add-third-party
+
+
+
+
+
+
+ org.apache.maven.plugins
+ maven-release-plugin
+ 3.0.0-M5
+
+ ${releaseArgs}
+
+
+
+ org.apache.maven.plugins
+ maven-surefire-plugin
+ 3.0.0-M5
+
+
+
+
+
+
+
+ org.apache.maven.plugins
+ maven-shade-plugin
+
+
+ com.diffplug.spotless
+ spotless-maven-plugin
+ ${spotless.version}
+
+
+
+ 1.7
+
+
+
+
+
+ org.apache.flink,org.apache.flink.shaded,,javax,java,scala,\#
+
+
+
+
+
+
+
+
+ spotless-check
+ validate
+
+ check
+
+
+
+
+
+ org.apache.maven.plugins
+ maven-checkstyle-plugin
+ ${maven-checkstyle-plugin.version}
+
+
+ com.puppycrawl.tools
+ checkstyle
+
+ ${checkstyle.version}
+
+
+
+
+ validate
+ validate
+
+ check
+
+
+
+
+ true
+ ../tools/maven/suppressions.xml
+ true
+ ../tools/maven/checkstyle.xml
+ true
+ true
+
+
+
+ org.apache.maven.plugins
+ maven-compiler-plugin
+
+
@@ -359,6 +559,7 @@ under the License.
17
+
custom-env
@@ -379,6 +580,8 @@ under the License.
+
+
general-env
@@ -394,5 +597,38 @@ under the License.
+
+
+
+ apache-release
+
+
+
+ org.apache.maven.plugins
+ maven-gpg-plugin
+
+
+
+ org.apache.maven.plugins
+ maven-javadoc-plugin
+
+
+
+ org.apache.maven.plugins
+ maven-source-plugin
+
+
+
+ org.codehaus.mojo
+ license-maven-plugin
+
+
+
+ org.apache.maven.plugins
+ maven-release-plugin
+
+
+
+
From 5bf6b425460d63b0051f0426f2ff2fa6575b85b0 Mon Sep 17 00:00:00 2001
From: wudi <676366545@qq.com>
Date: Wed, 25 Feb 2026 19:11:04 +0800
Subject: [PATCH 03/22] add flink2.x and it moudle part1
---
.github/workflows/build-connector.yml | 3 +-
.../flink-doris-connector-1.20/pom.xml | 58 ++
.../doris/flink/catalog/DorisCatalog.java | 12 +-
.../flink/catalog/DorisCatalogFactory.java | 0
.../apache/doris/flink/sink/DorisSink.java | 11 +-
.../flink/sink/batch/DorisBatchSink.java | 4 +-
.../sink/batch/DorisBatchWriterAdapter.java | 88 +++
.../sink/copy/DorisCopyWriterAdapter.java | 71 +++
.../sink/writer/DorisAbstractWriter.java | 0
.../flink/sink/writer/DorisWriterAdapter.java | 97 +++
.../DorisAsyncLookupFunctionAdapter.java | 88 +++
.../flink/table/DorisDynamicTableFactory.java | 0
.../flink/table/DorisDynamicTableSink.java | 0
.../flink/table/DorisDynamicTableSource.java | 0
.../catalog/DorisCatalogFactoryTest.java | 0
.../doris/flink/catalog/DorisCatalogTest.java | 6 +-
.../doris/flink/example/CatalogExample.java | 0
.../flink/example/DataGen2DorisExample.java | 0
.../example/DorisDateAndTimestampSqlTest.java | 0
.../DorisIntranetAccessSinkExample.java | 0
.../example/DorisSinkArraySQLExample.java | 0
.../flink/example/DorisSinkBatchExample.java | 0
.../doris/flink/example/DorisSinkExample.java | 0
.../example/DorisSinkExampleRowData.java | 0
.../example/DorisSinkMultiTableExample.java | 0
.../flink/example/DorisSinkSQLExample.java | 0
.../DorisSinkStreamMultiTableExample.java | 0
.../flink/example/DorisSourceExample.java | 0
.../flink/example/DorisSourceSinkExample.java | 0
.../flink/example/LookupJoinCdcExample.java | 0
.../flink/example/LookupJoinExample.java | 0
.../doris/flink/sink/DorisSinkTest.java | 0
.../flink/sink/batch/TestDorisBatchSink.java | 0
.../flink/source/DorisSourceExampleTest.java | 0
.../source/DorisSourceITCaseForOldApi.java | 51 --
.../table/DorisDynamicTableFactoryTest.java | 9 +-
.../table/DorisDynamicTableSourceTest.java | 0
.../apache/doris/flink/utils/CatalogUtil.java | 44 ++
.../flink/utils/MockMultiTableSource.java | 2 +-
.../apache/doris/flink/utils/MockSource.java | 3 +-
.../doris/flink/utils/MockSourceFunction.java | 5 +
.../flink-doris-connector-2.2/pom.xml | 292 +--------
.../doris/flink/catalog/DorisCatalog.java | 580 ++++++++++++++++++
.../flink/catalog/DorisCatalogFactory.java | 131 ++++
.../flink/datastream/DorisSourceFunction.java | 118 ++++
.../apache/doris/flink/sink/DorisSink.java | 230 +++++++
.../flink/sink/batch/DorisBatchSink.java | 116 ++++
.../sink/batch/DorisBatchWriterAdapter.java | 88 +++
.../sink/copy/DorisCopyWriterAdapter.java | 71 +++
.../sink/writer/DorisAbstractWriter.java | 25 +
.../flink/sink/writer/DorisWriterAdapter.java | 97 +++
.../flink/table/DorisAsyncLookupAdapter.java | 96 +++
.../flink/table/DorisDynamicTableFactory.java | 306 +++++++++
.../flink/table/DorisDynamicTableSink.java | 198 ++++++
.../flink/table/DorisDynamicTableSource.java | 258 ++++++++
.../flink/table/DorisJdbcLookupAdapter.java | 100 +++
.../apache/doris/flink/utils/CatalogUtil.java | 52 ++
.../flink/utils/MockMultiTableSource.java | 148 +++++
.../apache/doris/flink/utils/MockSource.java | 115 ++++
.../doris/flink/utils/MockSourceFunction.java | 5 +
.../doris/flink/catalog/DorisTypeMapper.java | 5 +-
.../flink/cfg/DorisConnectionOptions.java | 1 +
.../doris/flink/cfg/DorisReadOptions.java | 1 +
.../flink/sink/batch/BatchRecordBuffer.java | 20 +-
.../flink/sink/batch/DorisBatchWriter.java | 34 +-
.../flink/sink/copy/BatchRecordBuffer.java | 20 +-
.../flink/sink/copy/DorisCopyWriter.java | 22 +-
.../doris/flink/sink/writer/DorisWriter.java | 30 +-
.../table/DorisRowDataJdbcLookupFunction.java | 18 +-
.../tools/cdc/postgres/PostgresType.java | 42 +-
.../sink/batch/TestDorisBatchWriter.java | 8 +-
.../flink/sink/copy/TestDorisCopyWriter.java | 11 +-
.../flink/sink/writer/TestDorisWriter.java | 16 +-
.../flink-doris-connector-it/pom.xml | 127 ++++
.../flink/catalog/DorisCatalogITCase.java | 53 +-
.../flink/lookup/DorisLookupTableITCase.java | 4 +-
.../flink/sink/DorisSinkFailoverITCase.java | 0
.../doris/flink/sink/DorisSinkITCase.java | 17 +-
.../sink/DorisSinkMultiTblFailoverITCase.java | 4 +-
.../doris/flink/source/DorisSourceITCase.java | 47 +-
.../src/test/resources/log4j2-test.properties | 25 +
flink-doris-connector/pom.xml | 96 ++-
82 files changed, 3594 insertions(+), 585 deletions(-)
rename flink-doris-connector/{flink-doris-connector-base => flink-doris-connector-1.20}/src/main/java/org/apache/doris/flink/catalog/DorisCatalog.java (98%)
rename flink-doris-connector/{flink-doris-connector-base => flink-doris-connector-1.20}/src/main/java/org/apache/doris/flink/catalog/DorisCatalogFactory.java (100%)
rename flink-doris-connector/{flink-doris-connector-base => flink-doris-connector-1.20}/src/main/java/org/apache/doris/flink/sink/DorisSink.java (96%)
rename flink-doris-connector/{flink-doris-connector-base => flink-doris-connector-1.20}/src/main/java/org/apache/doris/flink/sink/batch/DorisBatchSink.java (97%)
create mode 100644 flink-doris-connector/flink-doris-connector-1.20/src/main/java/org/apache/doris/flink/sink/batch/DorisBatchWriterAdapter.java
create mode 100644 flink-doris-connector/flink-doris-connector-1.20/src/main/java/org/apache/doris/flink/sink/copy/DorisCopyWriterAdapter.java
rename flink-doris-connector/{flink-doris-connector-base => flink-doris-connector-1.20}/src/main/java/org/apache/doris/flink/sink/writer/DorisAbstractWriter.java (100%)
create mode 100644 flink-doris-connector/flink-doris-connector-1.20/src/main/java/org/apache/doris/flink/sink/writer/DorisWriterAdapter.java
create mode 100644 flink-doris-connector/flink-doris-connector-1.20/src/main/java/org/apache/doris/flink/table/DorisAsyncLookupFunctionAdapter.java
rename flink-doris-connector/{flink-doris-connector-base => flink-doris-connector-1.20}/src/main/java/org/apache/doris/flink/table/DorisDynamicTableFactory.java (100%)
rename flink-doris-connector/{flink-doris-connector-base => flink-doris-connector-1.20}/src/main/java/org/apache/doris/flink/table/DorisDynamicTableSink.java (100%)
rename flink-doris-connector/{flink-doris-connector-base => flink-doris-connector-1.20}/src/main/java/org/apache/doris/flink/table/DorisDynamicTableSource.java (100%)
rename flink-doris-connector/{flink-doris-connector-base => flink-doris-connector-1.20}/src/test/java/org/apache/doris/flink/catalog/DorisCatalogFactoryTest.java (100%)
rename flink-doris-connector/{flink-doris-connector-base => flink-doris-connector-1.20}/src/test/java/org/apache/doris/flink/catalog/DorisCatalogTest.java (91%)
rename flink-doris-connector/{flink-doris-connector-base => flink-doris-connector-1.20}/src/test/java/org/apache/doris/flink/example/CatalogExample.java (100%)
rename flink-doris-connector/{flink-doris-connector-base => flink-doris-connector-1.20}/src/test/java/org/apache/doris/flink/example/DataGen2DorisExample.java (100%)
rename flink-doris-connector/{flink-doris-connector-base => flink-doris-connector-1.20}/src/test/java/org/apache/doris/flink/example/DorisDateAndTimestampSqlTest.java (100%)
rename flink-doris-connector/{flink-doris-connector-base => flink-doris-connector-1.20}/src/test/java/org/apache/doris/flink/example/DorisIntranetAccessSinkExample.java (100%)
rename flink-doris-connector/{flink-doris-connector-base => flink-doris-connector-1.20}/src/test/java/org/apache/doris/flink/example/DorisSinkArraySQLExample.java (100%)
rename flink-doris-connector/{flink-doris-connector-base => flink-doris-connector-1.20}/src/test/java/org/apache/doris/flink/example/DorisSinkBatchExample.java (100%)
rename flink-doris-connector/{flink-doris-connector-base => flink-doris-connector-1.20}/src/test/java/org/apache/doris/flink/example/DorisSinkExample.java (100%)
rename flink-doris-connector/{flink-doris-connector-base => flink-doris-connector-1.20}/src/test/java/org/apache/doris/flink/example/DorisSinkExampleRowData.java (100%)
rename flink-doris-connector/{flink-doris-connector-base => flink-doris-connector-1.20}/src/test/java/org/apache/doris/flink/example/DorisSinkMultiTableExample.java (100%)
rename flink-doris-connector/{flink-doris-connector-base => flink-doris-connector-1.20}/src/test/java/org/apache/doris/flink/example/DorisSinkSQLExample.java (100%)
rename flink-doris-connector/{flink-doris-connector-base => flink-doris-connector-1.20}/src/test/java/org/apache/doris/flink/example/DorisSinkStreamMultiTableExample.java (100%)
rename flink-doris-connector/{flink-doris-connector-base => flink-doris-connector-1.20}/src/test/java/org/apache/doris/flink/example/DorisSourceExample.java (100%)
rename flink-doris-connector/{flink-doris-connector-base => flink-doris-connector-1.20}/src/test/java/org/apache/doris/flink/example/DorisSourceSinkExample.java (100%)
rename flink-doris-connector/{flink-doris-connector-base => flink-doris-connector-1.20}/src/test/java/org/apache/doris/flink/example/LookupJoinCdcExample.java (100%)
rename flink-doris-connector/{flink-doris-connector-base => flink-doris-connector-1.20}/src/test/java/org/apache/doris/flink/example/LookupJoinExample.java (100%)
rename flink-doris-connector/{flink-doris-connector-base => flink-doris-connector-1.20}/src/test/java/org/apache/doris/flink/sink/DorisSinkTest.java (100%)
rename flink-doris-connector/{flink-doris-connector-base => flink-doris-connector-1.20}/src/test/java/org/apache/doris/flink/sink/batch/TestDorisBatchSink.java (100%)
rename flink-doris-connector/{flink-doris-connector-base => flink-doris-connector-1.20}/src/test/java/org/apache/doris/flink/source/DorisSourceExampleTest.java (100%)
delete mode 100644 flink-doris-connector/flink-doris-connector-1.20/src/test/java/org/apache/doris/flink/source/DorisSourceITCaseForOldApi.java
rename flink-doris-connector/{flink-doris-connector-base => flink-doris-connector-1.20}/src/test/java/org/apache/doris/flink/table/DorisDynamicTableFactoryTest.java (96%)
rename flink-doris-connector/{flink-doris-connector-base => flink-doris-connector-1.20}/src/test/java/org/apache/doris/flink/table/DorisDynamicTableSourceTest.java (100%)
create mode 100644 flink-doris-connector/flink-doris-connector-1.20/src/test/java/org/apache/doris/flink/utils/CatalogUtil.java
rename flink-doris-connector/{flink-doris-connector-base => flink-doris-connector-1.20}/src/test/java/org/apache/doris/flink/utils/MockMultiTableSource.java (98%)
rename flink-doris-connector/{flink-doris-connector-base => flink-doris-connector-1.20}/src/test/java/org/apache/doris/flink/utils/MockSource.java (96%)
create mode 100644 flink-doris-connector/flink-doris-connector-1.20/src/test/java/org/apache/doris/flink/utils/MockSourceFunction.java
create mode 100644 flink-doris-connector/flink-doris-connector-2.2/src/main/java/org/apache/doris/flink/catalog/DorisCatalog.java
create mode 100644 flink-doris-connector/flink-doris-connector-2.2/src/main/java/org/apache/doris/flink/catalog/DorisCatalogFactory.java
create mode 100644 flink-doris-connector/flink-doris-connector-2.2/src/main/java/org/apache/doris/flink/datastream/DorisSourceFunction.java
create mode 100644 flink-doris-connector/flink-doris-connector-2.2/src/main/java/org/apache/doris/flink/sink/DorisSink.java
create mode 100644 flink-doris-connector/flink-doris-connector-2.2/src/main/java/org/apache/doris/flink/sink/batch/DorisBatchSink.java
create mode 100644 flink-doris-connector/flink-doris-connector-2.2/src/main/java/org/apache/doris/flink/sink/batch/DorisBatchWriterAdapter.java
create mode 100644 flink-doris-connector/flink-doris-connector-2.2/src/main/java/org/apache/doris/flink/sink/copy/DorisCopyWriterAdapter.java
create mode 100644 flink-doris-connector/flink-doris-connector-2.2/src/main/java/org/apache/doris/flink/sink/writer/DorisAbstractWriter.java
create mode 100644 flink-doris-connector/flink-doris-connector-2.2/src/main/java/org/apache/doris/flink/sink/writer/DorisWriterAdapter.java
create mode 100644 flink-doris-connector/flink-doris-connector-2.2/src/main/java/org/apache/doris/flink/table/DorisAsyncLookupAdapter.java
create mode 100644 flink-doris-connector/flink-doris-connector-2.2/src/main/java/org/apache/doris/flink/table/DorisDynamicTableFactory.java
create mode 100644 flink-doris-connector/flink-doris-connector-2.2/src/main/java/org/apache/doris/flink/table/DorisDynamicTableSink.java
create mode 100644 flink-doris-connector/flink-doris-connector-2.2/src/main/java/org/apache/doris/flink/table/DorisDynamicTableSource.java
create mode 100644 flink-doris-connector/flink-doris-connector-2.2/src/main/java/org/apache/doris/flink/table/DorisJdbcLookupAdapter.java
create mode 100644 flink-doris-connector/flink-doris-connector-2.2/src/test/java/org/apache/doris/flink/utils/CatalogUtil.java
create mode 100644 flink-doris-connector/flink-doris-connector-2.2/src/test/java/org/apache/doris/flink/utils/MockMultiTableSource.java
create mode 100644 flink-doris-connector/flink-doris-connector-2.2/src/test/java/org/apache/doris/flink/utils/MockSource.java
create mode 100644 flink-doris-connector/flink-doris-connector-2.2/src/test/java/org/apache/doris/flink/utils/MockSourceFunction.java
create mode 100644 flink-doris-connector/flink-doris-connector-it/pom.xml
rename flink-doris-connector/{flink-doris-connector-base => flink-doris-connector-it}/src/test/java/org/apache/doris/flink/catalog/DorisCatalogITCase.java (88%)
rename flink-doris-connector/{flink-doris-connector-base => flink-doris-connector-it}/src/test/java/org/apache/doris/flink/lookup/DorisLookupTableITCase.java (99%)
rename flink-doris-connector/{flink-doris-connector-base => flink-doris-connector-it}/src/test/java/org/apache/doris/flink/sink/DorisSinkFailoverITCase.java (100%)
rename flink-doris-connector/{flink-doris-connector-base => flink-doris-connector-it}/src/test/java/org/apache/doris/flink/sink/DorisSinkITCase.java (97%)
rename flink-doris-connector/{flink-doris-connector-base => flink-doris-connector-it}/src/test/java/org/apache/doris/flink/sink/DorisSinkMultiTblFailoverITCase.java (99%)
rename flink-doris-connector/{flink-doris-connector-base => flink-doris-connector-it}/src/test/java/org/apache/doris/flink/source/DorisSourceITCase.java (95%)
create mode 100644 flink-doris-connector/flink-doris-connector-it/src/test/resources/log4j2-test.properties
diff --git a/.github/workflows/build-connector.yml b/.github/workflows/build-connector.yml
index 06a3b4481..fdf147e5a 100644
--- a/.github/workflows/build-connector.yml
+++ b/.github/workflows/build-connector.yml
@@ -83,5 +83,4 @@ jobs:
cd flink-doris-connector && mvn clean package \
-pl flink-doris-connector-1.20 -am \
-Dflink.version=1.20.0 \
- -Dflink.major.version=1.20 \
- -Dflink.python.id=flink-python
\ No newline at end of file
+ -Dflink.major.version=1.20
\ No newline at end of file
diff --git a/flink-doris-connector/flink-doris-connector-1.20/pom.xml b/flink-doris-connector/flink-doris-connector-1.20/pom.xml
index b7f81b4ea..0663914c5 100644
--- a/flink-doris-connector/flink-doris-connector-1.20/pom.xml
+++ b/flink-doris-connector/flink-doris-connector-1.20/pom.xml
@@ -176,6 +176,13 @@ under the License.
org.apache.flinkflink-connector-test-utils
+
+ org.apache.flink
+ flink-table-common
+ ${flink.version}
+ test-jar
+ test
+
@@ -201,6 +208,57 @@ under the License.
+
+ org.apache.maven.plugins
+ maven-shade-plugin
+ 3.4.1
+
+
+
+ org.apache.arrow
+ org.apache.doris.shaded.org.apache.arrow
+
+
+ io.netty
+ org.apache.doris.shaded.io.netty
+
+
+ com.fasterxml.jackson
+ org.apache.doris.shaded.com.fasterxml.jackson
+
+
+ org.apache.commons.codec
+ org.apache.doris.shaded.org.apache.commons.codec
+
+
+ com.google
+ org.apache.doris.shaded.com.google
+
+
+ org.apache.thrift
+ org.apache.doris.shaded.org.apache.thrift
+
+
+
+
+ *:*
+
+ META-INF/*.SF
+ META-INF/*.DSA
+ META-INF/*.RSA
+
+
+
+
+
+
+
+ shade
+
+ package
+
+
+
diff --git a/flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/catalog/DorisCatalog.java b/flink-doris-connector/flink-doris-connector-1.20/src/main/java/org/apache/doris/flink/catalog/DorisCatalog.java
similarity index 98%
rename from flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/catalog/DorisCatalog.java
rename to flink-doris-connector/flink-doris-connector-1.20/src/main/java/org/apache/doris/flink/catalog/DorisCatalog.java
index ca7dc1146..ed40695bc 100644
--- a/flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/catalog/DorisCatalog.java
+++ b/flink-doris-connector/flink-doris-connector-1.20/src/main/java/org/apache/doris/flink/catalog/DorisCatalog.java
@@ -404,8 +404,10 @@ public List listPartitions(ObjectPath tablePath)
@Override
public List listPartitions(
ObjectPath tablePath, CatalogPartitionSpec partitionSpec)
- throws TableNotExistException, TableNotPartitionedException,
- PartitionSpecInvalidException, CatalogException {
+ throws TableNotExistException,
+ TableNotPartitionedException,
+ PartitionSpecInvalidException,
+ CatalogException {
return Collections.emptyList();
}
@@ -434,8 +436,10 @@ public void createPartition(
CatalogPartitionSpec partitionSpec,
CatalogPartition partition,
boolean ignoreIfExists)
- throws TableNotExistException, TableNotPartitionedException,
- PartitionSpecInvalidException, PartitionAlreadyExistsException,
+ throws TableNotExistException,
+ TableNotPartitionedException,
+ PartitionSpecInvalidException,
+ PartitionAlreadyExistsException,
CatalogException {
throw new UnsupportedOperationException();
}
diff --git a/flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/catalog/DorisCatalogFactory.java b/flink-doris-connector/flink-doris-connector-1.20/src/main/java/org/apache/doris/flink/catalog/DorisCatalogFactory.java
similarity index 100%
rename from flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/catalog/DorisCatalogFactory.java
rename to flink-doris-connector/flink-doris-connector-1.20/src/main/java/org/apache/doris/flink/catalog/DorisCatalogFactory.java
diff --git a/flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/sink/DorisSink.java b/flink-doris-connector/flink-doris-connector-1.20/src/main/java/org/apache/doris/flink/sink/DorisSink.java
similarity index 96%
rename from flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/sink/DorisSink.java
rename to flink-doris-connector/flink-doris-connector-1.20/src/main/java/org/apache/doris/flink/sink/DorisSink.java
index d8e0d8277..d5eda431c 100644
--- a/flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/sink/DorisSink.java
+++ b/flink-doris-connector/flink-doris-connector-1.20/src/main/java/org/apache/doris/flink/sink/DorisSink.java
@@ -29,13 +29,14 @@
import org.apache.doris.flink.cfg.DorisOptions;
import org.apache.doris.flink.cfg.DorisReadOptions;
import org.apache.doris.flink.rest.RestService;
-import org.apache.doris.flink.sink.batch.DorisBatchWriter;
+import org.apache.doris.flink.sink.batch.DorisBatchWriterAdapter;
import org.apache.doris.flink.sink.committer.DorisCommitter;
import org.apache.doris.flink.sink.copy.CopyCommittableSerializer;
import org.apache.doris.flink.sink.copy.DorisCopyCommitter;
-import org.apache.doris.flink.sink.copy.DorisCopyWriter;
+import org.apache.doris.flink.sink.copy.DorisCopyWriterAdapter;
import org.apache.doris.flink.sink.writer.DorisAbstractWriter;
import org.apache.doris.flink.sink.writer.DorisWriter;
+import org.apache.doris.flink.sink.writer.DorisWriterAdapter;
import org.apache.doris.flink.sink.writer.DorisWriterState;
import org.apache.doris.flink.sink.writer.DorisWriterStateSerializer;
import org.apache.doris.flink.sink.writer.WriteMode;
@@ -111,7 +112,7 @@ public DorisAbstractWriter restoreWriter(
public DorisAbstractWriter getDorisAbstractWriter(
InitContext initContext, Collection states) {
if (WriteMode.STREAM_LOAD.equals(dorisExecutionOptions.getWriteMode())) {
- return new DorisWriter<>(
+ return new DorisWriterAdapter(
initContext,
states,
serializer,
@@ -119,10 +120,10 @@ public DorisAbstractWriter getDorisAbstractWriter(
dorisReadOptions,
dorisExecutionOptions);
} else if (WriteMode.STREAM_LOAD_BATCH.equals(dorisExecutionOptions.getWriteMode())) {
- return new DorisBatchWriter<>(
+ return new DorisBatchWriterAdapter(
initContext, serializer, dorisOptions, dorisReadOptions, dorisExecutionOptions);
} else if (WriteMode.COPY.equals(dorisExecutionOptions.getWriteMode())) {
- return new DorisCopyWriter(
+ return new DorisCopyWriterAdapter(
initContext, serializer, dorisOptions, dorisReadOptions, dorisExecutionOptions);
}
throw new IllegalArgumentException(
diff --git a/flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/sink/batch/DorisBatchSink.java b/flink-doris-connector/flink-doris-connector-1.20/src/main/java/org/apache/doris/flink/sink/batch/DorisBatchSink.java
similarity index 97%
rename from flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/sink/batch/DorisBatchSink.java
rename to flink-doris-connector/flink-doris-connector-1.20/src/main/java/org/apache/doris/flink/sink/batch/DorisBatchSink.java
index 993283d89..970466fdf 100644
--- a/flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/sink/batch/DorisBatchSink.java
+++ b/flink-doris-connector/flink-doris-connector-1.20/src/main/java/org/apache/doris/flink/sink/batch/DorisBatchSink.java
@@ -51,8 +51,8 @@ public DorisBatchSink(
@Override
public SinkWriter createWriter(InitContext initContext) throws IOException {
- DorisBatchWriter dorisBatchWriter =
- new DorisBatchWriter(
+ DorisBatchWriterAdapter dorisBatchWriter =
+ new DorisBatchWriterAdapter(
initContext,
serializer,
dorisOptions,
diff --git a/flink-doris-connector/flink-doris-connector-1.20/src/main/java/org/apache/doris/flink/sink/batch/DorisBatchWriterAdapter.java b/flink-doris-connector/flink-doris-connector-1.20/src/main/java/org/apache/doris/flink/sink/batch/DorisBatchWriterAdapter.java
new file mode 100644
index 000000000..2ec1578ee
--- /dev/null
+++ b/flink-doris-connector/flink-doris-connector-1.20/src/main/java/org/apache/doris/flink/sink/batch/DorisBatchWriterAdapter.java
@@ -0,0 +1,88 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under this work is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.flink.sink.batch;
+
+import org.apache.flink.api.connector.sink2.Sink;
+import org.apache.flink.runtime.checkpoint.CheckpointIDCounter;
+
+import org.apache.doris.flink.cfg.DorisExecutionOptions;
+import org.apache.doris.flink.cfg.DorisOptions;
+import org.apache.doris.flink.cfg.DorisReadOptions;
+import org.apache.doris.flink.sink.DorisCommittable;
+import org.apache.doris.flink.sink.writer.DorisAbstractWriter;
+import org.apache.doris.flink.sink.writer.DorisWriterState;
+import org.apache.doris.flink.sink.writer.serializer.DorisRecordSerializer;
+
+import java.io.IOException;
+import java.util.Collection;
+import java.util.List;
+
+/** Flink 1.20 specific wrapper for the shared core {@link DorisBatchWriter} implementation. */
+public class DorisBatchWriterAdapter
+ implements DorisAbstractWriter {
+
+ private final DorisBatchWriter delegate;
+
+ public DorisBatchWriterAdapter(
+ Sink.InitContext initContext,
+ DorisRecordSerializer serializer,
+ DorisOptions dorisOptions,
+ DorisReadOptions dorisReadOptions,
+ DorisExecutionOptions executionOptions) {
+
+ long restoreCheckpointId =
+ initContext
+ .getRestoredCheckpointId()
+ .orElse(CheckpointIDCounter.INITIAL_CHECKPOINT_ID - 1);
+ int subtaskId = initContext.getSubtaskId();
+
+ this.delegate =
+ new DorisBatchWriter<>(
+ restoreCheckpointId,
+ subtaskId,
+ serializer,
+ dorisOptions,
+ dorisReadOptions,
+ executionOptions);
+ }
+
+ @Override
+ public void write(IN in, Context context) throws IOException, InterruptedException {
+ delegate.write(in, context);
+ }
+
+ @Override
+ public void flush(boolean endOfInput) throws IOException, InterruptedException {
+ delegate.flush(endOfInput);
+ }
+
+ @Override
+ public Collection prepareCommit() throws IOException, InterruptedException {
+ return delegate.prepareCommit();
+ }
+
+ @Override
+ public List snapshotState(long checkpointId) throws IOException {
+ return delegate.snapshotState(checkpointId);
+ }
+
+ @Override
+ public void close() throws Exception {
+ delegate.close();
+ }
+}
diff --git a/flink-doris-connector/flink-doris-connector-1.20/src/main/java/org/apache/doris/flink/sink/copy/DorisCopyWriterAdapter.java b/flink-doris-connector/flink-doris-connector-1.20/src/main/java/org/apache/doris/flink/sink/copy/DorisCopyWriterAdapter.java
new file mode 100644
index 000000000..85c033f70
--- /dev/null
+++ b/flink-doris-connector/flink-doris-connector-1.20/src/main/java/org/apache/doris/flink/sink/copy/DorisCopyWriterAdapter.java
@@ -0,0 +1,71 @@
+package org.apache.doris.flink.sink.copy;
+
+import org.apache.flink.api.connector.sink2.Sink;
+import org.apache.flink.runtime.checkpoint.CheckpointIDCounter;
+
+import org.apache.doris.flink.cfg.DorisExecutionOptions;
+import org.apache.doris.flink.cfg.DorisOptions;
+import org.apache.doris.flink.cfg.DorisReadOptions;
+import org.apache.doris.flink.sink.writer.DorisAbstractWriter;
+import org.apache.doris.flink.sink.writer.DorisWriterState;
+import org.apache.doris.flink.sink.writer.serializer.DorisRecordSerializer;
+
+import java.io.IOException;
+import java.util.Collection;
+import java.util.List;
+
+/** Flink 1.20 specific wrapper for the shared core {@link DorisCopyWriter} implementation. */
+public class DorisCopyWriterAdapter
+ implements DorisAbstractWriter {
+
+ private final DorisCopyWriter delegate;
+
+ public DorisCopyWriterAdapter(
+ Sink.InitContext initContext,
+ DorisRecordSerializer serializer,
+ DorisOptions dorisOptions,
+ DorisReadOptions dorisReadOptions,
+ DorisExecutionOptions executionOptions) {
+
+ long restoreCheckpointId =
+ initContext
+ .getRestoredCheckpointId()
+ .orElse(CheckpointIDCounter.INITIAL_CHECKPOINT_ID - 1);
+ int subtaskId = initContext.getSubtaskId();
+
+ this.delegate =
+ new DorisCopyWriter<>(
+ restoreCheckpointId,
+ subtaskId,
+ serializer,
+ dorisOptions,
+ dorisReadOptions,
+ executionOptions);
+ }
+
+ @Override
+ public void write(IN in, Context context) throws IOException, InterruptedException {
+ delegate.write(in, context);
+ }
+
+ @Override
+ public void flush(boolean endOfInput) throws IOException, InterruptedException {
+ delegate.flush(endOfInput);
+ }
+
+ @Override
+ public Collection prepareCommit()
+ throws IOException, InterruptedException {
+ return delegate.prepareCommit();
+ }
+
+ @Override
+ public List snapshotState(long checkpointId) throws IOException {
+ return delegate.snapshotState(checkpointId);
+ }
+
+ @Override
+ public void close() throws Exception {
+ delegate.close();
+ }
+}
diff --git a/flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/sink/writer/DorisAbstractWriter.java b/flink-doris-connector/flink-doris-connector-1.20/src/main/java/org/apache/doris/flink/sink/writer/DorisAbstractWriter.java
similarity index 100%
rename from flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/sink/writer/DorisAbstractWriter.java
rename to flink-doris-connector/flink-doris-connector-1.20/src/main/java/org/apache/doris/flink/sink/writer/DorisAbstractWriter.java
diff --git a/flink-doris-connector/flink-doris-connector-1.20/src/main/java/org/apache/doris/flink/sink/writer/DorisWriterAdapter.java b/flink-doris-connector/flink-doris-connector-1.20/src/main/java/org/apache/doris/flink/sink/writer/DorisWriterAdapter.java
new file mode 100644
index 000000000..25646ccc8
--- /dev/null
+++ b/flink-doris-connector/flink-doris-connector-1.20/src/main/java/org/apache/doris/flink/sink/writer/DorisWriterAdapter.java
@@ -0,0 +1,97 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.flink.sink.writer;
+
+import org.apache.flink.api.connector.sink2.Sink;
+import org.apache.flink.metrics.groups.SinkWriterMetricGroup;
+import org.apache.flink.runtime.checkpoint.CheckpointIDCounter;
+
+import org.apache.doris.flink.cfg.DorisExecutionOptions;
+import org.apache.doris.flink.cfg.DorisOptions;
+import org.apache.doris.flink.cfg.DorisReadOptions;
+import org.apache.doris.flink.sink.DorisCommittable;
+import org.apache.doris.flink.sink.writer.serializer.DorisRecordSerializer;
+
+import java.io.IOException;
+import java.util.Collection;
+import java.util.List;
+
+/**
+ * Flink 1.20 specific wrapper for the shared core {@link DorisWriter} implementation.
+ *
+ *
This class adapts Flink 1.20's {@link Sink.InitContext} to the version-neutral core writer in
+ * the base module while implementing {@link DorisAbstractWriter} for use by the 1.20 connector
+ * module.
+ */
+public class DorisWriterAdapter
+ implements DorisAbstractWriter {
+
+ private final DorisWriter delegate;
+
+ public DorisWriterAdapter(
+ Sink.InitContext initContext,
+ Collection state,
+ DorisRecordSerializer serializer,
+ DorisOptions dorisOptions,
+ DorisReadOptions dorisReadOptions,
+ DorisExecutionOptions executionOptions) {
+
+ long lastCheckpointId =
+ initContext
+ .getRestoredCheckpointId()
+ .orElse(CheckpointIDCounter.INITIAL_CHECKPOINT_ID - 1);
+ int subtaskId = initContext.getSubtaskId();
+ SinkWriterMetricGroup sinkMetricGroup = initContext.metricGroup();
+
+ this.delegate =
+ new DorisWriter<>(
+ lastCheckpointId,
+ subtaskId,
+ sinkMetricGroup,
+ state,
+ serializer,
+ dorisOptions,
+ dorisReadOptions,
+ executionOptions);
+ }
+
+ @Override
+ public void write(IN in, Context context) throws IOException, InterruptedException {
+ delegate.write(in, context);
+ }
+
+ @Override
+ public void flush(boolean endOfInput) throws IOException, InterruptedException {
+ delegate.flush(endOfInput);
+ }
+
+ @Override
+ public Collection prepareCommit() throws IOException, InterruptedException {
+ return delegate.prepareCommit();
+ }
+
+ @Override
+ public List snapshotState(long checkpointId) throws IOException {
+ return delegate.snapshotState(checkpointId);
+ }
+
+ @Override
+ public void close() throws Exception {
+ delegate.close();
+ }
+}
diff --git a/flink-doris-connector/flink-doris-connector-1.20/src/main/java/org/apache/doris/flink/table/DorisAsyncLookupFunctionAdapter.java b/flink-doris-connector/flink-doris-connector-1.20/src/main/java/org/apache/doris/flink/table/DorisAsyncLookupFunctionAdapter.java
new file mode 100644
index 000000000..b55af389a
--- /dev/null
+++ b/flink-doris-connector/flink-doris-connector-1.20/src/main/java/org/apache/doris/flink/table/DorisAsyncLookupFunctionAdapter.java
@@ -0,0 +1,88 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.flink.table;
+
+import org.apache.flink.table.data.RowData;
+import org.apache.flink.table.functions.AsyncLookupFunction;
+import org.apache.flink.table.functions.FunctionContext;
+
+import java.io.IOException;
+import java.util.Collection;
+import java.util.concurrent.CompletableFuture;
+
+/**
+ * Adapter class to bridge DorisRowDataAsyncLookupFunction (extends AsyncTableFunction) to
+ * AsyncLookupFunction required by Flink 1.16+.
+ *
+ *
This adapter delegates all calls to the base DorisRowDataAsyncLookupFunction implementation
+ * without duplicating the core lookup logic, ensuring compatibility across Flink 1.15-1.20.
+ */
+public class DorisAsyncLookupFunctionAdapter extends AsyncLookupFunction {
+
+ private final DorisRowDataAsyncLookupFunction delegateFunction;
+ private final int[] keyIndex;
+
+ public DorisAsyncLookupFunctionAdapter(
+ DorisRowDataAsyncLookupFunction delegateFunction, int[] keyIndex) {
+ this.delegateFunction = delegateFunction;
+ this.keyIndex = keyIndex;
+ }
+
+ @Override
+ public void open(FunctionContext context) throws Exception {
+ delegateFunction.open(context);
+ }
+
+ @Override
+ public CompletableFuture> asyncLookup(RowData keyRow) {
+ CompletableFuture> future = new CompletableFuture<>();
+ try {
+ // Extract key values from RowData based on keyIndex
+ Object[] keys = new Object[keyIndex.length];
+ for (int i = 0; i < keyIndex.length; i++) {
+ // Note: The actual field extraction depends on the field type
+ // For simplicity, we assume RowData.getField exists in the implementation
+ // If not, you may need to use RowData field getters based on field types
+ keys[i] = extractField(keyRow, keyIndex[i]);
+ }
+ delegateFunction.eval(future, keys);
+ } catch (IOException e) {
+ future.completeExceptionally(e);
+ }
+ return future;
+ }
+
+ /**
+ * Extract field value from RowData at the specified position. This is a simplified
+ * implementation - you may need to adjust based on actual field types.
+ */
+ private Object extractField(RowData row, int pos) {
+ if (row.isNullAt(pos)) {
+ return null;
+ }
+ // Note: You might need to handle different field types here
+ // For now, we use a generic approach that should work for most cases
+ // In practice, you might need to check field types and use appropriate getters
+ return row;
+ }
+
+ @Override
+ public void close() throws Exception {
+ delegateFunction.close();
+ }
+}
diff --git a/flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/table/DorisDynamicTableFactory.java b/flink-doris-connector/flink-doris-connector-1.20/src/main/java/org/apache/doris/flink/table/DorisDynamicTableFactory.java
similarity index 100%
rename from flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/table/DorisDynamicTableFactory.java
rename to flink-doris-connector/flink-doris-connector-1.20/src/main/java/org/apache/doris/flink/table/DorisDynamicTableFactory.java
diff --git a/flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/table/DorisDynamicTableSink.java b/flink-doris-connector/flink-doris-connector-1.20/src/main/java/org/apache/doris/flink/table/DorisDynamicTableSink.java
similarity index 100%
rename from flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/table/DorisDynamicTableSink.java
rename to flink-doris-connector/flink-doris-connector-1.20/src/main/java/org/apache/doris/flink/table/DorisDynamicTableSink.java
diff --git a/flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/table/DorisDynamicTableSource.java b/flink-doris-connector/flink-doris-connector-1.20/src/main/java/org/apache/doris/flink/table/DorisDynamicTableSource.java
similarity index 100%
rename from flink-doris-connector/flink-doris-connector-base/src/main/java/org/apache/doris/flink/table/DorisDynamicTableSource.java
rename to flink-doris-connector/flink-doris-connector-1.20/src/main/java/org/apache/doris/flink/table/DorisDynamicTableSource.java
diff --git a/flink-doris-connector/flink-doris-connector-base/src/test/java/org/apache/doris/flink/catalog/DorisCatalogFactoryTest.java b/flink-doris-connector/flink-doris-connector-1.20/src/test/java/org/apache/doris/flink/catalog/DorisCatalogFactoryTest.java
similarity index 100%
rename from flink-doris-connector/flink-doris-connector-base/src/test/java/org/apache/doris/flink/catalog/DorisCatalogFactoryTest.java
rename to flink-doris-connector/flink-doris-connector-1.20/src/test/java/org/apache/doris/flink/catalog/DorisCatalogFactoryTest.java
diff --git a/flink-doris-connector/flink-doris-connector-base/src/test/java/org/apache/doris/flink/catalog/DorisCatalogTest.java b/flink-doris-connector/flink-doris-connector-1.20/src/test/java/org/apache/doris/flink/catalog/DorisCatalogTest.java
similarity index 91%
rename from flink-doris-connector/flink-doris-connector-base/src/test/java/org/apache/doris/flink/catalog/DorisCatalogTest.java
rename to flink-doris-connector/flink-doris-connector-1.20/src/test/java/org/apache/doris/flink/catalog/DorisCatalogTest.java
index d21ad7513..5fd4268e2 100644
--- a/flink-doris-connector/flink-doris-connector-base/src/test/java/org/apache/doris/flink/catalog/DorisCatalogTest.java
+++ b/flink-doris-connector/flink-doris-connector-1.20/src/test/java/org/apache/doris/flink/catalog/DorisCatalogTest.java
@@ -35,8 +35,10 @@ public class DorisCatalogTest {
@Before
public void setup()
- throws DatabaseAlreadyExistException, TableAlreadyExistException,
- TableNotExistException, DatabaseNotExistException {
+ throws DatabaseAlreadyExistException,
+ TableAlreadyExistException,
+ TableNotExistException,
+ DatabaseNotExistException {
DorisConnectionOptions connectionOptions =
new DorisConnectionOptions.DorisConnectionOptionsBuilder()
.withFenodes("127.0.0.1:8030")
diff --git a/flink-doris-connector/flink-doris-connector-base/src/test/java/org/apache/doris/flink/example/CatalogExample.java b/flink-doris-connector/flink-doris-connector-1.20/src/test/java/org/apache/doris/flink/example/CatalogExample.java
similarity index 100%
rename from flink-doris-connector/flink-doris-connector-base/src/test/java/org/apache/doris/flink/example/CatalogExample.java
rename to flink-doris-connector/flink-doris-connector-1.20/src/test/java/org/apache/doris/flink/example/CatalogExample.java
diff --git a/flink-doris-connector/flink-doris-connector-base/src/test/java/org/apache/doris/flink/example/DataGen2DorisExample.java b/flink-doris-connector/flink-doris-connector-1.20/src/test/java/org/apache/doris/flink/example/DataGen2DorisExample.java
similarity index 100%
rename from flink-doris-connector/flink-doris-connector-base/src/test/java/org/apache/doris/flink/example/DataGen2DorisExample.java
rename to flink-doris-connector/flink-doris-connector-1.20/src/test/java/org/apache/doris/flink/example/DataGen2DorisExample.java
diff --git a/flink-doris-connector/flink-doris-connector-base/src/test/java/org/apache/doris/flink/example/DorisDateAndTimestampSqlTest.java b/flink-doris-connector/flink-doris-connector-1.20/src/test/java/org/apache/doris/flink/example/DorisDateAndTimestampSqlTest.java
similarity index 100%
rename from flink-doris-connector/flink-doris-connector-base/src/test/java/org/apache/doris/flink/example/DorisDateAndTimestampSqlTest.java
rename to flink-doris-connector/flink-doris-connector-1.20/src/test/java/org/apache/doris/flink/example/DorisDateAndTimestampSqlTest.java
diff --git a/flink-doris-connector/flink-doris-connector-base/src/test/java/org/apache/doris/flink/example/DorisIntranetAccessSinkExample.java b/flink-doris-connector/flink-doris-connector-1.20/src/test/java/org/apache/doris/flink/example/DorisIntranetAccessSinkExample.java
similarity index 100%
rename from flink-doris-connector/flink-doris-connector-base/src/test/java/org/apache/doris/flink/example/DorisIntranetAccessSinkExample.java
rename to flink-doris-connector/flink-doris-connector-1.20/src/test/java/org/apache/doris/flink/example/DorisIntranetAccessSinkExample.java
diff --git a/flink-doris-connector/flink-doris-connector-base/src/test/java/org/apache/doris/flink/example/DorisSinkArraySQLExample.java b/flink-doris-connector/flink-doris-connector-1.20/src/test/java/org/apache/doris/flink/example/DorisSinkArraySQLExample.java
similarity index 100%
rename from flink-doris-connector/flink-doris-connector-base/src/test/java/org/apache/doris/flink/example/DorisSinkArraySQLExample.java
rename to flink-doris-connector/flink-doris-connector-1.20/src/test/java/org/apache/doris/flink/example/DorisSinkArraySQLExample.java
diff --git a/flink-doris-connector/flink-doris-connector-base/src/test/java/org/apache/doris/flink/example/DorisSinkBatchExample.java b/flink-doris-connector/flink-doris-connector-1.20/src/test/java/org/apache/doris/flink/example/DorisSinkBatchExample.java
similarity index 100%
rename from flink-doris-connector/flink-doris-connector-base/src/test/java/org/apache/doris/flink/example/DorisSinkBatchExample.java
rename to flink-doris-connector/flink-doris-connector-1.20/src/test/java/org/apache/doris/flink/example/DorisSinkBatchExample.java
diff --git a/flink-doris-connector/flink-doris-connector-base/src/test/java/org/apache/doris/flink/example/DorisSinkExample.java b/flink-doris-connector/flink-doris-connector-1.20/src/test/java/org/apache/doris/flink/example/DorisSinkExample.java
similarity index 100%
rename from flink-doris-connector/flink-doris-connector-base/src/test/java/org/apache/doris/flink/example/DorisSinkExample.java
rename to flink-doris-connector/flink-doris-connector-1.20/src/test/java/org/apache/doris/flink/example/DorisSinkExample.java
diff --git a/flink-doris-connector/flink-doris-connector-base/src/test/java/org/apache/doris/flink/example/DorisSinkExampleRowData.java b/flink-doris-connector/flink-doris-connector-1.20/src/test/java/org/apache/doris/flink/example/DorisSinkExampleRowData.java
similarity index 100%
rename from flink-doris-connector/flink-doris-connector-base/src/test/java/org/apache/doris/flink/example/DorisSinkExampleRowData.java
rename to flink-doris-connector/flink-doris-connector-1.20/src/test/java/org/apache/doris/flink/example/DorisSinkExampleRowData.java
diff --git a/flink-doris-connector/flink-doris-connector-base/src/test/java/org/apache/doris/flink/example/DorisSinkMultiTableExample.java b/flink-doris-connector/flink-doris-connector-1.20/src/test/java/org/apache/doris/flink/example/DorisSinkMultiTableExample.java
similarity index 100%
rename from flink-doris-connector/flink-doris-connector-base/src/test/java/org/apache/doris/flink/example/DorisSinkMultiTableExample.java
rename to flink-doris-connector/flink-doris-connector-1.20/src/test/java/org/apache/doris/flink/example/DorisSinkMultiTableExample.java
diff --git a/flink-doris-connector/flink-doris-connector-base/src/test/java/org/apache/doris/flink/example/DorisSinkSQLExample.java b/flink-doris-connector/flink-doris-connector-1.20/src/test/java/org/apache/doris/flink/example/DorisSinkSQLExample.java
similarity index 100%
rename from flink-doris-connector/flink-doris-connector-base/src/test/java/org/apache/doris/flink/example/DorisSinkSQLExample.java
rename to flink-doris-connector/flink-doris-connector-1.20/src/test/java/org/apache/doris/flink/example/DorisSinkSQLExample.java
diff --git a/flink-doris-connector/flink-doris-connector-base/src/test/java/org/apache/doris/flink/example/DorisSinkStreamMultiTableExample.java b/flink-doris-connector/flink-doris-connector-1.20/src/test/java/org/apache/doris/flink/example/DorisSinkStreamMultiTableExample.java
similarity index 100%
rename from flink-doris-connector/flink-doris-connector-base/src/test/java/org/apache/doris/flink/example/DorisSinkStreamMultiTableExample.java
rename to flink-doris-connector/flink-doris-connector-1.20/src/test/java/org/apache/doris/flink/example/DorisSinkStreamMultiTableExample.java
diff --git a/flink-doris-connector/flink-doris-connector-base/src/test/java/org/apache/doris/flink/example/DorisSourceExample.java b/flink-doris-connector/flink-doris-connector-1.20/src/test/java/org/apache/doris/flink/example/DorisSourceExample.java
similarity index 100%
rename from flink-doris-connector/flink-doris-connector-base/src/test/java/org/apache/doris/flink/example/DorisSourceExample.java
rename to flink-doris-connector/flink-doris-connector-1.20/src/test/java/org/apache/doris/flink/example/DorisSourceExample.java
diff --git a/flink-doris-connector/flink-doris-connector-base/src/test/java/org/apache/doris/flink/example/DorisSourceSinkExample.java b/flink-doris-connector/flink-doris-connector-1.20/src/test/java/org/apache/doris/flink/example/DorisSourceSinkExample.java
similarity index 100%
rename from flink-doris-connector/flink-doris-connector-base/src/test/java/org/apache/doris/flink/example/DorisSourceSinkExample.java
rename to flink-doris-connector/flink-doris-connector-1.20/src/test/java/org/apache/doris/flink/example/DorisSourceSinkExample.java
diff --git a/flink-doris-connector/flink-doris-connector-base/src/test/java/org/apache/doris/flink/example/LookupJoinCdcExample.java b/flink-doris-connector/flink-doris-connector-1.20/src/test/java/org/apache/doris/flink/example/LookupJoinCdcExample.java
similarity index 100%
rename from flink-doris-connector/flink-doris-connector-base/src/test/java/org/apache/doris/flink/example/LookupJoinCdcExample.java
rename to flink-doris-connector/flink-doris-connector-1.20/src/test/java/org/apache/doris/flink/example/LookupJoinCdcExample.java
diff --git a/flink-doris-connector/flink-doris-connector-base/src/test/java/org/apache/doris/flink/example/LookupJoinExample.java b/flink-doris-connector/flink-doris-connector-1.20/src/test/java/org/apache/doris/flink/example/LookupJoinExample.java
similarity index 100%
rename from flink-doris-connector/flink-doris-connector-base/src/test/java/org/apache/doris/flink/example/LookupJoinExample.java
rename to flink-doris-connector/flink-doris-connector-1.20/src/test/java/org/apache/doris/flink/example/LookupJoinExample.java
diff --git a/flink-doris-connector/flink-doris-connector-base/src/test/java/org/apache/doris/flink/sink/DorisSinkTest.java b/flink-doris-connector/flink-doris-connector-1.20/src/test/java/org/apache/doris/flink/sink/DorisSinkTest.java
similarity index 100%
rename from flink-doris-connector/flink-doris-connector-base/src/test/java/org/apache/doris/flink/sink/DorisSinkTest.java
rename to flink-doris-connector/flink-doris-connector-1.20/src/test/java/org/apache/doris/flink/sink/DorisSinkTest.java
diff --git a/flink-doris-connector/flink-doris-connector-base/src/test/java/org/apache/doris/flink/sink/batch/TestDorisBatchSink.java b/flink-doris-connector/flink-doris-connector-1.20/src/test/java/org/apache/doris/flink/sink/batch/TestDorisBatchSink.java
similarity index 100%
rename from flink-doris-connector/flink-doris-connector-base/src/test/java/org/apache/doris/flink/sink/batch/TestDorisBatchSink.java
rename to flink-doris-connector/flink-doris-connector-1.20/src/test/java/org/apache/doris/flink/sink/batch/TestDorisBatchSink.java
diff --git a/flink-doris-connector/flink-doris-connector-base/src/test/java/org/apache/doris/flink/source/DorisSourceExampleTest.java b/flink-doris-connector/flink-doris-connector-1.20/src/test/java/org/apache/doris/flink/source/DorisSourceExampleTest.java
similarity index 100%
rename from flink-doris-connector/flink-doris-connector-base/src/test/java/org/apache/doris/flink/source/DorisSourceExampleTest.java
rename to flink-doris-connector/flink-doris-connector-1.20/src/test/java/org/apache/doris/flink/source/DorisSourceExampleTest.java
diff --git a/flink-doris-connector/flink-doris-connector-1.20/src/test/java/org/apache/doris/flink/source/DorisSourceITCaseForOldApi.java b/flink-doris-connector/flink-doris-connector-1.20/src/test/java/org/apache/doris/flink/source/DorisSourceITCaseForOldApi.java
deleted file mode 100644
index fd625117e..000000000
--- a/flink-doris-connector/flink-doris-connector-1.20/src/test/java/org/apache/doris/flink/source/DorisSourceITCaseForOldApi.java
+++ /dev/null
@@ -1,51 +0,0 @@
-package org.apache.doris.flink.source;
-
-import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
-import org.apache.flink.util.CloseableIterator;
-
-import org.apache.doris.flink.catalog.doris.DataModel;
-import org.apache.doris.flink.cfg.DorisStreamOptions;
-import org.apache.doris.flink.container.AbstractITCaseService;
-import org.apache.doris.flink.datastream.DorisSourceFunction;
-import org.apache.doris.flink.deserialization.SimpleListDeserializationSchema;
-import org.junit.Test;
-
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.List;
-import java.util.Properties;
-
-import static org.apache.doris.flink.source.DorisSourceITCase.checkResultInAnyOrder;
-import static org.apache.doris.flink.source.DorisSourceITCase.initializeTable;
-
-public class DorisSourceITCaseForOldApi extends AbstractITCaseService {
-
- private static final String DATABASE = "test_source";
- public static final String TABLE_READ_OLD_API = "tbl_read_old_api";
-
- @Test
- public void testOldSourceApi() throws Exception {
- initializeTable(TABLE_READ_OLD_API, DataModel.UNIQUE);
- StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
- env.setParallelism(DEFAULT_PARALLELISM);
- Properties properties = new Properties();
- properties.put("fenodes", getFenodes());
- properties.put("username", getDorisUsername());
- properties.put("password", getDorisPassword());
- properties.put("table.identifier", DATABASE + "." + TABLE_READ_OLD_API);
- DorisStreamOptions options = new DorisStreamOptions(properties);
-
- List actual = new ArrayList<>();
- try (CloseableIterator> iterator =
- env.addSource(
- new DorisSourceFunction(
- options, new SimpleListDeserializationSchema()))
- .executeAndCollect()) {
- while (iterator.hasNext()) {
- actual.add(iterator.next().toString());
- }
- }
- List expected = Arrays.asList("[doris, 18]", "[flink, 10]", "[apache, 12]");
- checkResultInAnyOrder("testOldSourceApi", expected.toArray(), actual.toArray());
- }
-}
diff --git a/flink-doris-connector/flink-doris-connector-base/src/test/java/org/apache/doris/flink/table/DorisDynamicTableFactoryTest.java b/flink-doris-connector/flink-doris-connector-1.20/src/test/java/org/apache/doris/flink/table/DorisDynamicTableFactoryTest.java
similarity index 96%
rename from flink-doris-connector/flink-doris-connector-base/src/test/java/org/apache/doris/flink/table/DorisDynamicTableFactoryTest.java
rename to flink-doris-connector/flink-doris-connector-1.20/src/test/java/org/apache/doris/flink/table/DorisDynamicTableFactoryTest.java
index a3c210975..87c2f5205 100644
--- a/flink-doris-connector/flink-doris-connector-base/src/test/java/org/apache/doris/flink/table/DorisDynamicTableFactoryTest.java
+++ b/flink-doris-connector/flink-doris-connector-1.20/src/test/java/org/apache/doris/flink/table/DorisDynamicTableFactoryTest.java
@@ -20,6 +20,7 @@
import org.apache.flink.table.api.TableSchema;
import org.apache.flink.table.connector.sink.DynamicTableSink;
import org.apache.flink.table.connector.source.DynamicTableSource;
+import org.apache.flink.table.factories.utils.FactoryMocks;
import org.apache.doris.flink.cfg.DorisExecutionOptions;
import org.apache.doris.flink.cfg.DorisLookupOptions;
@@ -44,8 +45,6 @@
import static org.apache.doris.flink.cfg.ConfigurationOptions.FLIGHT_SQL_PORT_DEFAULT;
import static org.apache.doris.flink.cfg.ConfigurationOptions.USE_FLIGHT_SQL_DEFAULT;
import static org.apache.doris.flink.utils.FactoryMocks.SCHEMA;
-import static org.apache.flink.table.factories.utils.FactoryMocks.createTableSink;
-import static org.apache.flink.table.factories.utils.FactoryMocks.createTableSource;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNotEquals;
@@ -71,7 +70,7 @@ public void testDorisSourceProperties() {
properties.put("lookup.jdbc.read.thread-size", "1");
properties.put("source.use-flight-sql", "true");
properties.put("source.flight-sql-port", "-1");
- DynamicTableSource actual = createTableSource(SCHEMA, properties);
+ DynamicTableSource actual = FactoryMocks.createTableSource(SCHEMA, properties);
DorisOptions options =
DorisOptions.builder()
.setTableIdentifier("db.tbl")
@@ -147,7 +146,7 @@ public void testDorisSinkProperties() {
properties.put("sink.ignore.commit-error", "false");
properties.put("sink.parallelism", "1");
- DynamicTableSink actual = createTableSink(SCHEMA, properties);
+ DynamicTableSink actual = FactoryMocks.createTableSink(SCHEMA, properties);
DorisOptions options =
DorisOptions.builder()
.setTableIdentifier("db.tbl")
@@ -238,7 +237,7 @@ public void testDorisSinkProperties() {
assertNotEquals(actual, expected4);
executionOptions.setEnable2PC(true);
- DynamicTableSink actual2 = createTableSink(SCHEMA, new HashMap<>());
+ DynamicTableSink actual2 = FactoryMocks.createTableSink(SCHEMA, new HashMap<>());
assertNotEquals(actual2, expected);
}
diff --git a/flink-doris-connector/flink-doris-connector-base/src/test/java/org/apache/doris/flink/table/DorisDynamicTableSourceTest.java b/flink-doris-connector/flink-doris-connector-1.20/src/test/java/org/apache/doris/flink/table/DorisDynamicTableSourceTest.java
similarity index 100%
rename from flink-doris-connector/flink-doris-connector-base/src/test/java/org/apache/doris/flink/table/DorisDynamicTableSourceTest.java
rename to flink-doris-connector/flink-doris-connector-1.20/src/test/java/org/apache/doris/flink/table/DorisDynamicTableSourceTest.java
diff --git a/flink-doris-connector/flink-doris-connector-1.20/src/test/java/org/apache/doris/flink/utils/CatalogUtil.java b/flink-doris-connector/flink-doris-connector-1.20/src/test/java/org/apache/doris/flink/utils/CatalogUtil.java
new file mode 100644
index 000000000..9a890ab51
--- /dev/null
+++ b/flink-doris-connector/flink-doris-connector-1.20/src/test/java/org/apache/doris/flink/utils/CatalogUtil.java
@@ -0,0 +1,44 @@
+package org.apache.doris.flink.utils;
+
+import org.apache.flink.table.api.DataTypes;
+import org.apache.flink.table.api.TableSchema;
+import org.apache.flink.table.catalog.CatalogTable;
+import org.apache.flink.table.catalog.CatalogTableImpl;
+import org.apache.flink.table.types.AtomicDataType;
+import org.apache.flink.table.types.logical.VarCharType;
+
+import java.util.Map;
+
+public class CatalogUtil {
+ public static CatalogTable createTable(TableSchema tableSchema, Map options) {
+ return new CatalogTableImpl(tableSchema, options, "FlinkTable");
+ }
+
+ public static TableSchema getTableSchema() {
+ return TableSchema.builder()
+ .field("id", new AtomicDataType(new VarCharType(false, 128)))
+ .field("c_boolean", DataTypes.BOOLEAN())
+ .field("c_char", DataTypes.CHAR(1))
+ .field("c_date", DataTypes.DATE())
+ .field("c_datetime", DataTypes.TIMESTAMP(0))
+ .field("c_decimal", DataTypes.DECIMAL(10, 2))
+ .field("c_double", DataTypes.DOUBLE())
+ .field("c_float", DataTypes.FLOAT())
+ .field("c_int", DataTypes.INT())
+ .field("c_bigint", DataTypes.BIGINT())
+ .field("c_largeint", DataTypes.STRING())
+ .field("c_smallint", DataTypes.SMALLINT())
+ .field("c_string", DataTypes.STRING())
+ .field("c_tinyint", DataTypes.TINYINT())
+ .field("c_array", DataTypes.ARRAY(DataTypes.INT()))
+ .field("c_map", DataTypes.MAP(DataTypes.STRING(), DataTypes.STRING()))
+ .field("c_row", DataTypes.ROW())
+ .field("c_varbinary", DataTypes.VARBINARY(16))
+ .primaryKey("id")
+ .build();
+ }
+
+ public static String[] getColumns() {
+ return getTableSchema().getFieldNames();
+ }
+}
diff --git a/flink-doris-connector/flink-doris-connector-base/src/test/java/org/apache/doris/flink/utils/MockMultiTableSource.java b/flink-doris-connector/flink-doris-connector-1.20/src/test/java/org/apache/doris/flink/utils/MockMultiTableSource.java
similarity index 98%
rename from flink-doris-connector/flink-doris-connector-base/src/test/java/org/apache/doris/flink/utils/MockMultiTableSource.java
rename to flink-doris-connector/flink-doris-connector-1.20/src/test/java/org/apache/doris/flink/utils/MockMultiTableSource.java
index 287275eaf..ad37cda75 100644
--- a/flink-doris-connector/flink-doris-connector-base/src/test/java/org/apache/doris/flink/utils/MockMultiTableSource.java
+++ b/flink-doris-connector/flink-doris-connector-1.20/src/test/java/org/apache/doris/flink/utils/MockMultiTableSource.java
@@ -71,7 +71,7 @@ public MockMultiTableSource(
@Override
public void run(SourceContext ctx) throws Exception {
- int taskId = getRuntimeContext().getIndexOfThisSubtask();
+ int taskId = getRuntimeContext().getTaskInfo().getIndexOfThisSubtask();
while (this.running && id < this.numEventsTotal) {
id = id + 1;
for (int i = 1; i <= tableNum; i++) {
diff --git a/flink-doris-connector/flink-doris-connector-base/src/test/java/org/apache/doris/flink/utils/MockSource.java b/flink-doris-connector/flink-doris-connector-1.20/src/test/java/org/apache/doris/flink/utils/MockSource.java
similarity index 96%
rename from flink-doris-connector/flink-doris-connector-base/src/test/java/org/apache/doris/flink/utils/MockSource.java
rename to flink-doris-connector/flink-doris-connector-1.20/src/test/java/org/apache/doris/flink/utils/MockSource.java
index b7e83196d..1fd4dfebd 100644
--- a/flink-doris-connector/flink-doris-connector-base/src/test/java/org/apache/doris/flink/utils/MockSource.java
+++ b/flink-doris-connector/flink-doris-connector-1.20/src/test/java/org/apache/doris/flink/utils/MockSource.java
@@ -25,6 +25,7 @@
import org.apache.flink.runtime.state.FunctionSnapshotContext;
import org.apache.flink.streaming.api.checkpoint.CheckpointedFunction;
import org.apache.flink.streaming.api.functions.source.RichParallelSourceFunction;
+import org.apache.flink.streaming.api.functions.source.SourceFunction.SourceContext;
import org.apache.flink.util.SerializableObject;
import org.slf4j.Logger;
@@ -60,7 +61,7 @@ public MockSource(int numEventsTotal, int failCheckpointId) {
@Override
public void run(SourceContext ctx) throws Exception {
- int taskId = getRuntimeContext().getIndexOfThisSubtask();
+ int taskId = getRuntimeContext().getTaskInfo().getIndexOfThisSubtask();
while (this.running && id < this.numEventsTotal) {
String record = ++id + "," + taskId;
ctx.collect(record);
diff --git a/flink-doris-connector/flink-doris-connector-1.20/src/test/java/org/apache/doris/flink/utils/MockSourceFunction.java b/flink-doris-connector/flink-doris-connector-1.20/src/test/java/org/apache/doris/flink/utils/MockSourceFunction.java
new file mode 100644
index 000000000..c864d2b92
--- /dev/null
+++ b/flink-doris-connector/flink-doris-connector-1.20/src/test/java/org/apache/doris/flink/utils/MockSourceFunction.java
@@ -0,0 +1,5 @@
+package org.apache.doris.flink.utils;
+
+import org.apache.flink.streaming.api.functions.source.SourceFunction;
+
+public interface MockSourceFunction extends SourceFunction {}
diff --git a/flink-doris-connector/flink-doris-connector-2.2/pom.xml b/flink-doris-connector/flink-doris-connector-2.2/pom.xml
index 7ca88d838..0a67b40e2 100644
--- a/flink-doris-connector/flink-doris-connector-2.2/pom.xml
+++ b/flink-doris-connector/flink-doris-connector-2.2/pom.xml
@@ -35,242 +35,24 @@ under the License.
2.2.02.2flink-python
-
- 17
- 17org.apache.dorisflink-doris-connector-base
- ${revision}
- flink-2.2
+
org.apache.doris
- thrift-service
-
-
- org.apache.flink
- flink-table-planner-loader
- ${flink.version}
- provided
-
-
- org.apache.flink
- flink-connector-base
- ${flink.version}
- provided
-
-
- org.apache.flink
- flink-table-api-java-bridge
- ${flink.version}
- provided
-
-
- org.apache.flink
- flink-table-runtime
- ${flink.version}
- provided
-
-
- org.apache.flink
- ${flink.python.id}
- ${flink.version}
- provided
-
-
- org.apache.thrift
- libthrift
-
-
- httpclient
- org.apache.httpcomponents
-
-
- httpcore
- org.apache.httpcomponents
-
-
-
-
- org.apache.httpcomponents
- httpclient
-
-
- commons-codec
- commons-codec
-
-
-
-
- commons-codec
- commons-codec
-
-
- org.apache.arrow.adbc
- adbc-driver-flight-sql
-
-
- org.apache.arrow
- flight-sql-jdbc-core
-
-
- org.apache.arrow
- flight-sql-jdbc-driver
-
-
- org.apache.arrow
- arrow-vector
-
-
- org.apache.arrow.adbc
- adbc-driver-manager
-
-
-
-
- org.apache.arrow
- arrow-vector
-
-
- org.apache.arrow
- arrow-memory-netty
- runtime
-
-
- com.fasterxml.jackson.core
- jackson-annotations
-
-
- com.fasterxml.jackson.core
- jackson-core
-
-
- com.fasterxml.jackson.core
- jackson-databind
-
-
-
-
- com.fasterxml.jackson.core
- jackson-annotations
-
-
- com.fasterxml.jackson.core
- jackson-core
-
-
- com.fasterxml.jackson.core
- jackson-databind
-
-
- com.google.guava
- guava
-
-
- org.apache.logging.log4j
- log4j-slf4j-impl
- ${log4j.version}
- test
-
-
- org.apache.logging.log4j
- log4j-api
- ${log4j.version}
- test
-
-
- org.apache.logging.log4j
- log4j-core
- ${log4j.version}
- test
-
-
- org.apache.logging.log4j
- log4j-1.2-api
- ${log4j.version}
- test
-
-
-
- mysql
- mysql-connector-java
- ${mysql.driver.version}
- test
-
-
- com.oracle.ojdbc
- ojdbc8
- ${ojdbc.version}
- provided
-
-
- org.apache.flink
- flink-runtime-web
- ${flink.version}
- test
-
-
- org.hamcrest
- hamcrest-core
- ${hamcrest.version}
- test
-
-
- org.mockito
- mockito-core
- ${mockito.version}
- test
-
-
- org.mockito
- mockito-inline
- ${mockito.version}
- test
-
-
- junit
- junit
- ${junit.version}
- test
-
-
- org.testcontainers
- testcontainers
- ${testcontainers.version}
- test
-
-
- org.testcontainers
- mysql
- ${testcontainers.version}
- test
-
-
- org.apache.flink
- flink-table-common
- ${flink.version}
- test-jar
- test
-
-
- org.apache.flink
- flink-test-utils
- ${flink.version}
+ flink-doris-connector-base
+ ${revision}
+ teststestorg.apache.flink
- flink-connector-test-utils
- ${flink.version}
- test
-
-
- com.github.jsqlparser
- jsqlparser
+ flink-table-runtime
@@ -279,70 +61,6 @@ under the License.
org.apache.maven.pluginsmaven-compiler-plugin
-
- 17
- 17
-
-
-
- org.apache.maven.plugins
- maven-shade-plugin
- 3.4.1
-
-
-
- org.apache.arrow
- org.apache.doris.shaded.org.apache.arrow
-
-
- io.netty
- org.apache.doris.shaded.io.netty
-
-
- com.fasterxml.jackson
- org.apache.doris.shaded.com.fasterxml.jackson
-
-
- org.apache.commons.codec
- org.apache.doris.shaded.org.apache.commons.codec
-
-
- com.google
- org.apache.doris.shaded.com.google
-
-
- org.apache.thrift
- org.apache.doris.shaded.org.apache.thrift
-
-
-
-
- *:*
-
- META-INF/*.SF
- META-INF/*.DSA
- META-INF/*.RSA
-
-
-
-
-
-
-
- shade
-
- package
-
-
-
-
- org.apache.maven.plugins
- maven-checkstyle-plugin
-
- ../../tools/maven/suppressions.xml
- ../../tools/maven/checkstyle.xml
- true
- org.apache.maven.plugins
diff --git a/flink-doris-connector/flink-doris-connector-2.2/src/main/java/org/apache/doris/flink/catalog/DorisCatalog.java b/flink-doris-connector/flink-doris-connector-2.2/src/main/java/org/apache/doris/flink/catalog/DorisCatalog.java
new file mode 100644
index 000000000..5381b38d1
--- /dev/null
+++ b/flink-doris-connector/flink-doris-connector-2.2/src/main/java/org/apache/doris/flink/catalog/DorisCatalog.java
@@ -0,0 +1,580 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.flink.catalog;
+
+import org.apache.flink.annotation.VisibleForTesting;
+import org.apache.flink.table.api.Schema;
+import org.apache.flink.table.catalog.AbstractCatalog;
+import org.apache.flink.table.catalog.CatalogBaseTable;
+import org.apache.flink.table.catalog.CatalogDatabase;
+import org.apache.flink.table.catalog.CatalogDatabaseImpl;
+import org.apache.flink.table.catalog.CatalogFunction;
+import org.apache.flink.table.catalog.CatalogPartition;
+import org.apache.flink.table.catalog.CatalogPartitionSpec;
+import org.apache.flink.table.catalog.CatalogTable;
+import org.apache.flink.table.catalog.ObjectPath;
+import org.apache.flink.table.catalog.exceptions.CatalogException;
+import org.apache.flink.table.catalog.exceptions.DatabaseAlreadyExistException;
+import org.apache.flink.table.catalog.exceptions.DatabaseNotEmptyException;
+import org.apache.flink.table.catalog.exceptions.DatabaseNotExistException;
+import org.apache.flink.table.catalog.exceptions.FunctionAlreadyExistException;
+import org.apache.flink.table.catalog.exceptions.FunctionNotExistException;
+import org.apache.flink.table.catalog.exceptions.PartitionAlreadyExistsException;
+import org.apache.flink.table.catalog.exceptions.PartitionNotExistException;
+import org.apache.flink.table.catalog.exceptions.PartitionSpecInvalidException;
+import org.apache.flink.table.catalog.exceptions.TableAlreadyExistException;
+import org.apache.flink.table.catalog.exceptions.TableNotExistException;
+import org.apache.flink.table.catalog.exceptions.TableNotPartitionedException;
+import org.apache.flink.table.catalog.exceptions.TablePartitionedException;
+import org.apache.flink.table.catalog.stats.CatalogColumnStatistics;
+import org.apache.flink.table.catalog.stats.CatalogTableStatistics;
+import org.apache.flink.table.expressions.Expression;
+import org.apache.flink.table.factories.Factory;
+import org.apache.flink.table.types.DataType;
+import org.apache.flink.util.Preconditions;
+import org.apache.flink.util.StringUtils;
+
+import org.apache.commons.compress.utils.Lists;
+import org.apache.doris.flink.catalog.doris.DorisSchemaFactory;
+import org.apache.doris.flink.catalog.doris.DorisSystem;
+import org.apache.doris.flink.catalog.doris.FieldSchema;
+import org.apache.doris.flink.catalog.doris.TableSchema;
+import org.apache.doris.flink.cfg.DorisConnectionOptions;
+import org.apache.doris.flink.table.DorisDynamicTableFactory;
+import org.apache.doris.flink.tools.cdc.DorisTableConfig;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.sql.Connection;
+import java.sql.DriverManager;
+import java.sql.PreparedStatement;
+import java.sql.ResultSet;
+import java.sql.ResultSetMetaData;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+import java.util.StringJoiner;
+
+import static org.apache.doris.flink.catalog.DorisCatalogOptions.DEFAULT_DATABASE;
+import static org.apache.doris.flink.catalog.DorisCatalogOptions.getCreateTableProps;
+import static org.apache.doris.flink.table.DorisConfigOptions.FENODES;
+import static org.apache.doris.flink.table.DorisConfigOptions.IDENTIFIER;
+import static org.apache.doris.flink.table.DorisConfigOptions.PASSWORD;
+import static org.apache.doris.flink.table.DorisConfigOptions.SINK_LABEL_PREFIX;
+import static org.apache.doris.flink.table.DorisConfigOptions.TABLE_IDENTIFIER;
+import static org.apache.doris.flink.table.DorisConfigOptions.USERNAME;
+import static org.apache.flink.table.factories.FactoryUtil.CONNECTOR;
+import static org.apache.flink.util.Preconditions.checkArgument;
+import static org.apache.flink.util.Preconditions.checkNotNull;
+
+/** catalog for flink. */
+public class DorisCatalog extends AbstractCatalog {
+
+ private static final Logger LOG = LoggerFactory.getLogger(DorisCatalog.class);
+ private DorisSystem dorisSystem;
+ private final DorisConnectionOptions connectionOptions;
+ private final Map properties;
+
+ public DorisCatalog(
+ String catalogName,
+ DorisConnectionOptions connectionOptions,
+ String defaultDatabase,
+ Map properties) {
+ super(catalogName, defaultDatabase);
+ this.connectionOptions = connectionOptions;
+ this.properties = Collections.unmodifiableMap(properties);
+ }
+
+ @Override
+ public void open() throws CatalogException {
+ dorisSystem = new DorisSystem(connectionOptions);
+ }
+
+ @Override
+ public synchronized void close() throws CatalogException {
+ try {
+ LOG.info("Closed catalog {} ", getName());
+ } catch (Exception e) {
+ throw new CatalogException(String.format("Closing catalog %s failed.", getName()), e);
+ }
+ }
+
+ @Override
+ public Optional getFactory() {
+ return Optional.of(new DorisDynamicTableFactory());
+ }
+
+ // ------------- databases -------------
+
+ @Override
+ public List listDatabases() throws CatalogException {
+ return dorisSystem.listDatabases();
+ }
+
+ @Override
+ public CatalogDatabase getDatabase(String databaseName)
+ throws DatabaseNotExistException, CatalogException {
+ if (databaseExists(databaseName)) {
+ return new CatalogDatabaseImpl(Collections.emptyMap(), null);
+ } else {
+ throw new DatabaseNotExistException(getName(), databaseName);
+ }
+ }
+
+ @Override
+ public boolean databaseExists(String databaseName) throws CatalogException {
+ checkArgument(!StringUtils.isNullOrWhitespaceOnly(databaseName));
+ return listDatabases().contains(databaseName);
+ }
+
+ @Override
+ public void createDatabase(String name, CatalogDatabase database, boolean ignoreIfExists)
+ throws DatabaseAlreadyExistException, CatalogException {
+ if (databaseExists(name)) {
+ if (ignoreIfExists) {
+ return;
+ }
+ throw new DatabaseAlreadyExistException(getName(), name);
+ } else {
+ dorisSystem.createDatabase(name);
+ }
+ }
+
+ @Override
+ public void dropDatabase(String name, boolean ignoreIfNotExists, boolean cascade)
+ throws DatabaseNotEmptyException, CatalogException, DatabaseNotExistException {
+ if (!databaseExists(name)) {
+ if (ignoreIfNotExists) {
+ return;
+ }
+ throw new DatabaseNotExistException(getName(), name);
+ }
+
+ if (!cascade && !listTables(name).isEmpty()) {
+ throw new DatabaseNotEmptyException(getName(), name);
+ }
+ dorisSystem.dropDatabase(name);
+ }
+
+ @Override
+ public void alterDatabase(String name, CatalogDatabase newDatabase, boolean ignoreIfNotExists)
+ throws DatabaseNotExistException, CatalogException {
+ throw new UnsupportedOperationException();
+ }
+
+ // ------------- tables -------------
+
+ @Override
+ public List listTables(String databaseName)
+ throws DatabaseNotExistException, CatalogException {
+ Preconditions.checkState(
+ org.apache.commons.lang3.StringUtils.isNotBlank(databaseName),
+ "Database name must not be blank.");
+ if (!databaseExists(databaseName)) {
+ throw new DatabaseNotExistException(getName(), databaseName);
+ }
+
+ return dorisSystem.extractColumnValuesBySQL(
+ "SELECT TABLE_NAME FROM information_schema.`TABLES` WHERE TABLE_SCHEMA = ?",
+ 1,
+ null,
+ databaseName);
+ }
+
+ @Override
+ public List listViews(String databaseName)
+ throws DatabaseNotExistException, CatalogException {
+ return Collections.emptyList();
+ }
+
+ @Override
+ public CatalogBaseTable getTable(ObjectPath tablePath)
+ throws TableNotExistException, CatalogException {
+ if (!tableExists(tablePath)) {
+ throw new TableNotExistException(getName(), tablePath);
+ }
+ String databaseName = tablePath.getDatabaseName();
+ String tableName = tablePath.getObjectName();
+ Map props = new HashMap<>(properties);
+ props.put(CONNECTOR.key(), IDENTIFIER);
+ if (!props.containsKey(FENODES.key())) {
+ props.put(FENODES.key(), queryFenodes());
+ }
+ props.put(USERNAME.key(), connectionOptions.getUsername());
+ props.put(PASSWORD.key(), connectionOptions.getPassword());
+ props.put(TABLE_IDENTIFIER.key(), databaseName + "." + tableName);
+
+ String labelPrefix = props.getOrDefault(SINK_LABEL_PREFIX.key(), "");
+ props.put(SINK_LABEL_PREFIX.key(), String.join("_", labelPrefix, databaseName, tableName));
+ // remove catalog option
+ props.remove(DEFAULT_DATABASE.key());
+ return CatalogTable.newBuilder()
+ .schema(createTableSchema(databaseName, tableName))
+ .comment(null)
+ .partitionKeys(Lists.newArrayList())
+ .options(props)
+ .build();
+ }
+
+ @VisibleForTesting
+ protected String queryFenodes() {
+ try (Connection conn =
+ DriverManager.getConnection(
+ connectionOptions.getJdbcUrl(),
+ connectionOptions.getUsername(),
+ connectionOptions.getPassword())) {
+ StringJoiner fenodes = new StringJoiner(",");
+ PreparedStatement ps = conn.prepareStatement("SHOW FRONTENDS");
+ ResultSet resultSet = ps.executeQuery();
+
+ // find target ip column name, Version 1.2 is IP, version 2.x is Host
+ String field = "";
+ ResultSetMetaData metaData = resultSet.getMetaData();
+ for (int i = 1; i <= metaData.getColumnCount(); i++) {
+ String columnName = metaData.getColumnName(i);
+ if (columnName.equalsIgnoreCase("IP") || columnName.equalsIgnoreCase("Host")) {
+ field = columnName;
+ break;
+ }
+ }
+
+ while (resultSet.next()) {
+ String ip = resultSet.getString(field);
+ String port = resultSet.getString("HttpPort");
+ fenodes.add(ip + ":" + port);
+ }
+ return fenodes.toString();
+ } catch (Exception e) {
+ throw new CatalogException("Failed getting fenodes", e);
+ }
+ }
+
+ private Schema createTableSchema(String databaseName, String tableName) {
+ try (Connection conn =
+ DriverManager.getConnection(
+ connectionOptions.getJdbcUrl(),
+ connectionOptions.getUsername(),
+ connectionOptions.getPassword())) {
+ PreparedStatement ps =
+ conn.prepareStatement(
+ String.format(
+ "SELECT COLUMN_NAME,DATA_TYPE,COLUMN_SIZE,DECIMAL_DIGITS FROM `information_schema`.`COLUMNS` WHERE `TABLE_SCHEMA`= '%s' AND `TABLE_NAME`= '%s'",
+ databaseName, tableName));
+
+ List columnNames = new ArrayList<>();
+ List columnTypes = new ArrayList<>();
+ ResultSet resultSet = ps.executeQuery();
+ while (resultSet.next()) {
+ String columnName = resultSet.getString("COLUMN_NAME");
+ String columnType = resultSet.getString("DATA_TYPE");
+ long columnSize = resultSet.getLong("COLUMN_SIZE");
+ long columnDigit = resultSet.getLong("DECIMAL_DIGITS");
+ DataType flinkType =
+ DorisTypeMapper.toFlinkType(
+ columnName, columnType, (int) columnSize, (int) columnDigit);
+ columnNames.add(columnName);
+ columnTypes.add(flinkType);
+ }
+ Schema.Builder schemaBuilder = Schema.newBuilder().fromFields(columnNames, columnTypes);
+ Schema tableSchema = schemaBuilder.build();
+ return tableSchema;
+ } catch (Exception e) {
+ throw new CatalogException(
+ String.format(
+ "Failed getting catalog %s database %s table %s",
+ getName(), databaseName, tableName),
+ e);
+ }
+ }
+
+ @Override
+ public boolean tableExists(ObjectPath tablePath) throws CatalogException {
+ try {
+ return databaseExists(tablePath.getDatabaseName())
+ && listTables(tablePath.getDatabaseName()).contains(tablePath.getObjectName());
+ } catch (DatabaseNotExistException e) {
+ return false;
+ }
+ }
+
+ @Override
+ public void dropTable(ObjectPath tablePath, boolean ignoreIfNotExists)
+ throws TableNotExistException, CatalogException {
+ if (!tableExists(tablePath)) {
+ if (ignoreIfNotExists) {
+ return;
+ }
+ throw new TableNotExistException(getName(), tablePath);
+ }
+
+ dorisSystem.dropTable(
+ String.format("%s.%s", tablePath.getDatabaseName(), tablePath.getObjectName()));
+ }
+
+ @Override
+ public void renameTable(ObjectPath tablePath, String newTableName, boolean ignoreIfNotExists)
+ throws TableNotExistException, TableAlreadyExistException, CatalogException {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public void createTable(ObjectPath tablePath, CatalogBaseTable table, boolean ignoreIfExists)
+ throws TableAlreadyExistException, DatabaseNotExistException, CatalogException {
+ checkNotNull(tablePath, "tablePath cannot be null");
+ checkNotNull(table, "table cannot be null");
+
+ if (!databaseExists(tablePath.getDatabaseName())) {
+ throw new DatabaseNotExistException(getName(), tablePath.getDatabaseName());
+ }
+ if (tableExists(tablePath)) {
+ if (ignoreIfExists) {
+ return;
+ }
+ throw new TableAlreadyExistException(getName(), tablePath);
+ }
+
+ Map options = table.getOptions();
+ if (!IDENTIFIER.equals(options.get(CONNECTOR.key()))) {
+ return;
+ }
+
+ List primaryKeys = getCreateDorisKeys(table.getSchema());
+ TableSchema schema =
+ DorisSchemaFactory.createTableSchema(
+ tablePath.getDatabaseName(),
+ tablePath.getObjectName(),
+ getCreateDorisColumns(table.getSchema()),
+ primaryKeys,
+ new DorisTableConfig(getCreateTableProps(options)),
+ table.getComment());
+
+ dorisSystem.createTable(schema);
+ }
+
+ public List getCreateDorisKeys(org.apache.flink.table.legacy.api.TableSchema schema) {
+ Preconditions.checkState(schema.getPrimaryKey().isPresent(), "primary key cannot be null");
+ return schema.getPrimaryKey().get().getColumns();
+ }
+
+ public Map getCreateDorisColumns(
+ org.apache.flink.table.legacy.api.TableSchema schema) {
+ String[] fieldNames = schema.getFieldNames();
+ DataType[] fieldTypes = schema.getFieldDataTypes();
+
+ Map fields = new LinkedHashMap<>();
+ for (int i = 0; i < fieldNames.length; i++) {
+ fields.put(
+ fieldNames[i],
+ new FieldSchema(
+ fieldNames[i], DorisTypeMapper.toDorisType(fieldTypes[i]), null));
+ }
+ return fields;
+ }
+
+ @Override
+ public void alterTable(
+ ObjectPath tablePath, CatalogBaseTable newTable, boolean ignoreIfNotExists)
+ throws TableNotExistException, CatalogException {
+ throw new UnsupportedOperationException();
+ }
+
+ // ------------- partitions -------------
+
+ @Override
+ public List listPartitions(ObjectPath tablePath)
+ throws TableNotExistException, TableNotPartitionedException, CatalogException {
+ return Collections.emptyList();
+ }
+
+ @Override
+ public List listPartitions(
+ ObjectPath tablePath, CatalogPartitionSpec partitionSpec)
+ throws TableNotExistException,
+ TableNotPartitionedException,
+ PartitionSpecInvalidException,
+ CatalogException {
+ return Collections.emptyList();
+ }
+
+ @Override
+ public List listPartitionsByFilter(
+ ObjectPath tablePath, List filters)
+ throws TableNotExistException, TableNotPartitionedException, CatalogException {
+ return Collections.emptyList();
+ }
+
+ @Override
+ public CatalogPartition getPartition(ObjectPath tablePath, CatalogPartitionSpec partitionSpec)
+ throws PartitionNotExistException, CatalogException {
+ throw new PartitionNotExistException(getName(), tablePath, partitionSpec);
+ }
+
+ @Override
+ public boolean partitionExists(ObjectPath tablePath, CatalogPartitionSpec partitionSpec)
+ throws CatalogException {
+ return false;
+ }
+
+ @Override
+ public void createPartition(
+ ObjectPath tablePath,
+ CatalogPartitionSpec partitionSpec,
+ CatalogPartition partition,
+ boolean ignoreIfExists)
+ throws TableNotExistException,
+ TableNotPartitionedException,
+ PartitionSpecInvalidException,
+ PartitionAlreadyExistsException,
+ CatalogException {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public void dropPartition(
+ ObjectPath tablePath, CatalogPartitionSpec partitionSpec, boolean ignoreIfNotExists)
+ throws PartitionNotExistException, CatalogException {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public void alterPartition(
+ ObjectPath tablePath,
+ CatalogPartitionSpec partitionSpec,
+ CatalogPartition newPartition,
+ boolean ignoreIfNotExists)
+ throws PartitionNotExistException, CatalogException {
+ throw new UnsupportedOperationException();
+ }
+
+ // ------------- functions -------------
+
+ @Override
+ public List listFunctions(String dbName)
+ throws DatabaseNotExistException, CatalogException {
+ return Collections.emptyList();
+ }
+
+ @Override
+ public CatalogFunction getFunction(ObjectPath functionPath)
+ throws FunctionNotExistException, CatalogException {
+ throw new FunctionNotExistException(getName(), functionPath);
+ }
+
+ @Override
+ public boolean functionExists(ObjectPath functionPath) throws CatalogException {
+ return false;
+ }
+
+ @Override
+ public void createFunction(
+ ObjectPath functionPath, CatalogFunction function, boolean ignoreIfExists)
+ throws FunctionAlreadyExistException, DatabaseNotExistException, CatalogException {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public void alterFunction(
+ ObjectPath functionPath, CatalogFunction newFunction, boolean ignoreIfNotExists)
+ throws FunctionNotExistException, CatalogException {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public void dropFunction(ObjectPath functionPath, boolean ignoreIfNotExists)
+ throws FunctionNotExistException, CatalogException {
+ throw new UnsupportedOperationException();
+ }
+
+ // ------------- statistics -------------
+
+ @Override
+ public CatalogTableStatistics getTableStatistics(ObjectPath tablePath)
+ throws TableNotExistException, CatalogException {
+ return CatalogTableStatistics.UNKNOWN;
+ }
+
+ @Override
+ public CatalogColumnStatistics getTableColumnStatistics(ObjectPath tablePath)
+ throws TableNotExistException, CatalogException {
+ return CatalogColumnStatistics.UNKNOWN;
+ }
+
+ @Override
+ public CatalogTableStatistics getPartitionStatistics(
+ ObjectPath tablePath, CatalogPartitionSpec partitionSpec)
+ throws PartitionNotExistException, CatalogException {
+ return CatalogTableStatistics.UNKNOWN;
+ }
+
+ @Override
+ public CatalogColumnStatistics getPartitionColumnStatistics(
+ ObjectPath tablePath, CatalogPartitionSpec partitionSpec)
+ throws PartitionNotExistException, CatalogException {
+ return CatalogColumnStatistics.UNKNOWN;
+ }
+
+ @Override
+ public void alterTableStatistics(
+ ObjectPath tablePath, CatalogTableStatistics tableStatistics, boolean ignoreIfNotExists)
+ throws TableNotExistException, CatalogException {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public void alterTableColumnStatistics(
+ ObjectPath tablePath,
+ CatalogColumnStatistics columnStatistics,
+ boolean ignoreIfNotExists)
+ throws TableNotExistException, CatalogException, TablePartitionedException {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public void alterPartitionStatistics(
+ ObjectPath tablePath,
+ CatalogPartitionSpec partitionSpec,
+ CatalogTableStatistics partitionStatistics,
+ boolean ignoreIfNotExists)
+ throws PartitionNotExistException, CatalogException {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public void alterPartitionColumnStatistics(
+ ObjectPath tablePath,
+ CatalogPartitionSpec partitionSpec,
+ CatalogColumnStatistics columnStatistics,
+ boolean ignoreIfNotExists)
+ throws PartitionNotExistException, CatalogException {
+ throw new UnsupportedOperationException();
+ }
+
+ @VisibleForTesting
+ public DorisConnectionOptions getConnectionOptions() {
+ return connectionOptions;
+ }
+
+ @VisibleForTesting
+ public Map getProperties() {
+ return properties;
+ }
+}
diff --git a/flink-doris-connector/flink-doris-connector-2.2/src/main/java/org/apache/doris/flink/catalog/DorisCatalogFactory.java b/flink-doris-connector/flink-doris-connector-2.2/src/main/java/org/apache/doris/flink/catalog/DorisCatalogFactory.java
new file mode 100644
index 000000000..30fc8e5c3
--- /dev/null
+++ b/flink-doris-connector/flink-doris-connector-2.2/src/main/java/org/apache/doris/flink/catalog/DorisCatalogFactory.java
@@ -0,0 +1,131 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.flink.catalog;
+
+import org.apache.flink.configuration.ConfigOption;
+import org.apache.flink.configuration.Configuration;
+import org.apache.flink.table.catalog.Catalog;
+import org.apache.flink.table.factories.CatalogFactory;
+import org.apache.flink.table.factories.FactoryUtil;
+
+import org.apache.doris.flink.cfg.DorisConnectionOptions;
+
+import java.util.HashSet;
+import java.util.Set;
+
+import static org.apache.doris.flink.catalog.DorisCatalogOptions.DEFAULT_DATABASE;
+import static org.apache.doris.flink.catalog.DorisCatalogOptions.TABLE_PROPERTIES_PREFIX;
+import static org.apache.doris.flink.table.DorisConfigOptions.DORIS_BATCH_SIZE;
+import static org.apache.doris.flink.table.DorisConfigOptions.DORIS_DESERIALIZE_ARROW_ASYNC;
+import static org.apache.doris.flink.table.DorisConfigOptions.DORIS_DESERIALIZE_QUEUE_SIZE;
+import static org.apache.doris.flink.table.DorisConfigOptions.DORIS_EXEC_MEM_LIMIT;
+import static org.apache.doris.flink.table.DorisConfigOptions.DORIS_REQUEST_CONNECT_TIMEOUT_MS;
+import static org.apache.doris.flink.table.DorisConfigOptions.DORIS_REQUEST_QUERY_TIMEOUT_S;
+import static org.apache.doris.flink.table.DorisConfigOptions.DORIS_REQUEST_READ_TIMEOUT_MS;
+import static org.apache.doris.flink.table.DorisConfigOptions.DORIS_REQUEST_RETRIES;
+import static org.apache.doris.flink.table.DorisConfigOptions.DORIS_TABLET_SIZE;
+import static org.apache.doris.flink.table.DorisConfigOptions.FENODES;
+import static org.apache.doris.flink.table.DorisConfigOptions.IDENTIFIER;
+import static org.apache.doris.flink.table.DorisConfigOptions.JDBC_URL;
+import static org.apache.doris.flink.table.DorisConfigOptions.PASSWORD;
+import static org.apache.doris.flink.table.DorisConfigOptions.SINK_BUFFER_COUNT;
+import static org.apache.doris.flink.table.DorisConfigOptions.SINK_BUFFER_SIZE;
+import static org.apache.doris.flink.table.DorisConfigOptions.SINK_CHECK_INTERVAL;
+import static org.apache.doris.flink.table.DorisConfigOptions.SINK_ENABLE_2PC;
+import static org.apache.doris.flink.table.DorisConfigOptions.SINK_ENABLE_DELETE;
+import static org.apache.doris.flink.table.DorisConfigOptions.SINK_LABEL_PREFIX;
+import static org.apache.doris.flink.table.DorisConfigOptions.SINK_MAX_RETRIES;
+import static org.apache.doris.flink.table.DorisConfigOptions.SINK_PARALLELISM;
+import static org.apache.doris.flink.table.DorisConfigOptions.SINK_USE_CACHE;
+import static org.apache.doris.flink.table.DorisConfigOptions.SOURCE_USE_OLD_API;
+import static org.apache.doris.flink.table.DorisConfigOptions.STREAM_LOAD_PROP_PREFIX;
+import static org.apache.doris.flink.table.DorisConfigOptions.TABLE_IDENTIFIER;
+import static org.apache.doris.flink.table.DorisConfigOptions.USERNAME;
+
+/** Factory for {@link DorisCatalog}. */
+public class DorisCatalogFactory implements CatalogFactory {
+
+ @Override
+ public String factoryIdentifier() {
+ return IDENTIFIER;
+ }
+
+ @Override
+ public Set> requiredOptions() {
+ final Set> options = new HashSet<>();
+ options.add(JDBC_URL);
+ options.add(USERNAME);
+ options.add(PASSWORD);
+ return options;
+ }
+
+ @Override
+ public Set> optionalOptions() {
+ final Set> options = new HashSet<>();
+ options.add(JDBC_URL);
+ options.add(DEFAULT_DATABASE);
+
+ options.add(FENODES);
+ options.add(TABLE_IDENTIFIER);
+ options.add(USERNAME);
+ options.add(PASSWORD);
+
+ options.add(DORIS_TABLET_SIZE);
+ options.add(DORIS_REQUEST_CONNECT_TIMEOUT_MS);
+ options.add(DORIS_REQUEST_READ_TIMEOUT_MS);
+ options.add(DORIS_REQUEST_QUERY_TIMEOUT_S);
+ options.add(DORIS_REQUEST_RETRIES);
+ options.add(DORIS_DESERIALIZE_ARROW_ASYNC);
+ options.add(DORIS_DESERIALIZE_QUEUE_SIZE);
+ options.add(DORIS_BATCH_SIZE);
+ options.add(DORIS_EXEC_MEM_LIMIT);
+
+ options.add(SINK_CHECK_INTERVAL);
+ options.add(SINK_ENABLE_2PC);
+ options.add(SINK_MAX_RETRIES);
+ options.add(SINK_ENABLE_DELETE);
+ options.add(SINK_LABEL_PREFIX);
+ options.add(SINK_BUFFER_SIZE);
+ options.add(SINK_BUFFER_COUNT);
+ options.add(SINK_PARALLELISM);
+ options.add(SINK_USE_CACHE);
+
+ options.add(SOURCE_USE_OLD_API);
+ return options;
+ }
+
+ @Override
+ public Catalog createCatalog(Context context) {
+ final FactoryUtil.CatalogFactoryHelper helper =
+ FactoryUtil.createCatalogFactoryHelper(this, context);
+ helper.validateExcept(STREAM_LOAD_PROP_PREFIX, TABLE_PROPERTIES_PREFIX);
+
+ DorisConnectionOptions connectionOptions =
+ new DorisConnectionOptions.DorisConnectionOptionsBuilder()
+ .withFenodes(helper.getOptions().get(FENODES))
+ .withJdbcUrl(helper.getOptions().get(JDBC_URL))
+ .withUsername(helper.getOptions().get(USERNAME))
+ .withPassword(helper.getOptions().get(PASSWORD))
+ .build();
+ return new DorisCatalog(
+ context.getName(),
+ connectionOptions,
+ helper.getOptions().get(DEFAULT_DATABASE),
+ ((Configuration) helper.getOptions()).toMap());
+ }
+}
diff --git a/flink-doris-connector/flink-doris-connector-2.2/src/main/java/org/apache/doris/flink/datastream/DorisSourceFunction.java b/flink-doris-connector/flink-doris-connector-2.2/src/main/java/org/apache/doris/flink/datastream/DorisSourceFunction.java
new file mode 100644
index 000000000..3229b1d8f
--- /dev/null
+++ b/flink-doris-connector/flink-doris-connector-2.2/src/main/java/org/apache/doris/flink/datastream/DorisSourceFunction.java
@@ -0,0 +1,118 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.flink.datastream;
+
+import org.apache.flink.annotation.PublicEvolving;
+import org.apache.flink.api.common.functions.OpenContext;
+import org.apache.flink.api.common.typeinfo.TypeInformation;
+import org.apache.flink.api.java.typeutils.ResultTypeQueryable;
+import org.apache.flink.streaming.api.functions.source.legacy.RichParallelSourceFunction;
+
+import org.apache.doris.flink.cfg.DorisOptions;
+import org.apache.doris.flink.cfg.DorisReadOptions;
+import org.apache.doris.flink.cfg.DorisStreamOptions;
+import org.apache.doris.flink.deserialization.DorisDeserializationSchema;
+import org.apache.doris.flink.exception.DorisException;
+import org.apache.doris.flink.rest.PartitionDefinition;
+import org.apache.doris.flink.rest.RestService;
+import org.apache.doris.flink.source.reader.DorisValueReader;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.ArrayList;
+import java.util.List;
+
+/** DorisSource. */
+@Deprecated
+@PublicEvolving
+public class DorisSourceFunction extends RichParallelSourceFunction>
+ implements ResultTypeQueryable> {
+
+ private static final Logger logger = LoggerFactory.getLogger(DorisSourceFunction.class);
+
+ private final DorisDeserializationSchema> deserializer;
+ private final DorisOptions options;
+ private final DorisReadOptions readOptions;
+ private transient volatile boolean isRunning;
+ private List dorisPartitions;
+ private List taskDorisPartitions = new ArrayList<>();
+
+ public DorisSourceFunction(
+ DorisStreamOptions streamOptions, DorisDeserializationSchema> deserializer) {
+ this.deserializer = deserializer;
+ this.options = streamOptions.getOptions();
+ this.readOptions = streamOptions.getReadOptions();
+ try {
+ this.dorisPartitions = RestService.findPartitions(options, readOptions, logger);
+ logger.info("Doris partitions size {}", dorisPartitions.size());
+ } catch (DorisException e) {
+ throw new RuntimeException("Failed fetch doris partitions");
+ }
+ }
+
+ @Override
+ public void open(OpenContext parameters) throws Exception {
+ super.open(parameters);
+ this.isRunning = true;
+ assignTaskPartitions();
+ }
+
+ /** Assign partitions to each task. */
+ private void assignTaskPartitions() {
+ int taskIndex = getRuntimeContext().getTaskInfo().getIndexOfThisSubtask();
+ int totalTasks = getRuntimeContext().getTaskInfo().getNumberOfParallelSubtasks();
+
+ for (int i = 0; i < dorisPartitions.size(); i++) {
+ if (i % totalTasks == taskIndex) {
+ taskDorisPartitions.add(dorisPartitions.get(i));
+ }
+ }
+ logger.info("subtask {} process {} partitions ", taskIndex, taskDorisPartitions.size());
+ }
+
+ @Override
+ public void run(SourceContext> sourceContext) {
+ for (PartitionDefinition partitions : taskDorisPartitions) {
+ try (DorisValueReader valueReader =
+ new DorisValueReader(partitions, options, readOptions)) {
+ while (isRunning && valueReader.hasNext()) {
+ List next = valueReader.next();
+ sourceContext.collect(next);
+ }
+ } catch (Exception e) {
+ logger.error("close reader resource failed,", e);
+ }
+ }
+ }
+
+ @Override
+ public void cancel() {
+ isRunning = false;
+ }
+
+ @Override
+ public void close() throws Exception {
+ super.close();
+ isRunning = false;
+ }
+
+ @Override
+ public TypeInformation> getProducedType() {
+ return this.deserializer.getProducedType();
+ }
+}
diff --git a/flink-doris-connector/flink-doris-connector-2.2/src/main/java/org/apache/doris/flink/sink/DorisSink.java b/flink-doris-connector/flink-doris-connector-2.2/src/main/java/org/apache/doris/flink/sink/DorisSink.java
new file mode 100644
index 000000000..f5f0cccd9
--- /dev/null
+++ b/flink-doris-connector/flink-doris-connector-2.2/src/main/java/org/apache/doris/flink/sink/DorisSink.java
@@ -0,0 +1,230 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.flink.sink;
+
+import org.apache.flink.annotation.PublicEvolving;
+import org.apache.flink.annotation.VisibleForTesting;
+import org.apache.flink.api.connector.sink2.Committer;
+import org.apache.flink.api.connector.sink2.CommitterInitContext;
+import org.apache.flink.api.connector.sink2.Sink;
+import org.apache.flink.api.connector.sink2.SupportsCommitter;
+import org.apache.flink.api.connector.sink2.SupportsWriterState;
+import org.apache.flink.api.connector.sink2.WriterInitContext;
+import org.apache.flink.core.io.SimpleVersionedSerializer;
+import org.apache.flink.util.Preconditions;
+
+import org.apache.doris.flink.cfg.DorisExecutionOptions;
+import org.apache.doris.flink.cfg.DorisOptions;
+import org.apache.doris.flink.cfg.DorisReadOptions;
+import org.apache.doris.flink.rest.RestService;
+import org.apache.doris.flink.sink.batch.DorisBatchWriterAdapter;
+import org.apache.doris.flink.sink.committer.DorisCommitter;
+import org.apache.doris.flink.sink.copy.CopyCommittableSerializer;
+import org.apache.doris.flink.sink.copy.DorisCopyCommitter;
+import org.apache.doris.flink.sink.copy.DorisCopyWriterAdapter;
+import org.apache.doris.flink.sink.writer.DorisAbstractWriter;
+import org.apache.doris.flink.sink.writer.DorisWriter;
+import org.apache.doris.flink.sink.writer.DorisWriterAdapter;
+import org.apache.doris.flink.sink.writer.DorisWriterState;
+import org.apache.doris.flink.sink.writer.DorisWriterStateSerializer;
+import org.apache.doris.flink.sink.writer.WriteMode;
+import org.apache.doris.flink.sink.writer.serializer.DorisRecordSerializer;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.util.Collection;
+import java.util.Collections;
+
+/**
+ * Load data into Doris based on 2PC. see {@link DorisWriter} and {@link DorisCommitter}.
+ *
+ * @param type of record.
+ */
+@PublicEvolving
+public class DorisSink
+ implements Sink,
+ SupportsWriterState,
+ SupportsCommitter {
+ private static final Logger LOG = LoggerFactory.getLogger(DorisSink.class);
+ private final DorisOptions dorisOptions;
+ private final DorisReadOptions dorisReadOptions;
+ private final DorisExecutionOptions dorisExecutionOptions;
+ private final DorisRecordSerializer serializer;
+
+ public DorisSink(
+ DorisOptions dorisOptions,
+ DorisReadOptions dorisReadOptions,
+ DorisExecutionOptions dorisExecutionOptions,
+ DorisRecordSerializer serializer) {
+ this.dorisOptions = dorisOptions;
+ this.dorisReadOptions = dorisReadOptions;
+ this.dorisExecutionOptions = dorisExecutionOptions;
+ this.serializer = serializer;
+ checkKeyType();
+ }
+
+ /** The uniq model has 2pc close by default unless 2pc is forced open. */
+ private void checkKeyType() {
+ if (dorisExecutionOptions.enabled2PC()
+ && !dorisExecutionOptions.force2PC()
+ && RestService.isUniqueKeyType(dorisOptions, dorisReadOptions, LOG)) {
+ dorisExecutionOptions.setEnable2PC(false);
+ }
+ }
+
+ @Override
+ public DorisAbstractWriter createWriter(WriterInitContext initContext) throws IOException {
+ return getDorisAbstractWriter(initContext, Collections.emptyList());
+ }
+
+ @Override
+ public Committer createCommitter(CommitterInitContext committerInitContext) throws IOException {
+ if (WriteMode.STREAM_LOAD.equals(dorisExecutionOptions.getWriteMode())
+ || WriteMode.STREAM_LOAD_BATCH.equals(dorisExecutionOptions.getWriteMode())) {
+ return new DorisCommitter(dorisOptions, dorisReadOptions, dorisExecutionOptions);
+ } else if (WriteMode.COPY.equals(dorisExecutionOptions.getWriteMode())) {
+ return new DorisCopyCommitter(dorisOptions, dorisExecutionOptions.getMaxRetries());
+ }
+ throw new IllegalArgumentException(
+ "Unsupported write mode " + dorisExecutionOptions.getWriteMode());
+ }
+
+ @Override
+ public DorisAbstractWriter restoreWriter(
+ WriterInitContext initContext, Collection recoveredState)
+ throws IOException {
+ return getDorisAbstractWriter(initContext, recoveredState);
+ }
+
+ @VisibleForTesting
+ public DorisAbstractWriter getDorisAbstractWriter(
+ WriterInitContext initContext, Collection states) {
+ if (WriteMode.STREAM_LOAD.equals(dorisExecutionOptions.getWriteMode())) {
+ return new DorisWriterAdapter(
+ initContext,
+ states,
+ serializer,
+ dorisOptions,
+ dorisReadOptions,
+ dorisExecutionOptions);
+ } else if (WriteMode.STREAM_LOAD_BATCH.equals(dorisExecutionOptions.getWriteMode())) {
+ return new DorisBatchWriterAdapter(
+ initContext, serializer, dorisOptions, dorisReadOptions, dorisExecutionOptions);
+ } else if (WriteMode.COPY.equals(dorisExecutionOptions.getWriteMode())) {
+ return new DorisCopyWriterAdapter(
+ initContext, serializer, dorisOptions, dorisReadOptions, dorisExecutionOptions);
+ }
+ throw new IllegalArgumentException(
+ "Unsupported write mode " + dorisExecutionOptions.getWriteMode());
+ }
+
+ @Override
+ public SimpleVersionedSerializer getWriterStateSerializer() {
+ return new DorisWriterStateSerializer();
+ }
+
+ @Override
+ public SimpleVersionedSerializer getCommittableSerializer() {
+ if (WriteMode.STREAM_LOAD.equals(dorisExecutionOptions.getWriteMode())
+ || WriteMode.STREAM_LOAD_BATCH.equals(dorisExecutionOptions.getWriteMode())) {
+ return new DorisCommittableSerializer();
+ } else if (WriteMode.COPY.equals(dorisExecutionOptions.getWriteMode())) {
+ return new CopyCommittableSerializer();
+ }
+ throw new IllegalArgumentException(
+ "Unsupported write mode " + dorisExecutionOptions.getWriteMode());
+ }
+
+ public static Builder builder() {
+ return new Builder<>();
+ }
+
+ /**
+ * build for DorisSink.
+ *
+ * @param record type.
+ */
+ public static class Builder {
+ private DorisOptions dorisOptions;
+ private DorisReadOptions dorisReadOptions;
+ private DorisExecutionOptions dorisExecutionOptions;
+ private DorisRecordSerializer serializer;
+
+ /**
+ * Sets the DorisOptions for the DorisSink.
+ *
+ * @param dorisOptions the common options of the doris cluster.
+ * @return this DorisSink.Builder.
+ */
+ public Builder setDorisOptions(DorisOptions dorisOptions) {
+ this.dorisOptions = dorisOptions;
+ return this;
+ }
+
+ /**
+ * Sets the DorisReadOptions for the DorisSink.
+ *
+ * @param dorisReadOptions the read options of the DorisSink.
+ * @return this DorisSink.Builder.
+ */
+ public Builder setDorisReadOptions(DorisReadOptions dorisReadOptions) {
+ this.dorisReadOptions = dorisReadOptions;
+ return this;
+ }
+
+ /**
+ * Sets the DorisExecutionOptions for the DorisSink.
+ *
+ * @param dorisExecutionOptions the execution options of the DorisSink.
+ * @return this DorisSink.Builder.
+ */
+ public Builder setDorisExecutionOptions(DorisExecutionOptions dorisExecutionOptions) {
+ this.dorisExecutionOptions = dorisExecutionOptions;
+ return this;
+ }
+
+ /**
+ * Sets the {@link DorisRecordSerializer serializer} that transforms incoming records to
+ * DorisRecord
+ *
+ * @param serializer
+ * @return this DorisSink.Builder.
+ */
+ public Builder setSerializer(DorisRecordSerializer serializer) {
+ this.serializer = serializer;
+ return this;
+ }
+
+ /**
+ * Build the {@link DorisSink}.
+ *
+ * @return a DorisSink with the settings made for this builder.
+ */
+ public DorisSink build() {
+ Preconditions.checkNotNull(dorisOptions);
+ Preconditions.checkNotNull(dorisExecutionOptions);
+ Preconditions.checkNotNull(serializer);
+ if (dorisReadOptions == null) {
+ dorisReadOptions = DorisReadOptions.builder().build();
+ }
+ return new DorisSink<>(
+ dorisOptions, dorisReadOptions, dorisExecutionOptions, serializer);
+ }
+ }
+}
diff --git a/flink-doris-connector/flink-doris-connector-2.2/src/main/java/org/apache/doris/flink/sink/batch/DorisBatchSink.java b/flink-doris-connector/flink-doris-connector-2.2/src/main/java/org/apache/doris/flink/sink/batch/DorisBatchSink.java
new file mode 100644
index 000000000..de287a572
--- /dev/null
+++ b/flink-doris-connector/flink-doris-connector-2.2/src/main/java/org/apache/doris/flink/sink/batch/DorisBatchSink.java
@@ -0,0 +1,116 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.flink.sink.batch;
+
+import org.apache.flink.annotation.PublicEvolving;
+import org.apache.flink.annotation.VisibleForTesting;
+import org.apache.flink.api.connector.sink2.Sink;
+import org.apache.flink.api.connector.sink2.SinkWriter;
+import org.apache.flink.api.connector.sink2.WriterInitContext;
+import org.apache.flink.util.Preconditions;
+
+import org.apache.doris.flink.cfg.DorisExecutionOptions;
+import org.apache.doris.flink.cfg.DorisOptions;
+import org.apache.doris.flink.cfg.DorisReadOptions;
+import org.apache.doris.flink.sink.writer.serializer.DorisRecordSerializer;
+
+import java.io.IOException;
+
+@Deprecated
+@PublicEvolving
+public class DorisBatchSink implements Sink {
+ private final DorisOptions dorisOptions;
+ private final DorisReadOptions dorisReadOptions;
+ private final DorisExecutionOptions dorisExecutionOptions;
+ private final DorisRecordSerializer serializer;
+
+ public DorisBatchSink(
+ DorisOptions dorisOptions,
+ DorisReadOptions dorisReadOptions,
+ DorisExecutionOptions dorisExecutionOptions,
+ DorisRecordSerializer serializer) {
+ this.dorisOptions = dorisOptions;
+ this.dorisReadOptions = dorisReadOptions;
+ this.dorisExecutionOptions = dorisExecutionOptions;
+ this.serializer = serializer;
+ }
+
+ @Override
+ public SinkWriter createWriter(WriterInitContext initContext) throws IOException {
+ DorisBatchWriterAdapter dorisBatchWriter =
+ new DorisBatchWriterAdapter(
+ initContext,
+ serializer,
+ dorisOptions,
+ dorisReadOptions,
+ dorisExecutionOptions);
+ return dorisBatchWriter;
+ }
+
+ public static Builder builder() {
+ return new Builder<>();
+ }
+
+ /**
+ * build for DorisBatchSink.
+ *
+ * @param record type.
+ */
+ public static class Builder {
+ private DorisOptions dorisOptions;
+ private DorisReadOptions dorisReadOptions;
+ private DorisExecutionOptions dorisExecutionOptions;
+ private DorisRecordSerializer serializer;
+
+ public Builder setDorisOptions(DorisOptions dorisOptions) {
+ this.dorisOptions = dorisOptions;
+ return this;
+ }
+
+ public Builder setDorisReadOptions(DorisReadOptions dorisReadOptions) {
+ this.dorisReadOptions = dorisReadOptions;
+ return this;
+ }
+
+ public Builder setDorisExecutionOptions(DorisExecutionOptions dorisExecutionOptions) {
+ this.dorisExecutionOptions = dorisExecutionOptions;
+ return this;
+ }
+
+ public Builder setSerializer(DorisRecordSerializer serializer) {
+ this.serializer = serializer;
+ return this;
+ }
+
+ public DorisBatchSink build() {
+ Preconditions.checkNotNull(dorisOptions);
+ Preconditions.checkNotNull(dorisExecutionOptions);
+ Preconditions.checkNotNull(serializer);
+ if (dorisReadOptions == null) {
+ dorisReadOptions = DorisReadOptions.builder().build();
+ }
+ return new DorisBatchSink<>(
+ dorisOptions, dorisReadOptions, dorisExecutionOptions, serializer);
+ }
+ }
+
+ @VisibleForTesting
+ public DorisReadOptions getDorisReadOptions() {
+ return dorisReadOptions;
+ }
+}
diff --git a/flink-doris-connector/flink-doris-connector-2.2/src/main/java/org/apache/doris/flink/sink/batch/DorisBatchWriterAdapter.java b/flink-doris-connector/flink-doris-connector-2.2/src/main/java/org/apache/doris/flink/sink/batch/DorisBatchWriterAdapter.java
new file mode 100644
index 000000000..d6c49f8d9
--- /dev/null
+++ b/flink-doris-connector/flink-doris-connector-2.2/src/main/java/org/apache/doris/flink/sink/batch/DorisBatchWriterAdapter.java
@@ -0,0 +1,88 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under this work is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.flink.sink.batch;
+
+import org.apache.flink.api.connector.sink2.WriterInitContext;
+import org.apache.flink.runtime.checkpoint.CheckpointIDCounter;
+
+import org.apache.doris.flink.cfg.DorisExecutionOptions;
+import org.apache.doris.flink.cfg.DorisOptions;
+import org.apache.doris.flink.cfg.DorisReadOptions;
+import org.apache.doris.flink.sink.DorisCommittable;
+import org.apache.doris.flink.sink.writer.DorisAbstractWriter;
+import org.apache.doris.flink.sink.writer.DorisWriterState;
+import org.apache.doris.flink.sink.writer.serializer.DorisRecordSerializer;
+
+import java.io.IOException;
+import java.util.Collection;
+import java.util.List;
+
+/** Flink 2.2 specific wrapper for the shared core {@link DorisBatchWriter} implementation. */
+public class DorisBatchWriterAdapter
+ implements DorisAbstractWriter {
+
+ private final DorisBatchWriter delegate;
+
+ public DorisBatchWriterAdapter(
+ WriterInitContext initContext,
+ DorisRecordSerializer serializer,
+ DorisOptions dorisOptions,
+ DorisReadOptions dorisReadOptions,
+ DorisExecutionOptions executionOptions) {
+
+ long restoreCheckpointId =
+ initContext
+ .getRestoredCheckpointId()
+ .orElse(CheckpointIDCounter.INITIAL_CHECKPOINT_ID - 1);
+ int subtaskId = initContext.getTaskInfo().getIndexOfThisSubtask();
+
+ this.delegate =
+ new DorisBatchWriter<>(
+ restoreCheckpointId,
+ subtaskId,
+ serializer,
+ dorisOptions,
+ dorisReadOptions,
+ executionOptions);
+ }
+
+ @Override
+ public void write(IN in, Context context) throws IOException, InterruptedException {
+ delegate.write(in, context);
+ }
+
+ @Override
+ public void flush(boolean endOfInput) throws IOException, InterruptedException {
+ delegate.flush(endOfInput);
+ }
+
+ @Override
+ public Collection prepareCommit() throws IOException, InterruptedException {
+ return delegate.prepareCommit();
+ }
+
+ @Override
+ public List snapshotState(long checkpointId) throws IOException {
+ return delegate.snapshotState(checkpointId);
+ }
+
+ @Override
+ public void close() throws Exception {
+ delegate.close();
+ }
+}
diff --git a/flink-doris-connector/flink-doris-connector-2.2/src/main/java/org/apache/doris/flink/sink/copy/DorisCopyWriterAdapter.java b/flink-doris-connector/flink-doris-connector-2.2/src/main/java/org/apache/doris/flink/sink/copy/DorisCopyWriterAdapter.java
new file mode 100644
index 000000000..13b94027e
--- /dev/null
+++ b/flink-doris-connector/flink-doris-connector-2.2/src/main/java/org/apache/doris/flink/sink/copy/DorisCopyWriterAdapter.java
@@ -0,0 +1,71 @@
+package org.apache.doris.flink.sink.copy;
+
+import org.apache.flink.api.connector.sink2.WriterInitContext;
+import org.apache.flink.runtime.checkpoint.CheckpointIDCounter;
+
+import org.apache.doris.flink.cfg.DorisExecutionOptions;
+import org.apache.doris.flink.cfg.DorisOptions;
+import org.apache.doris.flink.cfg.DorisReadOptions;
+import org.apache.doris.flink.sink.writer.DorisAbstractWriter;
+import org.apache.doris.flink.sink.writer.DorisWriterState;
+import org.apache.doris.flink.sink.writer.serializer.DorisRecordSerializer;
+
+import java.io.IOException;
+import java.util.Collection;
+import java.util.List;
+
+/** Flink 2.2 specific wrapper for the shared core {@link DorisCopyWriter} implementation. */
+public class DorisCopyWriterAdapter
+ implements DorisAbstractWriter {
+
+ private final DorisCopyWriter delegate;
+
+ public DorisCopyWriterAdapter(
+ WriterInitContext initContext,
+ DorisRecordSerializer serializer,
+ DorisOptions dorisOptions,
+ DorisReadOptions dorisReadOptions,
+ DorisExecutionOptions executionOptions) {
+
+ long restoreCheckpointId =
+ initContext
+ .getRestoredCheckpointId()
+ .orElse(CheckpointIDCounter.INITIAL_CHECKPOINT_ID - 1);
+ int subtaskId = initContext.getTaskInfo().getIndexOfThisSubtask();
+
+ this.delegate =
+ new DorisCopyWriter<>(
+ restoreCheckpointId,
+ subtaskId,
+ serializer,
+ dorisOptions,
+ dorisReadOptions,
+ executionOptions);
+ }
+
+ @Override
+ public void write(IN in, Context context) throws IOException, InterruptedException {
+ delegate.write(in, context);
+ }
+
+ @Override
+ public void flush(boolean endOfInput) throws IOException, InterruptedException {
+ delegate.flush(endOfInput);
+ }
+
+ @Override
+ public Collection prepareCommit()
+ throws IOException, InterruptedException {
+ return delegate.prepareCommit();
+ }
+
+ @Override
+ public List snapshotState(long checkpointId) throws IOException {
+ return delegate.snapshotState(checkpointId);
+ }
+
+ @Override
+ public void close() throws Exception {
+ delegate.close();
+ }
+}
diff --git a/flink-doris-connector/flink-doris-connector-2.2/src/main/java/org/apache/doris/flink/sink/writer/DorisAbstractWriter.java b/flink-doris-connector/flink-doris-connector-2.2/src/main/java/org/apache/doris/flink/sink/writer/DorisAbstractWriter.java
new file mode 100644
index 000000000..8e8941b6e
--- /dev/null
+++ b/flink-doris-connector/flink-doris-connector-2.2/src/main/java/org/apache/doris/flink/sink/writer/DorisAbstractWriter.java
@@ -0,0 +1,25 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.flink.sink.writer;
+
+import org.apache.flink.api.connector.sink2.CommittingSinkWriter;
+import org.apache.flink.api.connector.sink2.StatefulSinkWriter;
+
+/** Abstract for different Doris Writer: Stream Load, Copy ... */
+public interface DorisAbstractWriter
+ extends StatefulSinkWriter, CommittingSinkWriter {}
diff --git a/flink-doris-connector/flink-doris-connector-2.2/src/main/java/org/apache/doris/flink/sink/writer/DorisWriterAdapter.java b/flink-doris-connector/flink-doris-connector-2.2/src/main/java/org/apache/doris/flink/sink/writer/DorisWriterAdapter.java
new file mode 100644
index 000000000..7a3953a94
--- /dev/null
+++ b/flink-doris-connector/flink-doris-connector-2.2/src/main/java/org/apache/doris/flink/sink/writer/DorisWriterAdapter.java
@@ -0,0 +1,97 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.flink.sink.writer;
+
+import org.apache.flink.api.connector.sink2.WriterInitContext;
+import org.apache.flink.metrics.groups.SinkWriterMetricGroup;
+import org.apache.flink.runtime.checkpoint.CheckpointIDCounter;
+
+import org.apache.doris.flink.cfg.DorisExecutionOptions;
+import org.apache.doris.flink.cfg.DorisOptions;
+import org.apache.doris.flink.cfg.DorisReadOptions;
+import org.apache.doris.flink.sink.DorisCommittable;
+import org.apache.doris.flink.sink.writer.serializer.DorisRecordSerializer;
+
+import java.io.IOException;
+import java.util.Collection;
+import java.util.List;
+
+/**
+ * Flink 2.2 specific wrapper for the shared core {@link DorisWriter} implementation.
+ *
+ *
This class adapts Flink 2.2's {@link WriterInitContext} to the version-neutral core writer in
+ * the base module while implementing {@link DorisAbstractWriter} for use by the 2.2 connector
+ * module.
+ */
+public class DorisWriterAdapter
+ implements DorisAbstractWriter {
+
+ private final DorisWriter delegate;
+
+ public DorisWriterAdapter(
+ WriterInitContext initContext,
+ Collection state,
+ DorisRecordSerializer serializer,
+ DorisOptions dorisOptions,
+ DorisReadOptions dorisReadOptions,
+ DorisExecutionOptions executionOptions) {
+
+ long lastCheckpointId =
+ initContext
+ .getRestoredCheckpointId()
+ .orElse(CheckpointIDCounter.INITIAL_CHECKPOINT_ID - 1);
+ int subtaskId = initContext.getTaskInfo().getIndexOfThisSubtask();
+ SinkWriterMetricGroup sinkMetricGroup = initContext.metricGroup();
+
+ this.delegate =
+ new DorisWriter<>(
+ lastCheckpointId,
+ subtaskId,
+ sinkMetricGroup,
+ state,
+ serializer,
+ dorisOptions,
+ dorisReadOptions,
+ executionOptions);
+ }
+
+ @Override
+ public void write(IN in, Context context) throws IOException, InterruptedException {
+ delegate.write(in, context);
+ }
+
+ @Override
+ public void flush(boolean endOfInput) throws IOException, InterruptedException {
+ delegate.flush(endOfInput);
+ }
+
+ @Override
+ public Collection prepareCommit() throws IOException, InterruptedException {
+ return delegate.prepareCommit();
+ }
+
+ @Override
+ public List snapshotState(long checkpointId) throws IOException {
+ return delegate.snapshotState(checkpointId);
+ }
+
+ @Override
+ public void close() throws Exception {
+ delegate.close();
+ }
+}
diff --git a/flink-doris-connector/flink-doris-connector-2.2/src/main/java/org/apache/doris/flink/table/DorisAsyncLookupAdapter.java b/flink-doris-connector/flink-doris-connector-2.2/src/main/java/org/apache/doris/flink/table/DorisAsyncLookupAdapter.java
new file mode 100644
index 000000000..39f32723e
--- /dev/null
+++ b/flink-doris-connector/flink-doris-connector-2.2/src/main/java/org/apache/doris/flink/table/DorisAsyncLookupAdapter.java
@@ -0,0 +1,96 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+package org.apache.doris.flink.table;
+
+import org.apache.flink.table.data.RowData;
+import org.apache.flink.table.functions.AsyncLookupFunction;
+import org.apache.flink.table.functions.FunctionContext;
+import org.apache.flink.table.types.DataType;
+
+import org.apache.doris.flink.cfg.DorisLookupOptions;
+import org.apache.doris.flink.cfg.DorisOptions;
+import org.apache.doris.flink.deserialization.converter.DorisRowConverter;
+
+import java.io.IOException;
+import java.util.Collection;
+import java.util.concurrent.CompletableFuture;
+
+/**
+ * Adapter that bridges Flink 2.2 {@link AsyncLookupFunction} to the existing {@link
+ * DorisRowDataAsyncLookupFunction} which is implemented against the older {@link
+ * org.apache.flink.table.functions.AsyncTableFunction} API and shared with Flink 1.15–1.20.
+ */
+public class DorisAsyncLookupAdapter extends AsyncLookupFunction {
+
+ private final DorisRowDataAsyncLookupFunction delegate;
+ private final DorisRowConverter keyConverter;
+ private final int[] keyIndex;
+
+ public DorisAsyncLookupAdapter(
+ DorisOptions options,
+ DorisLookupOptions lookupOptions,
+ String[] selectFields,
+ DataType[] fieldTypes,
+ String[] conditionFields,
+ int[] keyIndex) {
+ this.delegate =
+ new DorisRowDataAsyncLookupFunction(
+ options,
+ lookupOptions,
+ selectFields,
+ fieldTypes,
+ conditionFields,
+ keyIndex);
+ this.keyIndex = keyIndex;
+ DataType[] keyTypes = new DataType[keyIndex.length];
+ for (int i = 0; i < keyIndex.length; i++) {
+ keyTypes[i] = fieldTypes[keyIndex[i]];
+ }
+ this.keyConverter = new DorisRowConverter(keyTypes);
+ }
+
+ @Override
+ public void open(FunctionContext context) throws Exception {
+ super.open(context);
+ delegate.open(context);
+ }
+
+ @Override
+ public CompletableFuture> asyncLookup(RowData keyRow) {
+ Object[] keys = new Object[keyIndex.length];
+ for (int i = 0; i < keyIndex.length; i++) {
+ keys[i] = keyConverter.convertExternal(keyRow, i);
+ }
+
+ CompletableFuture> future = new CompletableFuture<>();
+ try {
+ delegate.eval(future, keys);
+ } catch (IOException e) {
+ future.completeExceptionally(e);
+ }
+ return future;
+ }
+
+ @Override
+ public void close() throws Exception {
+ try {
+ delegate.close();
+ } finally {
+ super.close();
+ }
+ }
+}
diff --git a/flink-doris-connector/flink-doris-connector-2.2/src/main/java/org/apache/doris/flink/table/DorisDynamicTableFactory.java b/flink-doris-connector/flink-doris-connector-2.2/src/main/java/org/apache/doris/flink/table/DorisDynamicTableFactory.java
new file mode 100644
index 000000000..d931593fb
--- /dev/null
+++ b/flink-doris-connector/flink-doris-connector-2.2/src/main/java/org/apache/doris/flink/table/DorisDynamicTableFactory.java
@@ -0,0 +1,306 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.flink.table;
+
+import org.apache.flink.configuration.ConfigOption;
+import org.apache.flink.configuration.ReadableConfig;
+import org.apache.flink.table.connector.sink.DynamicTableSink;
+import org.apache.flink.table.connector.source.DynamicTableSource;
+import org.apache.flink.table.factories.DynamicTableSinkFactory;
+import org.apache.flink.table.factories.DynamicTableSourceFactory;
+import org.apache.flink.table.factories.FactoryUtil;
+import org.apache.flink.table.legacy.api.TableSchema;
+import org.apache.flink.table.types.DataType;
+import org.apache.flink.table.utils.TableSchemaUtils;
+
+import org.apache.doris.flink.cfg.DorisExecutionOptions;
+import org.apache.doris.flink.cfg.DorisLookupOptions;
+import org.apache.doris.flink.cfg.DorisOptions;
+import org.apache.doris.flink.cfg.DorisReadOptions;
+import org.apache.doris.flink.sink.writer.WriteMode;
+
+import java.util.HashSet;
+import java.util.Properties;
+import java.util.Set;
+
+import static org.apache.doris.flink.table.DorisConfigOptions.AUTO_REDIRECT;
+import static org.apache.doris.flink.table.DorisConfigOptions.BENODES;
+import static org.apache.doris.flink.table.DorisConfigOptions.DORIS_BATCH_SIZE;
+import static org.apache.doris.flink.table.DorisConfigOptions.DORIS_DESERIALIZE_ARROW_ASYNC;
+import static org.apache.doris.flink.table.DorisConfigOptions.DORIS_DESERIALIZE_QUEUE_SIZE;
+import static org.apache.doris.flink.table.DorisConfigOptions.DORIS_EXEC_MEM_LIMIT;
+import static org.apache.doris.flink.table.DorisConfigOptions.DORIS_FILTER_QUERY;
+import static org.apache.doris.flink.table.DorisConfigOptions.DORIS_READ_FIELD;
+import static org.apache.doris.flink.table.DorisConfigOptions.DORIS_REQUEST_CONNECT_TIMEOUT_MS;
+import static org.apache.doris.flink.table.DorisConfigOptions.DORIS_REQUEST_QUERY_TIMEOUT_S;
+import static org.apache.doris.flink.table.DorisConfigOptions.DORIS_REQUEST_READ_TIMEOUT_MS;
+import static org.apache.doris.flink.table.DorisConfigOptions.DORIS_REQUEST_RETRIES;
+import static org.apache.doris.flink.table.DorisConfigOptions.DORIS_TABLET_SIZE;
+import static org.apache.doris.flink.table.DorisConfigOptions.DORIS_THRIFT_MAX_MESSAGE_SIZE;
+import static org.apache.doris.flink.table.DorisConfigOptions.FENODES;
+import static org.apache.doris.flink.table.DorisConfigOptions.FLIGHT_SQL_PORT;
+import static org.apache.doris.flink.table.DorisConfigOptions.IDENTIFIER;
+import static org.apache.doris.flink.table.DorisConfigOptions.JDBC_URL;
+import static org.apache.doris.flink.table.DorisConfigOptions.LOOKUP_CACHE_MAX_ROWS;
+import static org.apache.doris.flink.table.DorisConfigOptions.LOOKUP_CACHE_TTL;
+import static org.apache.doris.flink.table.DorisConfigOptions.LOOKUP_JDBC_ASYNC;
+import static org.apache.doris.flink.table.DorisConfigOptions.LOOKUP_JDBC_READ_BATCH_QUEUE_SIZE;
+import static org.apache.doris.flink.table.DorisConfigOptions.LOOKUP_JDBC_READ_BATCH_SIZE;
+import static org.apache.doris.flink.table.DorisConfigOptions.LOOKUP_JDBC_READ_THREAD_SIZE;
+import static org.apache.doris.flink.table.DorisConfigOptions.LOOKUP_MAX_RETRIES;
+import static org.apache.doris.flink.table.DorisConfigOptions.PASSWORD;
+import static org.apache.doris.flink.table.DorisConfigOptions.SINK_BUFFER_COUNT;
+import static org.apache.doris.flink.table.DorisConfigOptions.SINK_BUFFER_FLUSH_INTERVAL;
+import static org.apache.doris.flink.table.DorisConfigOptions.SINK_BUFFER_FLUSH_MAX_BYTES;
+import static org.apache.doris.flink.table.DorisConfigOptions.SINK_BUFFER_FLUSH_MAX_ROWS;
+import static org.apache.doris.flink.table.DorisConfigOptions.SINK_BUFFER_SIZE;
+import static org.apache.doris.flink.table.DorisConfigOptions.SINK_CHECK_INTERVAL;
+import static org.apache.doris.flink.table.DorisConfigOptions.SINK_ENABLE_2PC;
+import static org.apache.doris.flink.table.DorisConfigOptions.SINK_ENABLE_BATCH_MODE;
+import static org.apache.doris.flink.table.DorisConfigOptions.SINK_ENABLE_DELETE;
+import static org.apache.doris.flink.table.DorisConfigOptions.SINK_FLUSH_QUEUE_SIZE;
+import static org.apache.doris.flink.table.DorisConfigOptions.SINK_HTTP_UTF8_CHARSET;
+import static org.apache.doris.flink.table.DorisConfigOptions.SINK_IGNORE_COMMIT_ERROR;
+import static org.apache.doris.flink.table.DorisConfigOptions.SINK_IGNORE_UPDATE_BEFORE;
+import static org.apache.doris.flink.table.DorisConfigOptions.SINK_LABEL_PREFIX;
+import static org.apache.doris.flink.table.DorisConfigOptions.SINK_MAX_RETRIES;
+import static org.apache.doris.flink.table.DorisConfigOptions.SINK_PARALLELISM;
+import static org.apache.doris.flink.table.DorisConfigOptions.SINK_USE_CACHE;
+import static org.apache.doris.flink.table.DorisConfigOptions.SINK_WRITE_MODE;
+import static org.apache.doris.flink.table.DorisConfigOptions.SOURCE_USE_OLD_API;
+import static org.apache.doris.flink.table.DorisConfigOptions.STREAM_LOAD_PROP_PREFIX;
+import static org.apache.doris.flink.table.DorisConfigOptions.TABLE_IDENTIFIER;
+import static org.apache.doris.flink.table.DorisConfigOptions.USERNAME;
+import static org.apache.doris.flink.table.DorisConfigOptions.USE_FLIGHT_SQL;
+
+/**
+ * The {@link DorisDynamicTableFactory} translates the catalog table to a table source.
+ *
+ *
Because the table source requires a decoding format, we are discovering the format using the
+ * provided {@link FactoryUtil} for convenience.
+ */
+public final class DorisDynamicTableFactory
+ implements DynamicTableSourceFactory, DynamicTableSinkFactory {
+
+ @Override
+ public String factoryIdentifier() {
+ return IDENTIFIER; // used for matching to `connector = '...'`
+ }
+
+ @Override
+ public Set> requiredOptions() {
+ final Set> options = new HashSet<>();
+ options.add(FENODES);
+ options.add(TABLE_IDENTIFIER);
+ return options;
+ }
+
+ @Override
+ public Set> optionalOptions() {
+ final Set> options = new HashSet<>();
+ options.add(FENODES);
+ options.add(BENODES);
+ options.add(TABLE_IDENTIFIER);
+ options.add(USERNAME);
+ options.add(PASSWORD);
+ options.add(JDBC_URL);
+ options.add(AUTO_REDIRECT);
+
+ options.add(DORIS_READ_FIELD);
+ options.add(DORIS_FILTER_QUERY);
+ options.add(DORIS_TABLET_SIZE);
+ options.add(DORIS_REQUEST_CONNECT_TIMEOUT_MS);
+ options.add(DORIS_REQUEST_READ_TIMEOUT_MS);
+ options.add(DORIS_REQUEST_QUERY_TIMEOUT_S);
+ options.add(DORIS_REQUEST_RETRIES);
+ options.add(DORIS_DESERIALIZE_ARROW_ASYNC);
+ options.add(DORIS_DESERIALIZE_QUEUE_SIZE);
+ options.add(DORIS_BATCH_SIZE);
+ options.add(DORIS_EXEC_MEM_LIMIT);
+ options.add(DORIS_THRIFT_MAX_MESSAGE_SIZE);
+
+ options.add(LOOKUP_CACHE_MAX_ROWS);
+ options.add(LOOKUP_CACHE_TTL);
+ options.add(LOOKUP_MAX_RETRIES);
+ options.add(LOOKUP_JDBC_ASYNC);
+ options.add(LOOKUP_JDBC_READ_BATCH_SIZE);
+ options.add(LOOKUP_JDBC_READ_THREAD_SIZE);
+ options.add(LOOKUP_JDBC_READ_BATCH_QUEUE_SIZE);
+
+ options.add(SINK_CHECK_INTERVAL);
+ options.add(SINK_ENABLE_2PC);
+ options.add(SINK_MAX_RETRIES);
+ options.add(SINK_ENABLE_DELETE);
+ options.add(SINK_LABEL_PREFIX);
+ options.add(SINK_BUFFER_SIZE);
+ options.add(SINK_BUFFER_COUNT);
+ options.add(SINK_PARALLELISM);
+ options.add(SINK_IGNORE_UPDATE_BEFORE);
+
+ options.add(SINK_ENABLE_BATCH_MODE);
+ options.add(SINK_BUFFER_FLUSH_MAX_ROWS);
+ options.add(SINK_BUFFER_FLUSH_MAX_BYTES);
+ options.add(SINK_FLUSH_QUEUE_SIZE);
+ options.add(SINK_BUFFER_FLUSH_INTERVAL);
+
+ options.add(SINK_USE_CACHE);
+
+ options.add(SOURCE_USE_OLD_API);
+ options.add(SINK_WRITE_MODE);
+ options.add(SINK_IGNORE_COMMIT_ERROR);
+
+ options.add(USE_FLIGHT_SQL);
+ options.add(FLIGHT_SQL_PORT);
+ options.add(SINK_HTTP_UTF8_CHARSET);
+ return options;
+ }
+
+ @Override
+ public DynamicTableSource createDynamicTableSource(Context context) {
+ // either implement your custom validation logic here ...
+ // or use the provided helper utility
+ final FactoryUtil.TableFactoryHelper helper =
+ FactoryUtil.createTableFactoryHelper(this, context);
+ // validate all options
+ helper.validateExcept(STREAM_LOAD_PROP_PREFIX);
+ // get the validated options
+ final ReadableConfig options = helper.getOptions();
+ // derive the produced data type (excluding computed columns) from the catalog table
+ final DataType producedDataType =
+ context.getCatalogTable().getSchema().toPhysicalRowDataType();
+ TableSchema physicalSchema =
+ TableSchemaUtils.getPhysicalSchema(context.getCatalogTable().getSchema());
+ // create and return dynamic table source
+ return new DorisDynamicTableSource(
+ getDorisOptions(helper.getOptions()),
+ getDorisReadOptions(helper.getOptions()),
+ getDorisLookupOptions(helper.getOptions()),
+ physicalSchema,
+ context.getPhysicalRowDataType());
+ }
+
+ private DorisOptions getDorisOptions(ReadableConfig readableConfig) {
+ final String fenodes = readableConfig.get(FENODES);
+ final String benodes = readableConfig.get(BENODES);
+ final DorisOptions.Builder builder =
+ DorisOptions.builder()
+ .setFenodes(fenodes)
+ .setBenodes(benodes)
+ .setAutoRedirect(readableConfig.get(AUTO_REDIRECT))
+ .setJdbcUrl(readableConfig.get(JDBC_URL))
+ .setTableIdentifier(readableConfig.get(TABLE_IDENTIFIER));
+
+ readableConfig.getOptional(USERNAME).ifPresent(builder::setUsername);
+ readableConfig.getOptional(PASSWORD).ifPresent(builder::setPassword);
+ return builder.build();
+ }
+
+ private DorisReadOptions getDorisReadOptions(ReadableConfig readableConfig) {
+ final DorisReadOptions.Builder builder = DorisReadOptions.builder();
+ builder.setDeserializeArrowAsync(readableConfig.get(DORIS_DESERIALIZE_ARROW_ASYNC))
+ .setDeserializeQueueSize(readableConfig.get(DORIS_DESERIALIZE_QUEUE_SIZE))
+ .setExecMemLimit(readableConfig.get(DORIS_EXEC_MEM_LIMIT).getBytes())
+ .setThriftMaxMessageSize(readableConfig.get(DORIS_THRIFT_MAX_MESSAGE_SIZE))
+ .setFilterQuery(readableConfig.get(DORIS_FILTER_QUERY))
+ .setReadFields(readableConfig.get(DORIS_READ_FIELD))
+ .setRequestQueryTimeoutS(
+ (int) readableConfig.get(DORIS_REQUEST_QUERY_TIMEOUT_S).getSeconds())
+ .setRequestBatchSize(readableConfig.get(DORIS_BATCH_SIZE))
+ .setRequestConnectTimeoutMs(
+ (int) readableConfig.get(DORIS_REQUEST_CONNECT_TIMEOUT_MS).toMillis())
+ .setRequestReadTimeoutMs(
+ (int) readableConfig.get(DORIS_REQUEST_READ_TIMEOUT_MS).toMillis())
+ .setRequestRetries(readableConfig.get(DORIS_REQUEST_RETRIES))
+ .setRequestTabletSize(readableConfig.get(DORIS_TABLET_SIZE))
+ .setUseOldApi(readableConfig.get(SOURCE_USE_OLD_API))
+ .setUseFlightSql(readableConfig.get(USE_FLIGHT_SQL))
+ .setFlightSqlPort(readableConfig.get(FLIGHT_SQL_PORT));
+ return builder.build();
+ }
+
+ private DorisExecutionOptions getDorisExecutionOptions(
+ ReadableConfig readableConfig, Properties streamLoadProp) {
+ final DorisExecutionOptions.Builder builder = DorisExecutionOptions.builder();
+ builder.setCheckInterval((int) readableConfig.get(SINK_CHECK_INTERVAL).toMillis());
+ builder.setMaxRetries(readableConfig.get(SINK_MAX_RETRIES));
+ builder.setBufferSize((int) readableConfig.get(SINK_BUFFER_SIZE).getBytes());
+ builder.setBufferCount(readableConfig.get(SINK_BUFFER_COUNT));
+ builder.setLabelPrefix(readableConfig.get(SINK_LABEL_PREFIX));
+ builder.setStreamLoadProp(streamLoadProp);
+ builder.setDeletable(readableConfig.get(SINK_ENABLE_DELETE));
+ builder.setIgnoreUpdateBefore(readableConfig.get(SINK_IGNORE_UPDATE_BEFORE));
+ builder.setIgnoreCommitError(readableConfig.get(SINK_IGNORE_COMMIT_ERROR));
+
+ if (!readableConfig.get(SINK_ENABLE_2PC)) {
+ builder.disable2PC();
+ } else if (readableConfig.getOptional(SINK_ENABLE_2PC).isPresent()) {
+ // force open 2pc
+ builder.enable2PC();
+ }
+
+ builder.setWriteMode(WriteMode.of(readableConfig.get(SINK_WRITE_MODE)));
+ builder.setBatchMode(readableConfig.get(SINK_ENABLE_BATCH_MODE));
+ // Compatible with previous versions
+ if (readableConfig.get(SINK_ENABLE_BATCH_MODE)) {
+ builder.setWriteMode(WriteMode.STREAM_LOAD_BATCH);
+ }
+ builder.setFlushQueueSize(readableConfig.get(SINK_FLUSH_QUEUE_SIZE));
+ builder.setBufferFlushMaxRows(readableConfig.get(SINK_BUFFER_FLUSH_MAX_ROWS));
+ builder.setBufferFlushMaxBytes(
+ (int) readableConfig.get(SINK_BUFFER_FLUSH_MAX_BYTES).getBytes());
+ builder.setBufferFlushIntervalMs(readableConfig.get(SINK_BUFFER_FLUSH_INTERVAL).toMillis());
+ builder.setUseCache(readableConfig.get(SINK_USE_CACHE));
+ builder.setHttpUtf8Charset(readableConfig.get(SINK_HTTP_UTF8_CHARSET));
+ return builder.build();
+ }
+
+ private DorisLookupOptions getDorisLookupOptions(ReadableConfig readableConfig) {
+ final DorisLookupOptions.Builder builder = DorisLookupOptions.builder();
+ builder.setCacheExpireMs(readableConfig.get(LOOKUP_CACHE_TTL).toMillis());
+ builder.setCacheMaxSize(readableConfig.get(LOOKUP_CACHE_MAX_ROWS));
+ builder.setMaxRetryTimes(readableConfig.get(LOOKUP_MAX_RETRIES));
+ builder.setJdbcReadBatchSize(readableConfig.get(LOOKUP_JDBC_READ_BATCH_SIZE));
+ builder.setJdbcReadBatchQueueSize(readableConfig.get(LOOKUP_JDBC_READ_BATCH_QUEUE_SIZE));
+ builder.setJdbcReadThreadSize(readableConfig.get(LOOKUP_JDBC_READ_THREAD_SIZE));
+ builder.setAsync(readableConfig.get(LOOKUP_JDBC_ASYNC));
+ return builder.build();
+ }
+
+ @Override
+ public DynamicTableSink createDynamicTableSink(Context context) {
+ final FactoryUtil.TableFactoryHelper helper =
+ FactoryUtil.createTableFactoryHelper(this, context);
+
+ // validate all options
+ helper.validateExcept(STREAM_LOAD_PROP_PREFIX);
+ // sink parallelism
+ final Integer parallelism = helper.getOptions().get(SINK_PARALLELISM);
+
+ Properties streamLoadProp =
+ DorisConfigOptions.getStreamLoadProp(context.getCatalogTable().getOptions());
+ TableSchema physicalSchema =
+ TableSchemaUtils.getPhysicalSchema(context.getCatalogTable().getSchema());
+ // create and return dynamic table sink
+ return new DorisDynamicTableSink(
+ getDorisOptions(helper.getOptions()),
+ getDorisReadOptions(helper.getOptions()),
+ getDorisExecutionOptions(helper.getOptions(), streamLoadProp),
+ physicalSchema,
+ parallelism);
+ }
+}
diff --git a/flink-doris-connector/flink-doris-connector-2.2/src/main/java/org/apache/doris/flink/table/DorisDynamicTableSink.java b/flink-doris-connector/flink-doris-connector-2.2/src/main/java/org/apache/doris/flink/table/DorisDynamicTableSink.java
new file mode 100644
index 000000000..b3d75ded5
--- /dev/null
+++ b/flink-doris-connector/flink-doris-connector-2.2/src/main/java/org/apache/doris/flink/table/DorisDynamicTableSink.java
@@ -0,0 +1,198 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.flink.table;
+
+import org.apache.flink.table.connector.ChangelogMode;
+import org.apache.flink.table.connector.sink.DynamicTableSink;
+import org.apache.flink.table.connector.sink.SinkV2Provider;
+import org.apache.flink.table.connector.sink.abilities.SupportsOverwrite;
+import org.apache.flink.table.data.RowData;
+import org.apache.flink.table.legacy.api.TableSchema;
+import org.apache.flink.util.Preconditions;
+
+import org.apache.doris.flink.cfg.DorisExecutionOptions;
+import org.apache.doris.flink.cfg.DorisOptions;
+import org.apache.doris.flink.cfg.DorisReadOptions;
+import org.apache.doris.flink.connection.SimpleJdbcConnectionProvider;
+import org.apache.doris.flink.exception.DorisSystemException;
+import org.apache.doris.flink.rest.RestService;
+import org.apache.doris.flink.sink.DorisSink;
+import org.apache.doris.flink.sink.writer.serializer.RowDataSerializer;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.sql.Connection;
+import java.sql.Statement;
+import java.util.Arrays;
+import java.util.Objects;
+import java.util.Properties;
+import java.util.stream.Collectors;
+
+import static org.apache.doris.flink.sink.writer.LoadConstants.COLUMNS_KEY;
+import static org.apache.doris.flink.sink.writer.LoadConstants.CSV;
+import static org.apache.doris.flink.sink.writer.LoadConstants.DORIS_DELETE_SIGN;
+import static org.apache.doris.flink.sink.writer.LoadConstants.FIELD_DELIMITER_DEFAULT;
+import static org.apache.doris.flink.sink.writer.LoadConstants.FIELD_DELIMITER_KEY;
+import static org.apache.doris.flink.sink.writer.LoadConstants.FORMAT_KEY;
+
+/** DorisDynamicTableSink. */
+public class DorisDynamicTableSink implements DynamicTableSink, SupportsOverwrite {
+ private static final Logger LOG = LoggerFactory.getLogger(DorisDynamicTableSink.class);
+ private final DorisOptions options;
+ private final DorisReadOptions readOptions;
+ private final DorisExecutionOptions executionOptions;
+ private final TableSchema tableSchema;
+ private final Integer sinkParallelism;
+ private boolean overwrite = false;
+
+ public DorisDynamicTableSink(
+ DorisOptions options,
+ DorisReadOptions readOptions,
+ DorisExecutionOptions executionOptions,
+ TableSchema tableSchema,
+ Integer sinkParallelism) {
+ this.options = options;
+ this.readOptions = readOptions;
+ this.executionOptions = executionOptions;
+ this.tableSchema = tableSchema;
+ this.sinkParallelism = sinkParallelism;
+ }
+
+ @Override
+ public ChangelogMode getChangelogMode(ChangelogMode changelogMode) {
+ if (executionOptions.getIgnoreUpdateBefore()) {
+ return ChangelogMode.upsert();
+ } else {
+ return ChangelogMode.all();
+ }
+ }
+
+ @Override
+ public SinkRuntimeProvider getSinkRuntimeProvider(Context context) {
+ Properties loadProperties = executionOptions.getStreamLoadProp();
+ boolean deletable =
+ executionOptions.getDeletable()
+ && RestService.isUniqueKeyType(options, readOptions, LOG);
+ if (!loadProperties.containsKey(COLUMNS_KEY)) {
+ String[] fieldNames = tableSchema.getFieldNames();
+ Preconditions.checkState(fieldNames != null && fieldNames.length > 0);
+ String columns =
+ String.join(
+ ",",
+ Arrays.stream(fieldNames)
+ .map(
+ item ->
+ String.format(
+ "`%s`", item.trim().replace("`", "")))
+ .collect(Collectors.toList()));
+ if (deletable) {
+ columns = String.format("%s,%s", columns, DORIS_DELETE_SIGN);
+ }
+ loadProperties.put(COLUMNS_KEY, columns);
+ }
+
+ RowDataSerializer.Builder serializerBuilder = RowDataSerializer.builder();
+ serializerBuilder
+ .setFieldNames(tableSchema.getFieldNames())
+ .setFieldType(tableSchema.getFieldDataTypes())
+ .setType(loadProperties.getProperty(FORMAT_KEY, CSV))
+ .enableDelete(deletable)
+ .setFieldDelimiter(
+ loadProperties.getProperty(FIELD_DELIMITER_KEY, FIELD_DELIMITER_DEFAULT));
+
+ DorisSink.Builder dorisSinkBuilder = DorisSink.builder();
+ dorisSinkBuilder
+ .setDorisOptions(options)
+ .setDorisReadOptions(readOptions)
+ .setDorisExecutionOptions(executionOptions)
+ .setSerializer(serializerBuilder.build());
+ DorisSink dorisSink = dorisSinkBuilder.build();
+
+ // for insert overwrite
+ if (overwrite) {
+ if (context.isBounded()) {
+ // execute jdbc query to truncate table
+ Preconditions.checkArgument(
+ options.getJdbcUrl() != null, "jdbc-url is required for Overwrite mode.");
+ // todo: should be written to a temporary table first,
+ // and then use GlobalCommitter to perform the rename.
+ truncateTable();
+ } else {
+ throw new IllegalStateException("Streaming mode not support overwrite.");
+ }
+ }
+ return SinkV2Provider.of(dorisSink, sinkParallelism);
+ }
+
+ private void truncateTable() {
+ String truncateQuery = "TRUNCATE TABLE " + options.getTableIdentifier();
+ SimpleJdbcConnectionProvider jdbcConnectionProvider =
+ new SimpleJdbcConnectionProvider(options);
+ try (Connection connection = jdbcConnectionProvider.getOrEstablishConnection();
+ Statement statement = connection.createStatement()) {
+ LOG.info("Executing truncate query: {}", truncateQuery);
+ statement.execute(truncateQuery);
+ } catch (Exception e) {
+ LOG.error("Failed to execute truncate query: {}", truncateQuery, e);
+ throw new DorisSystemException(
+ String.format("Failed to execute truncate query: %s", truncateQuery), e);
+ }
+ }
+
+ @Override
+ public DynamicTableSink copy() {
+ DorisDynamicTableSink sink =
+ new DorisDynamicTableSink(
+ options, readOptions, executionOptions, tableSchema, sinkParallelism);
+ sink.overwrite = overwrite;
+ return sink;
+ }
+
+ @Override
+ public String asSummaryString() {
+ return "Doris Table Sink";
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (this == o) {
+ return true;
+ }
+ if (o == null || getClass() != o.getClass()) {
+ return false;
+ }
+ DorisDynamicTableSink that = (DorisDynamicTableSink) o;
+ return Objects.equals(options, that.options)
+ && Objects.equals(readOptions, that.readOptions)
+ && Objects.equals(executionOptions, that.executionOptions)
+ && Objects.equals(tableSchema, that.tableSchema)
+ && Objects.equals(sinkParallelism, that.sinkParallelism)
+ && Objects.equals(overwrite, that.overwrite);
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(
+ options, readOptions, executionOptions, tableSchema, sinkParallelism, overwrite);
+ }
+
+ @Override
+ public void applyOverwrite(boolean overwrite) {
+ this.overwrite = overwrite;
+ }
+}
diff --git a/flink-doris-connector/flink-doris-connector-2.2/src/main/java/org/apache/doris/flink/table/DorisDynamicTableSource.java b/flink-doris-connector/flink-doris-connector-2.2/src/main/java/org/apache/doris/flink/table/DorisDynamicTableSource.java
new file mode 100644
index 000000000..4cecadd98
--- /dev/null
+++ b/flink-doris-connector/flink-doris-connector-2.2/src/main/java/org/apache/doris/flink/table/DorisDynamicTableSource.java
@@ -0,0 +1,258 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.flink.table;
+
+import org.apache.flink.annotation.VisibleForTesting;
+import org.apache.flink.table.connector.ChangelogMode;
+import org.apache.flink.table.connector.Projection;
+import org.apache.flink.table.connector.source.DynamicTableSource;
+import org.apache.flink.table.connector.source.InputFormatProvider;
+import org.apache.flink.table.connector.source.LookupTableSource;
+import org.apache.flink.table.connector.source.ScanTableSource;
+import org.apache.flink.table.connector.source.SourceProvider;
+import org.apache.flink.table.connector.source.abilities.SupportsFilterPushDown;
+import org.apache.flink.table.connector.source.abilities.SupportsLimitPushDown;
+import org.apache.flink.table.connector.source.abilities.SupportsProjectionPushDown;
+import org.apache.flink.table.connector.source.lookup.AsyncLookupFunctionProvider;
+import org.apache.flink.table.connector.source.lookup.LookupFunctionProvider;
+import org.apache.flink.table.data.RowData;
+import org.apache.flink.table.expressions.ResolvedExpression;
+import org.apache.flink.table.legacy.api.TableSchema;
+import org.apache.flink.table.types.DataType;
+import org.apache.flink.table.types.logical.RowType;
+import org.apache.flink.util.StringUtils;
+
+import org.apache.doris.flink.cfg.DorisLookupOptions;
+import org.apache.doris.flink.cfg.DorisOptions;
+import org.apache.doris.flink.cfg.DorisReadOptions;
+import org.apache.doris.flink.deserialization.RowDataDeserializationSchema;
+import org.apache.doris.flink.exception.DorisException;
+import org.apache.doris.flink.rest.PartitionDefinition;
+import org.apache.doris.flink.rest.RestService;
+import org.apache.doris.flink.source.DorisSource;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+import java.util.Objects;
+import java.util.stream.Collectors;
+
+/** The {@link DorisDynamicTableSource} is used during planning. */
+public final class DorisDynamicTableSource
+ implements ScanTableSource,
+ LookupTableSource,
+ SupportsFilterPushDown,
+ SupportsProjectionPushDown,
+ SupportsLimitPushDown {
+
+ private static final Logger LOG = LoggerFactory.getLogger(DorisDynamicTableSource.class);
+ private final DorisOptions options;
+ private final DorisReadOptions readOptions;
+ private DorisLookupOptions lookupOptions;
+ private TableSchema physicalSchema;
+ private List resolvedFilterQuery = new ArrayList<>();
+ private DataType physicalRowDataType;
+
+ public DorisDynamicTableSource(
+ DorisOptions options,
+ DorisReadOptions readOptions,
+ DorisLookupOptions lookupOptions,
+ TableSchema physicalSchema,
+ DataType physicalRowDataType) {
+ this.options = options;
+ this.lookupOptions = lookupOptions;
+ this.readOptions = readOptions;
+ this.physicalSchema = physicalSchema;
+ this.physicalRowDataType = physicalRowDataType;
+ }
+
+ @Override
+ public ChangelogMode getChangelogMode() {
+ // in our example the format decides about the changelog mode
+ // but it could also be the source itself
+ return ChangelogMode.insertOnly();
+ }
+
+ @Override
+ public ScanRuntimeProvider getScanRuntimeProvider(ScanContext runtimeProviderContext) {
+ if (!resolvedFilterQuery.isEmpty()) {
+ String filterQuery = resolvedFilterQuery.stream().collect(Collectors.joining(" AND "));
+ if (!StringUtils.isNullOrWhitespaceOnly(readOptions.getFilterQuery())) {
+ filterQuery =
+ String.format("(%s) AND (%s)", readOptions.getFilterQuery(), filterQuery);
+ }
+ readOptions.setFilterQuery(filterQuery);
+ }
+
+ if (StringUtils.isNullOrWhitespaceOnly(readOptions.getReadFields())) {
+ String[] selectFields =
+ DataType.getFieldNames(physicalRowDataType).toArray(new String[0]);
+ readOptions.setReadFields(
+ Arrays.stream(selectFields)
+ .map(item -> String.format("`%s`", item.trim().replace("`", "")))
+ .collect(Collectors.joining(", ")));
+ }
+
+ if (readOptions.getUseOldApi()) {
+ List dorisPartitions;
+ try {
+ dorisPartitions = RestService.findPartitions(options, readOptions, LOG);
+ } catch (DorisException e) {
+ throw new RuntimeException("Failed fetch doris partitions");
+ }
+ DorisRowDataInputFormat.Builder builder =
+ DorisRowDataInputFormat.builder()
+ .setFenodes(options.getFenodes())
+ .setBenodes(options.getBenodes())
+ .setUsername(options.getUsername())
+ .setPassword(options.getPassword())
+ .setTableIdentifier(options.getTableIdentifier())
+ .setPartitions(dorisPartitions)
+ .setReadOptions(readOptions)
+ .setRowType((RowType) physicalRowDataType.getLogicalType());
+ return InputFormatProvider.of(builder.build());
+ } else {
+ // Read data using the interface of the FLIP-27 specification
+ DorisSource build =
+ DorisSource.builder()
+ .setDorisReadOptions(readOptions)
+ .setDorisOptions(options)
+ .setDeserializer(
+ new RowDataDeserializationSchema(
+ (RowType) physicalRowDataType.getLogicalType()))
+ .build();
+ return SourceProvider.of(build);
+ }
+ }
+
+ @Override
+ public LookupRuntimeProvider getLookupRuntimeProvider(LookupContext context) {
+ String[] keyNames = new String[context.getKeys().length];
+ int[] keyIndexs = new int[context.getKeys().length];
+ for (int i = 0; i < keyNames.length; i++) {
+ int[] innerKeyArr = context.getKeys()[i];
+ keyNames[i] = DataType.getFieldNames(physicalRowDataType).get(innerKeyArr[0]);
+ keyIndexs[i] = innerKeyArr[0];
+ }
+ String[] selectFields = DataType.getFieldNames(physicalRowDataType).toArray(new String[0]);
+ DataType[] fieldTypes =
+ DataType.getFieldDataTypes(physicalRowDataType).toArray(new DataType[0]);
+ if (lookupOptions.isAsync()) {
+ return AsyncLookupFunctionProvider.of(
+ new DorisAsyncLookupAdapter(
+ options, lookupOptions, selectFields, fieldTypes, keyNames, keyIndexs));
+ } else {
+ return LookupFunctionProvider.of(
+ new DorisJdbcLookupAdapter(
+ options, lookupOptions, selectFields, fieldTypes, keyNames, keyIndexs));
+ }
+ }
+
+ @Override
+ public DynamicTableSource copy() {
+ // filterQuery/readFields of readOption may be overwritten in union all sql
+ DorisDynamicTableSource newSource =
+ new DorisDynamicTableSource(
+ options,
+ readOptions.copy(),
+ lookupOptions,
+ physicalSchema,
+ physicalRowDataType);
+ newSource.resolvedFilterQuery = new ArrayList<>(this.resolvedFilterQuery);
+ return newSource;
+ }
+
+ @Override
+ public String asSummaryString() {
+ return "Doris Table Source";
+ }
+
+ @Override
+ public Result applyFilters(List filters) {
+ List acceptedFilters = new ArrayList<>();
+ List remainingFilters = new ArrayList<>();
+
+ DorisExpressionVisitor expressionVisitor = new DorisExpressionVisitor();
+ for (ResolvedExpression filter : filters) {
+ String filterQuery = filter.accept(expressionVisitor);
+ if (!StringUtils.isNullOrWhitespaceOnly(filterQuery)) {
+ acceptedFilters.add(filter);
+ this.resolvedFilterQuery.add(filterQuery);
+ } else {
+ remainingFilters.add(filter);
+ }
+ }
+ return Result.of(acceptedFilters, remainingFilters);
+ }
+
+ @Override
+ public boolean supportsNestedProjection() {
+ return false;
+ }
+
+ @Override
+ public void applyProjection(int[][] projectedFields, DataType producedDataType) {
+ this.physicalRowDataType = Projection.of(projectedFields).project(physicalRowDataType);
+ String[] selectFields = DataType.getFieldNames(physicalRowDataType).toArray(new String[0]);
+ this.readOptions.setReadFields(
+ Arrays.stream(selectFields)
+ .map(item -> String.format("`%s`", item.trim().replace("`", "")))
+ .collect(Collectors.joining(", ")));
+ }
+
+ @VisibleForTesting
+ public List getResolvedFilterQuery() {
+ return resolvedFilterQuery;
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (this == o) {
+ return true;
+ }
+ if (o == null || getClass() != o.getClass()) {
+ return false;
+ }
+ DorisDynamicTableSource that = (DorisDynamicTableSource) o;
+ return Objects.equals(options, that.options)
+ && Objects.equals(readOptions, that.readOptions)
+ && Objects.equals(lookupOptions, that.lookupOptions)
+ && Objects.equals(physicalSchema, that.physicalSchema)
+ && Objects.equals(resolvedFilterQuery, that.resolvedFilterQuery)
+ && Objects.equals(physicalRowDataType, that.physicalRowDataType);
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(
+ options,
+ readOptions,
+ lookupOptions,
+ physicalSchema,
+ resolvedFilterQuery,
+ physicalRowDataType);
+ }
+
+ @Override
+ public void applyLimit(long limit) {
+ // partial limit push down to reduce the amount of data scanned
+ readOptions.setRowLimit(limit);
+ }
+}
diff --git a/flink-doris-connector/flink-doris-connector-2.2/src/main/java/org/apache/doris/flink/table/DorisJdbcLookupAdapter.java b/flink-doris-connector/flink-doris-connector-2.2/src/main/java/org/apache/doris/flink/table/DorisJdbcLookupAdapter.java
new file mode 100644
index 000000000..69e745984
--- /dev/null
+++ b/flink-doris-connector/flink-doris-connector-2.2/src/main/java/org/apache/doris/flink/table/DorisJdbcLookupAdapter.java
@@ -0,0 +1,100 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+package org.apache.doris.flink.table;
+
+import org.apache.flink.table.data.RowData;
+import org.apache.flink.table.functions.FunctionContext;
+import org.apache.flink.table.functions.LookupFunction;
+import org.apache.flink.table.types.DataType;
+import org.apache.flink.util.Collector;
+
+import org.apache.doris.flink.cfg.DorisLookupOptions;
+import org.apache.doris.flink.cfg.DorisOptions;
+import org.apache.doris.flink.deserialization.converter.DorisRowConverter;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.List;
+
+/**
+ * Adapter that bridges Flink 2.2 {@link LookupFunction} to the existing {@link
+ * DorisRowDataJdbcLookupFunction} which is implemented against the older {@link
+ * org.apache.flink.table.functions.TableFunction} API and shared with Flink 1.15–1.20.
+ */
+public class DorisJdbcLookupAdapter extends LookupFunction {
+
+ private final DorisRowDataJdbcLookupFunction delegate;
+ private final DorisRowConverter keyConverter;
+ private final int[] keyIndex;
+
+ public DorisJdbcLookupAdapter(
+ DorisOptions options,
+ DorisLookupOptions lookupOptions,
+ String[] selectFields,
+ DataType[] fieldTypes,
+ String[] conditionFields,
+ int[] keyIndex) {
+ this.delegate =
+ new DorisRowDataJdbcLookupFunction(
+ options,
+ lookupOptions,
+ selectFields,
+ fieldTypes,
+ conditionFields,
+ keyIndex);
+ this.keyIndex = keyIndex;
+ DataType[] keyTypes = new DataType[keyIndex.length];
+ for (int i = 0; i < keyIndex.length; i++) {
+ keyTypes[i] = fieldTypes[keyIndex[i]];
+ }
+ this.keyConverter = new DorisRowConverter(keyTypes);
+ }
+
+ @Override
+ public void open(FunctionContext context) throws Exception {
+ super.open(context);
+ delegate.open(context);
+ }
+
+ @Override
+ public Collection lookup(RowData keyRow) throws IOException {
+ final List results = new ArrayList<>();
+ delegate.setCollector(
+ new Collector() {
+ @Override
+ public void collect(RowData record) {
+ results.add(record);
+ }
+
+ @Override
+ public void close() {}
+ });
+
+ delegate.eval(keyRow);
+ return results;
+ }
+
+ @Override
+ public void close() throws Exception {
+ try {
+ delegate.close();
+ } finally {
+ super.close();
+ }
+ }
+}
diff --git a/flink-doris-connector/flink-doris-connector-2.2/src/test/java/org/apache/doris/flink/utils/CatalogUtil.java b/flink-doris-connector/flink-doris-connector-2.2/src/test/java/org/apache/doris/flink/utils/CatalogUtil.java
new file mode 100644
index 000000000..ccb48680b
--- /dev/null
+++ b/flink-doris-connector/flink-doris-connector-2.2/src/test/java/org/apache/doris/flink/utils/CatalogUtil.java
@@ -0,0 +1,52 @@
+package org.apache.doris.flink.utils;
+
+import org.apache.flink.table.api.DataTypes;
+import org.apache.flink.table.api.Schema;
+import org.apache.flink.table.api.Schema.UnresolvedColumn;
+import org.apache.flink.table.catalog.CatalogTable;
+import org.apache.flink.table.types.AtomicDataType;
+import org.apache.flink.table.types.logical.VarCharType;
+
+import java.util.ArrayList;
+import java.util.Map;
+
+public class CatalogUtil {
+ public static CatalogTable createTable(Schema tableSchema, Map options) {
+ return CatalogTable.newBuilder()
+ .comment("FlinkTable")
+ .partitionKeys(new ArrayList<>())
+ .schema(tableSchema)
+ .options(options)
+ .build();
+ }
+
+ public static Schema getTableSchema() {
+ return Schema.newBuilder()
+ .column("id", new AtomicDataType(new VarCharType(false, 128)))
+ .column("c_boolean", DataTypes.BOOLEAN())
+ .column("c_char", DataTypes.CHAR(1))
+ .column("c_date", DataTypes.DATE())
+ .column("c_datetime", DataTypes.TIMESTAMP(0))
+ .column("c_decimal", DataTypes.DECIMAL(10, 2))
+ .column("c_double", DataTypes.DOUBLE())
+ .column("c_float", DataTypes.FLOAT())
+ .column("c_int", DataTypes.INT())
+ .column("c_bigint", DataTypes.BIGINT())
+ .column("c_largeint", DataTypes.STRING())
+ .column("c_smallint", DataTypes.SMALLINT())
+ .column("c_string", DataTypes.STRING())
+ .column("c_tinyint", DataTypes.TINYINT())
+ .column("c_array", DataTypes.ARRAY(DataTypes.INT()))
+ .column("c_map", DataTypes.MAP(DataTypes.STRING(), DataTypes.STRING()))
+ .column("c_row", DataTypes.ROW())
+ .column("c_varbinary", DataTypes.VARBINARY(16))
+ .primaryKey("id")
+ .build();
+ }
+
+ public static String[] getColumns() {
+ return getTableSchema().getColumns().stream()
+ .map(UnresolvedColumn::getName)
+ .toArray(String[]::new);
+ }
+}
diff --git a/flink-doris-connector/flink-doris-connector-2.2/src/test/java/org/apache/doris/flink/utils/MockMultiTableSource.java b/flink-doris-connector/flink-doris-connector-2.2/src/test/java/org/apache/doris/flink/utils/MockMultiTableSource.java
new file mode 100644
index 000000000..ca7aefbb1
--- /dev/null
+++ b/flink-doris-connector/flink-doris-connector-2.2/src/test/java/org/apache/doris/flink/utils/MockMultiTableSource.java
@@ -0,0 +1,148 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.flink.utils;
+
+import org.apache.flink.api.common.state.CheckpointListener;
+import org.apache.flink.api.common.state.ListState;
+import org.apache.flink.api.common.state.ListStateDescriptor;
+import org.apache.flink.api.common.typeinfo.TypeInformation;
+import org.apache.flink.runtime.state.FunctionInitializationContext;
+import org.apache.flink.runtime.state.FunctionSnapshotContext;
+import org.apache.flink.streaming.api.checkpoint.CheckpointedFunction;
+import org.apache.flink.streaming.api.functions.source.legacy.RichParallelSourceFunction;
+import org.apache.flink.util.SerializableObject;
+
+import org.apache.doris.flink.sink.batch.RecordWithMeta;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.Collections;
+import java.util.Iterator;
+
+/*
+ * MockMultiTableSource is a mock source for testing multi-table source
+ * */
+public class MockMultiTableSource extends RichParallelSourceFunction