Skip to content

Fix: Adjust VARCHAR byte length calculation to fully support UTF-8 #621

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
cs17899219 wants to merge 1 commit into
base: master
Choose a base branch
from
Open

Fix: Adjust VARCHAR byte length calculation to fully support UTF-8 #621

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions flink-doris-connector/src/main/java/org/apache/doris/flink/catalog/DorisTypeMapper.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -155,7 +155,7 @@ private static class LogicalTypeVisitor extends LogicalTypeDefaultVisitor<String

@Override
public String visit(CharType charType) {
long length = charType.getLength() * 3L;
long length = charType.getLength() * 4L;
if (length <= MAX_CHAR_SIZE) {
return String.format("%s(%s)", DorisType.CHAR, length);
} else {
Expand All @@ -166,7 +166,7 @@ public String visit(CharType charType) {
@Override
public String visit(VarCharType varCharType) {
// Flink varchar length max value is int, it may overflow after multiplying by 3
long length = varCharType.getLength() * 3L;
long length = varCharType.getLength() * 4L;
return length >= MAX_VARCHAR_SIZE ? STRING : String.format("%s(%s)", VARCHAR, length);
}

Expand Down
4 changes: 2 additions & 2 deletions .../apache/doris/flink/sink/writer/serializer/jsondebezium/JsonDebeziumSchemaChangeImpl.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -148,12 +148,12 @@ private String handleType(String type) {
if (type == null || "".equals(type)) {
return "";
}
// varchar len * 3
// varchar len * 4
Pattern pattern = Pattern.compile("varchar\\(([1-9][0-9]*)\\)", Pattern.CASE_INSENSITIVE);
Matcher matcher = pattern.matcher(type);
if (matcher.find()) {
String len = matcher.group(1);
return String.format("varchar(%d)", Math.min(Integer.parseInt(len) * 3, 65533));
return String.format("varchar(%d)", Math.min(Integer.parseInt(len) * 4, 65533));
}

return type;
Expand Down
4 changes: 2 additions & 2 deletions flink-doris-connector/src/main/java/org/apache/doris/flink/tools/cdc/db2/Db2Type.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -76,9 +76,9 @@ public static String toDorisType(String db2Type, Integer precision, Integer scal
case VARCHAR:
case LONG_VARCHAR:
Preconditions.checkNotNull(precision);
return precision * 3 > 65533
return precision * 4 > 65533
? DorisType.STRING
: String.format("%s(%s)", DorisType.VARCHAR, precision * 3);
: String.format("%s(%s)", DorisType.VARCHAR, precision * 4);
case TIMESTAMP:
return String.format(
"%s(%s)", DorisType.DATETIME_V2, Math.min(scale == null ? 0 : scale, 6));
Expand Down
4 changes: 2 additions & 2 deletions flink-doris-connector/src/main/java/org/apache/doris/flink/tools/cdc/mysql/MysqlType.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -192,9 +192,9 @@ public static String toDorisType(String type, Integer length, Integer scale) {
case CHAR:
case VARCHAR:
Preconditions.checkNotNull(length);
return length * 3 > 65533
return length * 4 > 65533
? DorisType.STRING
: String.format("%s(%s)", DorisType.VARCHAR, length * 3);
: String.format("%s(%s)", DorisType.VARCHAR, length * 4);
case TINYTEXT:
case TEXT:
case MEDIUMTEXT:
Expand Down
4 changes: 2 additions & 2 deletions flink-doris-connector/src/main/java/org/apache/doris/flink/tools/cdc/oracle/OracleType.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -91,9 +91,9 @@ public static String toDorisType(String oracleType, Integer precision, Integer s
case NCHAR:
case NVARCHAR2:
Preconditions.checkNotNull(precision);
return precision * 3 > 65533
return precision * 4 > 65533
? DorisType.STRING
: String.format("%s(%s)", DorisType.VARCHAR, precision * 3);
: String.format("%s(%s)", DorisType.VARCHAR, precision * 4);
case LONG:
case RAW:
case LONG_RAW:
Expand Down
4 changes: 2 additions & 2 deletions ...doris-connector/src/main/java/org/apache/doris/flink/tools/cdc/postgres/PostgresType.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -107,9 +107,9 @@ public static String toDorisType(String postgresType, Integer precision, Integer
case BPCHAR:
case VARCHAR:
Preconditions.checkNotNull(precision);
return precision * 3 > 65533
return precision * 4 > 65533
? DorisType.STRING
: String.format("%s(%s)", DorisType.VARCHAR, precision * 3);
: String.format("%s(%s)", DorisType.VARCHAR, precision * 4);
case POINT:
case LINE:
case LSEG:
Expand Down
4 changes: 2 additions & 2 deletions ...ris-connector/src/main/java/org/apache/doris/flink/tools/cdc/sqlserver/SqlServerType.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -94,9 +94,9 @@ public static String toDorisType(String originSqlServerType, Integer precision,
case VARCHAR:
case NCHAR:
case NVARCHAR:
return precision * 3 > 65533
return precision * 4 > 65533
? DorisType.STRING
: String.format("%s(%s)", DorisType.VARCHAR, precision * 3);
: String.format("%s(%s)", DorisType.VARCHAR, precision * 4);
case TEXT:
case NTEXT:
case TIME:
Expand Down
6 changes: 3 additions & 3 deletions flink-doris-connector/src/test/java/org/apache/doris/flink/catalog/DorisTypeMapperTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -109,17 +109,17 @@ public void testCharType() {
DataType result = DorisTypeMapper.toFlinkType("col", "CHAR", 10, 0);
assertEquals(DataTypes.CHAR(10), result);
String dorisType = DorisTypeMapper.toDorisType(DataTypes.CHAR(10));
assertEquals("CHAR(30)", dorisType);
assertEquals("CHAR(40)", dorisType);
dorisType = DorisTypeMapper.toDorisType(DataTypes.CHAR(100));
assertEquals("VARCHAR(300)", dorisType);
assertEquals("VARCHAR(400)", dorisType);
}

@Test
public void testVarcharType() {
DataType result = DorisTypeMapper.toFlinkType("col", "VARCHAR", 50, 0);
assertEquals(DataTypes.VARCHAR(50), result);
String dorisType = DorisTypeMapper.toDorisType(DataTypes.VARCHAR(50));
assertEquals("VARCHAR(150)", dorisType);
assertEquals("VARCHAR(200)", dorisType);
}

@Test
Expand Down
Loading