Skip to content

Commit 20ed72c

Browse files
authored
[IcebergIO] Support TableIdentifiers with special characters (#38876)
1 parent 42daed3 commit 20ed72c

18 files changed

Lines changed: 152 additions & 34 deletions
Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
11
{
22
"comment": "Modify this file in a trivial way to cause this test suite to run.",
3-
"modification": 1
3+
"modification": 2
44
}

sdks/java/io/iceberg/src/main/java/org/apache/beam/sdk/io/iceberg/AddFiles.java

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -523,7 +523,7 @@ static <W, T> T transformValue(Transform<W, T> transform, Type type, ByteBuffer
523523
}
524524

525525
private Table getOrCreateTable(String filePath, FileFormat format) throws IOException {
526-
TableIdentifier tableId = TableIdentifier.parse(identifier);
526+
TableIdentifier tableId = IcebergUtils.parseTableIdentifier(identifier);
527527
try {
528528
return catalogConfig.catalog().loadTable(tableId);
529529
} catch (NoSuchTableException e) {
@@ -549,7 +549,7 @@ private Table getOrCreateTable(String filePath, FileFormat format) throws IOExce
549549
.create();
550550

551551
} catch (AlreadyExistsException e2) { // if table already exists, just load it
552-
return catalogConfig.catalog().loadTable(TableIdentifier.parse(identifier));
552+
return catalogConfig.catalog().loadTable(IcebergUtils.parseTableIdentifier(identifier));
553553
}
554554
}
555555
}
@@ -684,7 +684,7 @@ public void process(
684684
return;
685685
}
686686
if (table == null) {
687-
table = catalogConfig.catalog().loadTable(TableIdentifier.parse(identifier));
687+
table = catalogConfig.catalog().loadTable(IcebergUtils.parseTableIdentifier(identifier));
688688
}
689689

690690
PartitionSpec spec = checkStateNotNull(table.specs().get(batch.getKey()));
@@ -769,7 +769,7 @@ public void process(
769769
}
770770
String commitId = commitHash(manifests);
771771
if (table == null) {
772-
table = catalogConfig.catalog().loadTable(TableIdentifier.parse(identifier));
772+
table = catalogConfig.catalog().loadTable(IcebergUtils.parseTableIdentifier(identifier));
773773
}
774774
table.refresh();
775775
ensureNameMappingPresent(table);

sdks/java/io/iceberg/src/main/java/org/apache/beam/sdk/io/iceberg/AppendFilesToTables.java

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -47,7 +47,6 @@
4747
import org.apache.iceberg.Snapshot;
4848
import org.apache.iceberg.Table;
4949
import org.apache.iceberg.catalog.Catalog;
50-
import org.apache.iceberg.catalog.TableIdentifier;
5150
import org.apache.iceberg.io.FileIO;
5251
import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
5352
import org.slf4j.Logger;
@@ -75,7 +74,7 @@ public PCollection<KV<String, SnapshotInfo>> expand(PCollection<FileWriteResult>
7574
new SerializableFunction<FileWriteResult, String>() {
7675
@Override
7776
public String apply(FileWriteResult input) {
78-
return input.getTableIdentifier().toString();
77+
return IcebergUtils.tableIdentifierToString(input.getTableIdentifier());
7978
}
8079
}))
8180
.apply("Group metadata updates by table", GroupByKey.create())
@@ -128,7 +127,7 @@ public void processElement(
128127
BoundedWindow window)
129128
throws IOException {
130129
String tableStringIdentifier = element.getKey();
131-
Table table = getCatalog().loadTable(TableIdentifier.parse(element.getKey()));
130+
Table table = getCatalog().loadTable(IcebergUtils.parseTableIdentifier(element.getKey()));
132131
Iterable<FileWriteResult> fileWriteResults = element.getValue();
133132
if (shouldSkip(table, fileWriteResults)) {
134133
return;

sdks/java/io/iceberg/src/main/java/org/apache/beam/sdk/io/iceberg/AssignDestinationsAndPartitions.java

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -35,7 +35,6 @@
3535
import org.apache.iceberg.PartitionKey;
3636
import org.apache.iceberg.PartitionSpec;
3737
import org.apache.iceberg.Schema;
38-
import org.apache.iceberg.catalog.TableIdentifier;
3938
import org.apache.iceberg.exceptions.NoSuchTableException;
4039
import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
4140
import org.checkerframework.checker.nullness.qual.Nullable;
@@ -149,7 +148,7 @@ public void processElement(
149148
// see if table already exists with a spec
150149
spec =
151150
TableCache.getAndRefreshIfStale(
152-
catalogConfig, TableIdentifier.parse(tableIdentifier))
151+
catalogConfig, IcebergUtils.parseTableIdentifier(tableIdentifier))
153152
.spec();
154153

155154
} catch (NoSuchTableException ignored) {

sdks/java/io/iceberg/src/main/java/org/apache/beam/sdk/io/iceberg/FileWriteResult.java

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -44,7 +44,7 @@ abstract class FileWriteResult {
4444
@SchemaIgnore
4545
public TableIdentifier getTableIdentifier() {
4646
if (cachedTableIdentifier == null) {
47-
cachedTableIdentifier = TableIdentifier.parse(getTableIdentifierString());
47+
cachedTableIdentifier = IcebergUtils.parseTableIdentifier(getTableIdentifierString());
4848
}
4949
return cachedTableIdentifier;
5050
}
@@ -70,7 +70,7 @@ abstract static class Builder {
7070

7171
@SchemaIgnore
7272
public Builder setTableIdentifier(TableIdentifier tableId) {
73-
return setTableIdentifierString(tableId.toString());
73+
return setTableIdentifierString(IcebergUtils.tableIdentifierToString(tableId));
7474
}
7575

7676
public abstract FileWriteResult build();

sdks/java/io/iceberg/src/main/java/org/apache/beam/sdk/io/iceberg/IcebergCatalogConfig.java

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -155,7 +155,7 @@ public void createTable(
155155
Schema tableSchema,
156156
@Nullable List<String> partitionFields,
157157
@Nullable Map<String, String> properties) {
158-
TableIdentifier icebergIdentifier = TableIdentifier.parse(tableIdentifier);
158+
TableIdentifier icebergIdentifier = IcebergUtils.parseTableIdentifier(tableIdentifier);
159159
org.apache.iceberg.Schema icebergSchema = IcebergUtils.beamSchemaToIcebergSchema(tableSchema);
160160
PartitionSpec icebergSpec = PartitionUtils.toPartitionSpec(partitionFields, tableSchema);
161161
try {
@@ -178,7 +178,7 @@ public void createTable(
178178
}
179179

180180
public @Nullable IcebergTableInfo loadTable(String tableIdentifier) {
181-
TableIdentifier icebergIdentifier = TableIdentifier.parse(tableIdentifier);
181+
TableIdentifier icebergIdentifier = IcebergUtils.parseTableIdentifier(tableIdentifier);
182182
try {
183183
Table table = catalog().loadTable(icebergIdentifier);
184184
return new IcebergTableInfo(tableIdentifier, table);
@@ -270,7 +270,7 @@ public void updatePartitionSpec(
270270
}
271271

272272
public boolean dropTable(String tableIdentifier) {
273-
TableIdentifier icebergIdentifier = TableIdentifier.parse(tableIdentifier);
273+
TableIdentifier icebergIdentifier = IcebergUtils.parseTableIdentifier(tableIdentifier);
274274
return catalog().dropTable(icebergIdentifier);
275275
}
276276

sdks/java/io/iceberg/src/main/java/org/apache/beam/sdk/io/iceberg/IcebergCdcReadSchemaTransformProvider.java

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -41,7 +41,6 @@
4141
import org.apache.beam.sdk.values.Row;
4242
import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Enums;
4343
import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.base.Optional;
44-
import org.apache.iceberg.catalog.TableIdentifier;
4544
import org.checkerframework.checker.nullness.qual.Nullable;
4645
import org.joda.time.Duration;
4746

@@ -109,7 +108,7 @@ public PCollectionRowTuple expand(PCollectionRowTuple input) {
109108
IcebergIO.ReadRows readRows =
110109
IcebergIO.readRows(configuration.getIcebergCatalog())
111110
.withCdc()
112-
.from(TableIdentifier.parse(configuration.getTable()))
111+
.from(IcebergUtils.parseTableIdentifier(configuration.getTable()))
113112
.fromSnapshot(configuration.getFromSnapshot())
114113
.toSnapshot(configuration.getToSnapshot())
115114
.fromTimestamp(configuration.getFromTimestamp())

sdks/java/io/iceberg/src/main/java/org/apache/beam/sdk/io/iceberg/IcebergReadSchemaTransformProvider.java

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -37,7 +37,6 @@
3737
import org.apache.beam.sdk.values.PCollection;
3838
import org.apache.beam.sdk.values.PCollectionRowTuple;
3939
import org.apache.beam.sdk.values.Row;
40-
import org.apache.iceberg.catalog.TableIdentifier;
4140
import org.checkerframework.checker.nullness.qual.Nullable;
4241

4342
/**
@@ -93,7 +92,7 @@ public PCollectionRowTuple expand(PCollectionRowTuple input) {
9392
.getPipeline()
9493
.apply(
9594
IcebergIO.readRows(configuration.getIcebergCatalog())
96-
.from(TableIdentifier.parse(configuration.getTable()))
95+
.from(IcebergUtils.parseTableIdentifier(configuration.getTable()))
9796
.keeping(configuration.getKeep())
9897
.dropping(configuration.getDrop())
9998
.withFilter(configuration.getFilter()));

sdks/java/io/iceberg/src/main/java/org/apache/beam/sdk/io/iceberg/IcebergScanConfig.java

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -80,7 +80,9 @@ public enum ScanType {
8080
@Pure
8181
public Table getTable() {
8282
if (cachedTable == null) {
83-
cachedTable = TableCache.get(getCatalogConfig(), TableIdentifier.parse(getTableIdentifier()));
83+
cachedTable =
84+
TableCache.get(
85+
getCatalogConfig(), IcebergUtils.parseTableIdentifier(getTableIdentifier()));
8486
}
8587
return cachedTable;
8688
}
@@ -294,7 +296,7 @@ public abstract static class Builder {
294296
public abstract Builder setTableIdentifier(String tableIdentifier);
295297

296298
public Builder setTableIdentifier(TableIdentifier tableIdentifier) {
297-
return this.setTableIdentifier(tableIdentifier.toString());
299+
return this.setTableIdentifier(IcebergUtils.tableIdentifierToString(tableIdentifier));
298300
}
299301

300302
public Builder setTableIdentifier(String... names) {

sdks/java/io/iceberg/src/main/java/org/apache/beam/sdk/io/iceberg/IcebergUtils.java

Lines changed: 54 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -46,6 +46,8 @@
4646
import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.annotations.VisibleForTesting;
4747
import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableList;
4848
import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.ImmutableMap;
49+
import org.apache.iceberg.catalog.TableIdentifier;
50+
import org.apache.iceberg.catalog.TableIdentifierParser;
4951
import org.apache.iceberg.data.GenericRecord;
5052
import org.apache.iceberg.data.Record;
5153
import org.apache.iceberg.types.Type;
@@ -631,6 +633,58 @@ private static Object getLogicalTypeValue(Object icebergValue, Schema.FieldType
631633
return icebergValue;
632634
}
633635

636+
/** Serializes a table identifier without losing dots or other special characters in each part. */
637+
public static String tableIdentifierToString(TableIdentifier tableIdentifier) {
638+
TableIdentifier identifier = checkArgumentNotNull(tableIdentifier);
639+
return requiresJsonTableIdentifier(identifier)
640+
? TableIdentifierParser.toJson(identifier)
641+
: identifier.toString();
642+
}
643+
644+
/** Parses either Iceberg's JSON table identifier representation or the legacy dotted form. */
645+
public static TableIdentifier parseTableIdentifier(String table) {
646+
if (looksLikeJsonObject(table)) {
647+
return TableIdentifierParser.fromJson(table);
648+
}
649+
650+
return TableIdentifier.parse(table);
651+
}
652+
653+
private static boolean looksLikeJsonObject(@Nullable String value) {
654+
if (value == null) {
655+
return false;
656+
}
657+
658+
int start = 0;
659+
while (start < value.length() && Character.isWhitespace(value.charAt(start))) {
660+
start++;
661+
}
662+
if (start == value.length() || value.charAt(start) != '{') {
663+
return false;
664+
}
665+
666+
int end = value.length() - 1;
667+
while (end >= 0 && Character.isWhitespace(value.charAt(end))) {
668+
end--;
669+
}
670+
671+
return end >= 0 && value.charAt(end) == '}';
672+
}
673+
674+
private static boolean requiresJsonTableIdentifier(TableIdentifier identifier) {
675+
if (looksLikeJsonObject(identifier.toString())) {
676+
return true;
677+
}
678+
679+
for (String level : identifier.namespace().levels()) {
680+
if (level.contains(".")) {
681+
return true;
682+
}
683+
}
684+
685+
return identifier.name().contains(".");
686+
}
687+
634688
static <T> boolean isUnbounded(PCollection<T> input) {
635689
return input.isBounded().equals(PCollection.IsBounded.UNBOUNDED);
636690
}

0 commit comments

Comments
 (0)