Skip to content

Commit 9d2f65e

Browse files
morningmanmorningman
authored
[feat](iceberg) support aliyun dlf iceberg rest catalog (apache#60796)
### What problem does this PR solve? depends on apache#60856 Support Aliyun DLF (Data Lake Formation) as an Iceberg REST catalog. Main changes: - Preserve case-sensitive REST signing names by removing toLowerCase()on `rest.signing-name`, so names like "DlfNext" work correctly. - Prioritize non-S3 storage properties when multiple S3-compatible properties are returned from iceberg rest catalog, to avoid incorrectly using S3 credentials for non-S3 storage (e.g. OSS). - Remove redundant resetToUninitialized() override in HMSExternalCatalog that only delegates to super. - Add IcebergDlfRestCatalogTest, IcebergRestPropertiesTest and StoragePropertiesTest for DLF REST catalog integration and property resolution testing. ``` CREATE CATALOG ice PROPERTIES ( 'type' = 'iceberg', 'warehouse' = 'new_dlf_iceberg_catalog', 'iceberg.catalog.type' = 'rest', 'iceberg.rest.uri' = 'http://cn-beijing-vpc.dlf.aliyuncs.com/iceberg', 'iceberg.rest.sigv4-enabled' = 'true', 'iceberg.rest.signing-name' = 'DlfNext', 'iceberg.rest.access-key-id' = 'xx', 'iceberg.rest.secret-access-key' = 'xx', 'iceberg.rest.signing-region' = 'cn-beijing', 'iceberg.rest.vended-credentials-enabled' = 'true', 'io-impl' = 'org.apache.iceberg.rest.DlfFileIO', 'fs.oss.support' = 'true' ); ```
1 parent 8364e8a commit 9d2f65e

40 files changed

Lines changed: 1119 additions & 305 deletions

‎build.sh‎

Lines changed: 28 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -474,6 +474,12 @@ if [[ "$(echo "${DISABLE_BUILD_AZURE}" | tr '[:lower:]' '[:upper:]')" == "ON" ]]
474474
BUILD_AZURE='OFF'
475475
fi
476476

477+
if [[ "$(echo "${DISABLE_BUILD_JINDOFS}" | tr '[:lower:]' '[:upper:]')" == "ON" ]]; then
478+
BUILD_JINDOFS='OFF'
479+
else
480+
BUILD_JINDOFS='ON'
481+
fi
482+
477483
if [[ -z "${ENABLE_INJECTION_POINT}" ]]; then
478484
ENABLE_INJECTION_POINT='OFF'
479485
fi
@@ -795,7 +801,9 @@ if [[ "${BUILD_FE}" -eq 1 ]]; then
795801
cp -r -p "${DORIS_HOME}/conf/ldap.conf" "${DORIS_OUTPUT}/fe/conf"/
796802
cp -r -p "${DORIS_HOME}/conf/mysql_ssl_default_certificate" "${DORIS_OUTPUT}/fe/"/
797803
rm -rf "${DORIS_OUTPUT}/fe/lib"/*
798-
install -d "${DORIS_OUTPUT}/fe/lib/jindofs"
804+
if [[ "${BUILD_JINDOFS}" == "ON" ]]; then
805+
install -d "${DORIS_OUTPUT}/fe/lib/jindofs"
806+
fi
799807
cp -r -p "${DORIS_HOME}/fe/fe-core/target/lib"/* "${DORIS_OUTPUT}/fe/lib"/
800808
cp -r -p "${DORIS_HOME}/fe/fe-core/target/doris-fe.jar" "${DORIS_OUTPUT}/fe/lib"/
801809
if [[ "${WITH_TDE_DIR}" != "" ]]; then
@@ -805,13 +813,15 @@ if [[ "${BUILD_FE}" -eq 1 ]]; then
805813
#cp -r -p "${DORIS_HOME}/docs/build/help-resource.zip" "${DORIS_OUTPUT}/fe/lib"/
806814

807815
# copy jindofs jars, only support for Linux x64 or arm
808-
if [[ "${TARGET_SYSTEM}" == 'Linux' ]] && [[ "${TARGET_ARCH}" == 'x86_64' ]]; then
809-
cp -r -p "${DORIS_THIRDPARTY}"/installed/jindofs_libs/jindo-core-[0-9]*.jar "${DORIS_OUTPUT}/fe/lib/jindofs"/
810-
cp -r -p "${DORIS_THIRDPARTY}"/installed/jindofs_libs/jindo-core-linux-ubuntu22-x86_64-[0-9]*.jar "${DORIS_OUTPUT}/fe/lib/jindofs"/
811-
cp -r -p "${DORIS_THIRDPARTY}"/installed/jindofs_libs/jindo-sdk-[0-9]*.jar "${DORIS_OUTPUT}/fe/lib/jindofs"/
812-
elif [[ "${TARGET_SYSTEM}" == 'Linux' ]] && [[ "${TARGET_ARCH}" == 'aarch64' ]]; then
813-
cp -r -p "${DORIS_THIRDPARTY}"/installed/jindofs_libs/jindo-core-linux-el7-aarch64-[0-9]*.jar "${DORIS_OUTPUT}/fe/lib/jindofs"/
814-
cp -r -p "${DORIS_THIRDPARTY}"/installed/jindofs_libs/jindo-sdk-[0-9]*.jar "${DORIS_OUTPUT}/fe/lib/jindofs"/
816+
if [[ "${BUILD_JINDOFS}" == "ON" ]]; then
817+
if [[ "${TARGET_SYSTEM}" == 'Linux' ]] && [[ "${TARGET_ARCH}" == 'x86_64' ]]; then
818+
cp -r -p "${DORIS_THIRDPARTY}"/installed/jindofs_libs/jindo-core-[0-9]*.jar "${DORIS_OUTPUT}/fe/lib/jindofs"/
819+
cp -r -p "${DORIS_THIRDPARTY}"/installed/jindofs_libs/jindo-core-linux-ubuntu22-x86_64-[0-9]*.jar "${DORIS_OUTPUT}/fe/lib/jindofs"/
820+
cp -r -p "${DORIS_THIRDPARTY}"/installed/jindofs_libs/jindo-sdk-[0-9]*.jar "${DORIS_OUTPUT}/fe/lib/jindofs"/
821+
elif [[ "${TARGET_SYSTEM}" == 'Linux' ]] && [[ "${TARGET_ARCH}" == 'aarch64' ]]; then
822+
cp -r -p "${DORIS_THIRDPARTY}"/installed/jindofs_libs/jindo-core-linux-el7-aarch64-[0-9]*.jar "${DORIS_OUTPUT}/fe/lib/jindofs"/
823+
cp -r -p "${DORIS_THIRDPARTY}"/installed/jindofs_libs/jindo-sdk-[0-9]*.jar "${DORIS_OUTPUT}/fe/lib/jindofs"/
824+
fi
815825
fi
816826

817827
cp -r -p "${DORIS_HOME}/minidump" "${DORIS_OUTPUT}/fe"/
@@ -981,14 +991,16 @@ EOF
981991
done
982992

983993
# copy jindofs jars, only support for Linux x64 or arm
984-
install -d "${DORIS_OUTPUT}/be/lib/java_extensions/jindofs"/
985-
if [[ "${TARGET_SYSTEM}" == 'Linux' ]] && [[ "$TARGET_ARCH" == 'x86_64' ]]; then
986-
cp -r -p "${DORIS_THIRDPARTY}"/installed/jindofs_libs/jindo-core-[0-9]*.jar "${DORIS_OUTPUT}/be/lib/java_extensions/jindofs"/
987-
cp -r -p "${DORIS_THIRDPARTY}"/installed/jindofs_libs/jindo-core-linux-ubuntu22-x86_64-[0-9]*.jar "${DORIS_OUTPUT}/be/lib/java_extensions/jindofs"/
988-
cp -r -p "${DORIS_THIRDPARTY}"/installed/jindofs_libs/jindo-sdk-[0-9]*.jar "${DORIS_OUTPUT}/be/lib/java_extensions/jindofs"/
989-
elif [[ "${TARGET_SYSTEM}" == 'Linux' ]] && [[ "$TARGET_ARCH" == 'aarch64' ]]; then
990-
cp -r -p "${DORIS_THIRDPARTY}"/installed/jindofs_libs/jindo-core-linux-el7-aarch64-[0-9]*.jar "${DORIS_OUTPUT}/be/lib/java_extensions/jindofs"/
991-
cp -r -p "${DORIS_THIRDPARTY}"/installed/jindofs_libs/jindo-sdk-[0-9]*.jar "${DORIS_OUTPUT}/be/lib/java_extensions/jindofs"/
994+
if [[ "${BUILD_JINDOFS}" == "ON" ]]; then
995+
install -d "${DORIS_OUTPUT}/be/lib/java_extensions/jindofs"/
996+
if [[ "${TARGET_SYSTEM}" == 'Linux' ]] && [[ "$TARGET_ARCH" == 'x86_64' ]]; then
997+
cp -r -p "${DORIS_THIRDPARTY}"/installed/jindofs_libs/jindo-core-[0-9]*.jar "${DORIS_OUTPUT}/be/lib/java_extensions/jindofs"/
998+
cp -r -p "${DORIS_THIRDPARTY}"/installed/jindofs_libs/jindo-core-linux-ubuntu22-x86_64-[0-9]*.jar "${DORIS_OUTPUT}/be/lib/java_extensions/jindofs"/
999+
cp -r -p "${DORIS_THIRDPARTY}"/installed/jindofs_libs/jindo-sdk-[0-9]*.jar "${DORIS_OUTPUT}/be/lib/java_extensions/jindofs"/
1000+
elif [[ "${TARGET_SYSTEM}" == 'Linux' ]] && [[ "$TARGET_ARCH" == 'aarch64' ]]; then
1001+
cp -r -p "${DORIS_THIRDPARTY}"/installed/jindofs_libs/jindo-core-linux-el7-aarch64-[0-9]*.jar "${DORIS_OUTPUT}/be/lib/java_extensions/jindofs"/
1002+
cp -r -p "${DORIS_THIRDPARTY}"/installed/jindofs_libs/jindo-sdk-[0-9]*.jar "${DORIS_OUTPUT}/be/lib/java_extensions/jindofs"/
1003+
fi
9921004
fi
9931005

9941006
cp -r -p "${DORIS_THIRDPARTY}/installed/webroot"/* "${DORIS_OUTPUT}/be/www"/

‎fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogProperty.java‎

Lines changed: 18 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -18,10 +18,13 @@
1818
package org.apache.doris.datasource;
1919

2020
import org.apache.doris.common.UserException;
21+
import org.apache.doris.datasource.credentials.AbstractVendedCredentialsProvider;
22+
import org.apache.doris.datasource.credentials.VendedCredentialsFactory;
2123
import org.apache.doris.datasource.property.metastore.MetastoreProperties;
2224
import org.apache.doris.datasource.property.storage.StorageProperties;
2325

2426
import com.google.common.base.Preconditions;
27+
import com.google.common.collect.Lists;
2528
import com.google.common.collect.Maps;
2629
import com.google.gson.annotations.SerializedName;
2730
import org.apache.commons.collections4.MapUtils;
@@ -174,9 +177,20 @@ private void initStorageProperties() {
174177
synchronized (this) {
175178
if (storagePropertiesMap == null) {
176179
try {
177-
this.orderedStoragePropertiesList = StorageProperties.createAll(getProperties());
178-
this.storagePropertiesMap = orderedStoragePropertiesList.stream()
179-
.collect(Collectors.toMap(StorageProperties::getType, Function.identity()));
180+
boolean checkStorageProperties = true;
181+
AbstractVendedCredentialsProvider provider =
182+
VendedCredentialsFactory.getProviderType(getMetastoreProperties());
183+
if (provider != null) {
184+
checkStorageProperties = !provider.isVendedCredentialsEnabled(getMetastoreProperties());
185+
}
186+
if (checkStorageProperties) {
187+
this.orderedStoragePropertiesList = StorageProperties.createAll(getProperties());
188+
this.storagePropertiesMap = orderedStoragePropertiesList.stream()
189+
.collect(Collectors.toMap(StorageProperties::getType, Function.identity()));
190+
} else {
191+
this.orderedStoragePropertiesList = Lists.newArrayList();
192+
this.storagePropertiesMap = Maps.newHashMap();
193+
}
180194
} catch (UserException e) {
181195
LOG.warn("Failed to initialize catalog storage properties", e);
182196
throw new RuntimeException("Failed to initialize storage properties, error: "
@@ -199,16 +213,13 @@ public List<StorageProperties> getOrderedStoragePropertiesList() {
199213

200214
public void checkMetaStoreAndStorageProperties(Class msClass) {
201215
MetastoreProperties msProperties;
202-
List<StorageProperties> storageProperties;
203216
try {
204217
msProperties = MetastoreProperties.create(getProperties());
205-
storageProperties = StorageProperties.createAll(getProperties());
218+
initStorageProperties();
206219
} catch (UserException e) {
207220
throw new RuntimeException("Failed to initialize Catalog properties, error: "
208221
+ ExceptionUtils.getRootCauseMessage(e), e);
209222
}
210-
Preconditions.checkNotNull(storageProperties,
211-
"Storage properties are not configured properly");
212223
Preconditions.checkNotNull(msProperties, "Metastore properties are not configured properly");
213224
Preconditions.checkArgument(
214225
msClass.isInstance(msProperties),

‎fe/fe-core/src/main/java/org/apache/doris/datasource/credentials/AbstractVendedCredentialsProvider.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -85,7 +85,7 @@ public final <T> Map<StorageProperties.Type, StorageProperties> getStorageProper
8585
/**
8686
* Check whether to enable vendor credentials (subclass implementation)
8787
*/
88-
protected abstract boolean isVendedCredentialsEnabled(MetastoreProperties metastoreProperties);
88+
public abstract boolean isVendedCredentialsEnabled(MetastoreProperties metastoreProperties);
8989

9090
/**
9191
* Extract original vendored credentials from table objects (subclass implementation)

‎fe/fe-core/src/main/java/org/apache/doris/datasource/credentials/VendedCredentialsFactory.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -53,7 +53,7 @@ public static <T> Map<Type, StorageProperties> getStoragePropertiesMapWithVended
5353
/**
5454
* Select the right provider according to the MetastoreProperties type
5555
*/
56-
private static AbstractVendedCredentialsProvider getProviderType(MetastoreProperties metastoreProperties) {
56+
public static AbstractVendedCredentialsProvider getProviderType(MetastoreProperties metastoreProperties) {
5757
if (metastoreProperties == null) {
5858
return null;
5959
}

‎fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalCatalog.java‎

Lines changed: 0 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -154,11 +154,6 @@ protected void initLocalObjectsImpl() {
154154
metadataOps = hiveOps;
155155
}
156156

157-
@Override
158-
public synchronized void resetToUninitialized(boolean invalidCache) {
159-
super.resetToUninitialized(invalidCache);
160-
}
161-
162157
@Override
163158
public void onClose() {
164159
super.onClose();

‎fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergExternalCatalog.java‎

Lines changed: 1 addition & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -75,9 +75,7 @@ public IcebergExternalCatalog(long catalogId, String name, String comment) {
7575
protected void initCatalog() {
7676
try {
7777
msProperties = (AbstractIcebergProperties) catalogProperty.getMetastoreProperties();
78-
this.catalog = msProperties.initializeCatalog(getName(), catalogProperty
79-
.getOrderedStoragePropertiesList());
80-
78+
this.catalog = msProperties.initializeCatalog(getName(), catalogProperty.getOrderedStoragePropertiesList());
8179
this.icebergCatalogType = msProperties.getIcebergCatalogType();
8280
} catch (ClassCastException e) {
8381
throw new RuntimeException("Invalid properties for Iceberg catalog: " + getProperties(), e);

‎fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergVendedCredentialsProvider.java‎

Lines changed: 13 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,9 @@
2323

2424
import com.google.common.collect.Maps;
2525
import org.apache.iceberg.Table;
26+
import org.apache.iceberg.io.FileIO;
27+
import org.apache.iceberg.io.StorageCredential;
28+
import org.apache.iceberg.io.SupportsStorageCredentials;
2629

2730
import java.util.Map;
2831

@@ -38,7 +41,7 @@ public static IcebergVendedCredentialsProvider getInstance() {
3841
}
3942

4043
@Override
41-
protected boolean isVendedCredentialsEnabled(MetastoreProperties metastoreProperties) {
44+
public boolean isVendedCredentialsEnabled(MetastoreProperties metastoreProperties) {
4245
if (metastoreProperties instanceof IcebergRestProperties) {
4346
return ((IcebergRestProperties) metastoreProperties).isIcebergRestVendedCredentialsEnabled();
4447
}
@@ -57,7 +60,15 @@ protected <T> Map<String, String> extractRawVendedCredentials(T tableObject) {
5760
}
5861

5962
// Return table.io().properties() directly, and let StorageProperties.createAll() to convert the format
60-
return table.io().properties();
63+
FileIO fileIO = table.io();
64+
Map<String, String> ioProps = Maps.newHashMap(fileIO.properties());
65+
if (fileIO instanceof SupportsStorageCredentials) {
66+
SupportsStorageCredentials ssc = (SupportsStorageCredentials) fileIO;
67+
for (StorageCredential storageCredential : ssc.credentials()) {
68+
ioProps.putAll(storageCredential.config());
69+
}
70+
}
71+
return ioProps;
6172
}
6273

6374
@Override

‎fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonVendedCredentialsProvider.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -40,7 +40,7 @@ public static PaimonVendedCredentialsProvider getInstance() {
4040
}
4141

4242
@Override
43-
protected boolean isVendedCredentialsEnabled(MetastoreProperties metastoreProperties) {
43+
public boolean isVendedCredentialsEnabled(MetastoreProperties metastoreProperties) {
4444
// Paimon REST catalog always supports vended credentials if it's REST type
4545
return metastoreProperties instanceof PaimonRestMetaStoreProperties;
4646
}

‎fe/fe-core/src/main/java/org/apache/doris/datasource/property/ConnectorProperty.java‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -29,4 +29,6 @@
2929
boolean supported() default true;
3030

3131
boolean sensitive() default false;
32+
33+
boolean isRegionField() default false;
3234
}

‎fe/fe-core/src/main/java/org/apache/doris/datasource/property/metastore/AbstractIcebergProperties.java‎

Lines changed: 98 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -21,11 +21,19 @@
2121
import org.apache.doris.datasource.iceberg.IcebergExternalCatalog;
2222
import org.apache.doris.datasource.metacache.CacheSpec;
2323
import org.apache.doris.datasource.property.ConnectorProperty;
24+
import org.apache.doris.datasource.property.storage.AbstractS3CompatibleProperties;
25+
import org.apache.doris.datasource.property.storage.S3Properties;
2426
import org.apache.doris.datasource.property.storage.StorageProperties;
2527

28+
import com.google.common.base.Preconditions;
29+
import com.google.common.base.Strings;
2630
import lombok.Getter;
2731
import org.apache.commons.lang3.StringUtils;
32+
import org.apache.hadoop.conf.Configuration;
2833
import org.apache.iceberg.CatalogProperties;
34+
import org.apache.iceberg.CatalogUtil;
35+
import org.apache.iceberg.aws.AwsClientProperties;
36+
import org.apache.iceberg.aws.s3.S3FileIOProperties;
2937
import org.apache.iceberg.catalog.Catalog;
3038

3139
import java.util.HashMap;
@@ -82,6 +90,14 @@ public abstract class AbstractIcebergProperties extends MetastoreProperties {
8290
)
8391
protected String ioManifestCacheMaxContentLength;
8492

93+
@Getter
94+
@ConnectorProperty(
95+
names = {CatalogProperties.FILE_IO_IMPL},
96+
required = false,
97+
description = "Custom io impl for iceberg"
98+
)
99+
protected String ioImpl;
100+
85101
@Getter
86102
protected ExecutionAuthenticator executionAuthenticator = new ExecutionAuthenticator(){};
87103

@@ -114,7 +130,7 @@ protected AbstractIcebergProperties(Map<String, String> props) {
114130
* and deleting Iceberg tables.
115131
*/
116132
public final Catalog initializeCatalog(String catalogName,
117-
List<StorageProperties> storagePropertiesList) {
133+
List<StorageProperties> storagePropertiesList) {
118134
Map<String, String> catalogProps = new HashMap<>(getOrigProps());
119135
if (StringUtils.isNotBlank(warehouse)) {
120136
catalogProps.put(CatalogProperties.WAREHOUSE_LOCATION, warehouse);
@@ -179,4 +195,85 @@ protected abstract Catalog initCatalog(
179195
Map<String, String> catalogProps,
180196
List<StorageProperties> storagePropertiesList
181197
);
198+
199+
/**
200+
* Unified method to configure FileIO properties for Iceberg catalog.
201+
* This method handles all storage types (HDFS, S3, MinIO, etc.) by:
202+
* 1. Adding all storage properties to Hadoop Configuration (for HadoopFileIO / S3A access).
203+
* 2. Extracting S3-compatible properties into fileIOProperties map (for Iceberg S3FileIO).
204+
*
205+
* @param storagePropertiesList list of storage properties
206+
* @param fileIOProperties options map to be populated with S3 FileIO properties
207+
* @return Hadoop Configuration populated with all storage properties
208+
*/
209+
public void toFileIOProperties(List<StorageProperties> storagePropertiesList,
210+
Map<String, String> fileIOProperties, Configuration conf) {
211+
// We only support one S3-compatible storage property for FileIO configuration.
212+
// When multiple AbstractS3CompatibleProperties exist, prefer the first non-S3Properties one,
213+
// because a non-S3 type (e.g. OSSProperties, COSProperties) indicates the user has explicitly
214+
// specified a concrete S3-compatible storage, which should take priority over the generic S3Properties.
215+
AbstractS3CompatibleProperties s3Fallback = null;
216+
AbstractS3CompatibleProperties s3Target = null;
217+
for (StorageProperties storageProperties : storagePropertiesList) {
218+
if (conf != null && storageProperties.getHadoopStorageConfig() != null) {
219+
conf.addResource(storageProperties.getHadoopStorageConfig());
220+
}
221+
if (storageProperties instanceof AbstractS3CompatibleProperties) {
222+
if (s3Fallback == null) {
223+
s3Fallback = (AbstractS3CompatibleProperties) storageProperties;
224+
}
225+
if (s3Target == null && !(storageProperties instanceof S3Properties)) {
226+
s3Target = (AbstractS3CompatibleProperties) storageProperties;
227+
}
228+
}
229+
}
230+
AbstractS3CompatibleProperties chosen = s3Target != null ? s3Target : s3Fallback;
231+
if (chosen != null) {
232+
toS3FileIOProperties(chosen, fileIOProperties);
233+
} else {
234+
String region = AbstractS3CompatibleProperties.getRegionFromProperties(fileIOProperties);
235+
if (!Strings.isNullOrEmpty(region)) {
236+
fileIOProperties.put(AwsClientProperties.CLIENT_REGION, region);
237+
}
238+
}
239+
}
240+
241+
/**
242+
* Configure S3 FileIO properties for all S3-compatible storage types (S3, MinIO, etc.)
243+
* This method provides a unified way to convert S3-compatible properties to Iceberg S3FileIO format.
244+
*
245+
* @param s3Properties S3-compatible properties
246+
* @param options Options map to be populated with S3 FileIO properties
247+
*/
248+
private void toS3FileIOProperties(AbstractS3CompatibleProperties s3Properties, Map<String, String> options) {
249+
// Common properties - only set if not blank
250+
if (StringUtils.isNotBlank(s3Properties.getEndpoint())) {
251+
options.put(S3FileIOProperties.ENDPOINT, s3Properties.getEndpoint());
252+
}
253+
if (StringUtils.isNotBlank(s3Properties.getUsePathStyle())) {
254+
options.put(S3FileIOProperties.PATH_STYLE_ACCESS, s3Properties.getUsePathStyle());
255+
}
256+
if (StringUtils.isNotBlank(s3Properties.getRegion())) {
257+
options.put(AwsClientProperties.CLIENT_REGION, s3Properties.getRegion());
258+
}
259+
if (StringUtils.isNotBlank(s3Properties.getAccessKey())) {
260+
options.put(S3FileIOProperties.ACCESS_KEY_ID, s3Properties.getAccessKey());
261+
}
262+
if (StringUtils.isNotBlank(s3Properties.getSecretKey())) {
263+
options.put(S3FileIOProperties.SECRET_ACCESS_KEY, s3Properties.getSecretKey());
264+
}
265+
if (StringUtils.isNotBlank(s3Properties.getSessionToken())) {
266+
options.put(S3FileIOProperties.SESSION_TOKEN, s3Properties.getSessionToken());
267+
}
268+
}
269+
270+
protected Catalog buildIcebergCatalog(String catalogName, Map<String, String> options, Configuration conf) {
271+
// For Iceberg SDK, "type" means catalog type, such as hive, jdbc, rest.
272+
// But in Doris, "type" is "iceberg".
273+
// And Iceberg SDK does not allow with both "type" and "catalog-impl" properties,
274+
// So here we remove "type" and make sure "catalog-impl" is set.
275+
options.remove(CatalogUtil.ICEBERG_CATALOG_TYPE);
276+
Preconditions.checkArgument(options.containsKey(CatalogProperties.CATALOG_IMPL));
277+
return CatalogUtil.buildIcebergCatalog(catalogName, options, conf);
278+
}
182279
}

0 commit comments

Comments
 (0)