Skip to content

Commit f0dd4f0

Browse files
Tianqi-DotesTianqi-Dotes
authored
[Feature][task-plugins-12820] azure azure-sql datasource plugins (#13209)
* submit azure sql code * change mssql-jdbc version * merge changes to delete other * merge changes to delete other * add dependencies
1 parent 7643d28 commit f0dd4f0

38 files changed

Lines changed: 3978 additions & 16 deletions

File tree

‎dolphinscheduler-bom/pom.xml‎

Lines changed: 7 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -72,7 +72,7 @@
7272
<oshi-core.version>6.1.1</oshi-core.version>
7373
<clickhouse-jdbc.version>0.3.2-patch11</clickhouse-jdbc.version>
7474
<lz4-java.version>1.4.0</lz4-java.version>
75-
<mssql-jdbc.version>6.1.0.jre8</mssql-jdbc.version>
75+
<mssql-jdbc.version>11.2.1.jre8</mssql-jdbc.version>
7676
<presto-jdbc.version>0.238.1</presto-jdbc.version>
7777
<servlet-api.version>2.5</servlet-api.version>
7878
<guava-retry.version>2.0.0</guava-retry.version>
@@ -101,6 +101,7 @@
101101
<aspectj.version>1.9.7</aspectj.version>
102102
<zt-zip.version>1.15</zt-zip.version>
103103
<trino-jdbc.version>402</trino-jdbc.version>
104+
<azure-identity.version>1.7.1</azure-identity.version>
104105
</properties>
105106

106107
<dependencyManagement>
@@ -785,7 +786,11 @@
785786
<artifactId>zt-zip</artifactId>
786787
<version>${zt-zip.version}</version>
787788
</dependency>
788-
789+
<dependency>
790+
<groupId>com.azure</groupId>
791+
<artifactId>azure-identity</artifactId>
792+
<version>${azure-identity.version}</version>
793+
</dependency>
789794
</dependencies>
790795
</dependencyManagement>
791796
</project>

‎dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/constants/DataSourceConstants.java‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -95,4 +95,11 @@ public class DataSourceConstants {
9595
public static final String SPRING_DATASOURCE_MAX_ACTIVE = "spring.datasource.maxActive";
9696

9797
public static final String SPRING_DATASOURCE_TEST_ON_BORROW = "spring.datasource.testOnBorrow";
98+
99+
/**
100+
* azure static websites
101+
*/
102+
public static final String AZURE_SQL_DATABASE_SPN = "https://database.windows.net/";
103+
public static final String AZURE_SQL_DATABASE_TOKEN_SCOPE = "/.default";
104+
98105
}

‎dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-all/pom.xml‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -92,5 +92,10 @@
9292
<artifactId>dolphinscheduler-datasource-starrocks</artifactId>
9393
<version>${project.version}</version>
9494
</dependency>
95+
<dependency>
96+
<groupId>org.apache.dolphinscheduler</groupId>
97+
<artifactId>dolphinscheduler-datasource-azure-sql</artifactId>
98+
<version>${project.version}</version>
99+
</dependency>
95100
</dependencies>
96101
</project>

‎dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-azure-sql/pom.xml‎

Lines changed: 59 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,59 @@
1+
<?xml version="1.0" encoding="UTF-8"?>
2+
<!--
3+
~ Licensed to the Apache Software Foundation (ASF) under one or more
4+
~ contributor license agreements. See the NOTICE file distributed with
5+
~ this work for additional information regarding copyright ownership.
6+
~ The ASF licenses this file to You under the Apache License, Version 2.0
7+
~ (the "License"); you may not use this file except in compliance with
8+
~ the License. You may obtain a copy of the License at
9+
~
10+
~ http://www.apache.org/licenses/LICENSE-2.0
11+
~
12+
~ Unless required by applicable law or agreed to in writing, software
13+
~ distributed under the License is distributed on an "AS IS" BASIS,
14+
~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15+
~ See the License for the specific language governing permissions and
16+
~ limitations under the License.
17+
-->
18+
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
19+
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
20+
<modelVersion>4.0.0</modelVersion>
21+
<parent>
22+
<groupId>org.apache.dolphinscheduler</groupId>
23+
<artifactId>dolphinscheduler-datasource-plugin</artifactId>
24+
<version>dev-SNAPSHOT</version>
25+
</parent>
26+
27+
<artifactId>dolphinscheduler-datasource-azure-sql</artifactId>
28+
<packaging>jar</packaging>
29+
<name>${project.artifactId}</name>
30+
31+
<dependencies>
32+
<dependency>
33+
<groupId>org.apache.dolphinscheduler</groupId>
34+
<artifactId>dolphinscheduler-spi</artifactId>
35+
<scope>provided</scope>
36+
</dependency>
37+
<dependency>
38+
<groupId>org.apache.dolphinscheduler</groupId>
39+
<artifactId>dolphinscheduler-datasource-api</artifactId>
40+
<version>${project.version}</version>
41+
</dependency>
42+
43+
<dependency>
44+
<groupId>com.microsoft.sqlserver</groupId>
45+
<artifactId>mssql-jdbc</artifactId>
46+
<exclusions>
47+
<exclusion>
48+
<groupId>com.microsoft.azure</groupId>
49+
<artifactId>azure-keyvault</artifactId>
50+
</exclusion>
51+
</exclusions>
52+
</dependency>
53+
54+
<dependency>
55+
<groupId>com.azure</groupId>
56+
<artifactId>azure-identity</artifactId>
57+
</dependency>
58+
</dependencies>
59+
</project>

‎dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-azure-sql/src/main/java/org/apache/dolphinscheduler/plugin/datasource/azuresql/AzureSQLDataSourceChannel.java‎

Lines changed: 31 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,31 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one or more
3+
* contributor license agreements. See the NOTICE file distributed with
4+
* this work for additional information regarding copyright ownership.
5+
* The ASF licenses this file to You under the Apache License, Version 2.0
6+
* (the "License"); you may not use this file except in compliance with
7+
* the License. You may obtain a copy of the License at
8+
*
9+
* http://www.apache.org/licenses/LICENSE-2.0
10+
*
11+
* Unless required by applicable law or agreed to in writing, software
12+
* distributed under the License is distributed on an "AS IS" BASIS,
13+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
* See the License for the specific language governing permissions and
15+
* limitations under the License.
16+
*/
17+
18+
package org.apache.dolphinscheduler.plugin.datasource.azuresql;
19+
20+
import org.apache.dolphinscheduler.spi.datasource.BaseConnectionParam;
21+
import org.apache.dolphinscheduler.spi.datasource.DataSourceChannel;
22+
import org.apache.dolphinscheduler.spi.datasource.DataSourceClient;
23+
import org.apache.dolphinscheduler.spi.enums.DbType;
24+
25+
public class AzureSQLDataSourceChannel implements DataSourceChannel {
26+
27+
@Override
28+
public DataSourceClient createDataSourceClient(BaseConnectionParam baseConnectionParam, DbType dbType) {
29+
return new AzureSQLDataSourceClient(baseConnectionParam, dbType);
30+
}
31+
}

‎dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-azure-sql/src/main/java/org/apache/dolphinscheduler/plugin/datasource/azuresql/AzureSQLDataSourceChannelFactory.java‎

Lines changed: 37 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,37 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one or more
3+
* contributor license agreements. See the NOTICE file distributed with
4+
* this work for additional information regarding copyright ownership.
5+
* The ASF licenses this file to You under the Apache License, Version 2.0
6+
* (the "License"); you may not use this file except in compliance with
7+
* the License. You may obtain a copy of the License at
8+
*
9+
* http://www.apache.org/licenses/LICENSE-2.0
10+
*
11+
* Unless required by applicable law or agreed to in writing, software
12+
* distributed under the License is distributed on an "AS IS" BASIS,
13+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
* See the License for the specific language governing permissions and
15+
* limitations under the License.
16+
*/
17+
18+
package org.apache.dolphinscheduler.plugin.datasource.azuresql;
19+
20+
import org.apache.dolphinscheduler.spi.datasource.DataSourceChannel;
21+
import org.apache.dolphinscheduler.spi.datasource.DataSourceChannelFactory;
22+
23+
import com.google.auto.service.AutoService;
24+
25+
@AutoService(DataSourceChannelFactory.class)
26+
public class AzureSQLDataSourceChannelFactory implements DataSourceChannelFactory {
27+
28+
@Override
29+
public String getName() {
30+
return "azuresql";
31+
}
32+
33+
@Override
34+
public DataSourceChannel create() {
35+
return new AzureSQLDataSourceChannel();
36+
}
37+
}

‎dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-azure-sql/src/main/java/org/apache/dolphinscheduler/plugin/datasource/azuresql/AzureSQLDataSourceClient.java‎

Lines changed: 83 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,83 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one or more
3+
* contributor license agreements. See the NOTICE file distributed with
4+
* this work for additional information regarding copyright ownership.
5+
* The ASF licenses this file to You under the Apache License, Version 2.0
6+
* (the "License"); you may not use this file except in compliance with
7+
* the License. You may obtain a copy of the License at
8+
*
9+
* http://www.apache.org/licenses/LICENSE-2.0
10+
*
11+
* Unless required by applicable law or agreed to in writing, software
12+
* distributed under the License is distributed on an "AS IS" BASIS,
13+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
* See the License for the specific language governing permissions and
15+
* limitations under the License.
16+
*/
17+
18+
package org.apache.dolphinscheduler.plugin.datasource.azuresql;
19+
20+
import org.apache.dolphinscheduler.plugin.datasource.api.client.CommonDataSourceClient;
21+
import org.apache.dolphinscheduler.plugin.datasource.azuresql.param.AzureSQLAuthMode;
22+
import org.apache.dolphinscheduler.plugin.datasource.azuresql.param.AzureSQLConnectionParam;
23+
import org.apache.dolphinscheduler.plugin.datasource.azuresql.param.AzureSQLDataSourceProcessor;
24+
import org.apache.dolphinscheduler.spi.datasource.BaseConnectionParam;
25+
import org.apache.dolphinscheduler.spi.enums.DbType;
26+
27+
import java.sql.Connection;
28+
import java.sql.SQLException;
29+
import java.sql.Statement;
30+
import java.util.concurrent.TimeUnit;
31+
32+
import org.slf4j.Logger;
33+
import org.slf4j.LoggerFactory;
34+
35+
import com.google.common.base.Stopwatch;
36+
37+
public class AzureSQLDataSourceClient extends CommonDataSourceClient {
38+
39+
private static final Logger logger = LoggerFactory.getLogger(AzureSQLDataSourceClient.class);
40+
41+
public AzureSQLDataSourceClient(BaseConnectionParam baseConnectionParam, DbType dbType) {
42+
super(baseConnectionParam, dbType);
43+
}
44+
45+
@Override
46+
public Connection getConnection() {
47+
AzureSQLConnectionParam connectionParam = (AzureSQLConnectionParam) this.baseConnectionParam;
48+
if (!connectionParam.getMode().equals(AzureSQLAuthMode.ACCESSTOKEN)) {
49+
return super.getConnection();
50+
}
51+
return AzureSQLDataSourceProcessor.tokenGetConnection(connectionParam);
52+
}
53+
54+
@Override
55+
public void checkClient() {
56+
57+
AzureSQLConnectionParam connectionParam = (AzureSQLConnectionParam) this.baseConnectionParam;
58+
Stopwatch stopwatch = Stopwatch.createStarted();
59+
String validationQuery = this.baseConnectionParam.getValidationQuery();
60+
if (!connectionParam.getMode().equals(AzureSQLAuthMode.ACCESSTOKEN)) {
61+
// Checking data source client
62+
try {
63+
this.jdbcTemplate.execute(validationQuery);
64+
} catch (Exception e) {
65+
throw new RuntimeException("JDBC connect failed", e);
66+
} finally {
67+
logger.info("Time to execute check jdbc client with sql {} for {} ms ",
68+
this.baseConnectionParam.getValidationQuery(), stopwatch.elapsed(TimeUnit.MILLISECONDS));
69+
}
70+
} else {
71+
try (Statement statement = getConnection().createStatement()) {
72+
if (!statement.execute(validationQuery)) {
73+
throw new SQLException("execute check azure sql token client failed : " + validationQuery);
74+
}
75+
} catch (SQLException e) {
76+
throw new RuntimeException(e);
77+
} finally {
78+
logger.info("Time to execute check azure sql token client with sql {} for {} ms ",
79+
this.baseConnectionParam.getValidationQuery(), stopwatch.elapsed(TimeUnit.MILLISECONDS));
80+
}
81+
}
82+
}
83+
}

‎dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-azure-sql/src/main/java/org/apache/dolphinscheduler/plugin/datasource/azuresql/param/AzureSQLAuthMode.java‎

Lines changed: 68 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,68 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one or more
3+
* contributor license agreements. See the NOTICE file distributed with
4+
* this work for additional information regarding copyright ownership.
5+
* The ASF licenses this file to You under the Apache License, Version 2.0
6+
* (the "License"); you may not use this file except in compliance with
7+
* the License. You may obtain a copy of the License at
8+
*
9+
* http://www.apache.org/licenses/LICENSE-2.0
10+
*
11+
* Unless required by applicable law or agreed to in writing, software
12+
* distributed under the License is distributed on an "AS IS" BASIS,
13+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
* See the License for the specific language governing permissions and
15+
* limitations under the License.
16+
*/
17+
18+
package org.apache.dolphinscheduler.plugin.datasource.azuresql.param;
19+
20+
import static java.util.stream.Collectors.toMap;
21+
22+
import java.util.Arrays;
23+
import java.util.Map;
24+
import java.util.NoSuchElementException;
25+
26+
import com.fasterxml.jackson.annotation.JsonValue;
27+
import com.google.common.base.Functions;
28+
29+
public enum AzureSQLAuthMode {
30+
31+
SQL_PASSWORD(0, "SqlPassword"),
32+
AD_PASSWORD(1, "ActiveDirectoryPassword"),
33+
AD_MSI(2, "ActiveDirectoryMSI"),
34+
AD_SERVICE_PRINCIPAL(3, "ActiveDirectoryServicePrincipal"),
35+
ACCESSTOKEN(4, "accessToken"),
36+
;
37+
38+
private static final Map<Integer, AzureSQLAuthMode> AUTH_TYPE_MAP =
39+
Arrays.stream(AzureSQLAuthMode.values()).collect(toMap(AzureSQLAuthMode::getCode, Functions.identity()));
40+
private final int code;
41+
@JsonValue
42+
private final String descp;
43+
44+
AzureSQLAuthMode(int code, String descp) {
45+
this.code = code;
46+
this.descp = descp;
47+
}
48+
49+
public static AzureSQLAuthMode of(int type) {
50+
if (AUTH_TYPE_MAP.containsKey(type)) {
51+
return AUTH_TYPE_MAP.get(type);
52+
}
53+
return null;
54+
}
55+
56+
public static AzureSQLAuthMode ofName(String name) {
57+
return Arrays.stream(AzureSQLAuthMode.values()).filter(e -> e.name().equals(name)).findFirst()
58+
.orElseThrow(() -> new NoSuchElementException("no such auth type"));
59+
}
60+
61+
public int getCode() {
62+
return code;
63+
}
64+
65+
public String getDescp() {
66+
return descp;
67+
}
68+
}

‎dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-azure-sql/src/main/java/org/apache/dolphinscheduler/plugin/datasource/azuresql/param/AzureSQLConnectionParam.java‎

Lines changed: 37 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,37 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one or more
3+
* contributor license agreements. See the NOTICE file distributed with
4+
* this work for additional information regarding copyright ownership.
5+
* The ASF licenses this file to You under the Apache License, Version 2.0
6+
* (the "License"); you may not use this file except in compliance with
7+
* the License. You may obtain a copy of the License at
8+
*
9+
* http://www.apache.org/licenses/LICENSE-2.0
10+
*
11+
* Unless required by applicable law or agreed to in writing, software
12+
* distributed under the License is distributed on an "AS IS" BASIS,
13+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
* See the License for the specific language governing permissions and
15+
* limitations under the License.
16+
*/
17+
18+
package org.apache.dolphinscheduler.plugin.datasource.azuresql.param;
19+
20+
import org.apache.dolphinscheduler.spi.datasource.BaseConnectionParam;
21+
22+
import lombok.Getter;
23+
import lombok.Setter;
24+
import lombok.ToString;
25+
26+
@ToString
27+
@Getter
28+
@Setter
29+
public class AzureSQLConnectionParam extends BaseConnectionParam {
30+
31+
protected AzureSQLAuthMode mode;
32+
// AzureSQLAuthMode.AD_MSI
33+
protected String MSIClientId;
34+
// AzureSQLAuthMode.ACCESSTOKEN
35+
protected String endpoint;
36+
37+
}

0 commit comments

Comments
 (0)