Skip to content

Commit 0822e16

Browse files
hansvahansva
authored
Add downloader for JDBC drivers, fixes #7352 (#7353)
1 parent bf00f70 commit 0822e16

107 files changed

Lines changed: 2804 additions & 165 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎core/src/main/java/org/apache/hop/core/database/DriverDownload.java‎

Lines changed: 97 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,97 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one or more
3+
* contributor license agreements. See the NOTICE file distributed with
4+
* this work for additional information regarding copyright ownership.
5+
* The ASF licenses this file to You under the Apache License, Version 2.0
6+
* (the "License"); you may not use this file except in compliance with
7+
* the License. You may obtain a copy of the License at
8+
*
9+
* http://www.apache.org/licenses/LICENSE-2.0
10+
*
11+
* Unless required by applicable law or agreed to in writing, software
12+
* distributed under the License is distributed on an "AS IS" BASIS,
13+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
* See the License for the specific language governing permissions and
15+
* limitations under the License.
16+
*/
17+
18+
package org.apache.hop.core.database;
19+
20+
import java.util.List;
21+
import lombok.Builder;
22+
import lombok.Getter;
23+
24+
/**
25+
* Describes how a database's JDBC driver can be downloaded on demand. A database plugin declares
26+
* this from {@link IDatabase#getDriverDownload()} so the driver lives in one place - next to the
27+
* rest of the database metadata - and external plugins can point Hop at their own drivers.
28+
*
29+
* <p>This is metadata only; it carries no driver bytes, so it is safe to ship even for restricted
30+
* (Category X) drivers that Apache Hop may not bundle. Hop downloads the actual jar from Maven
31+
* Central at the user's explicit request, after the user accepts the vendor license.
32+
*/
33+
@Getter
34+
@Builder
35+
public class DriverDownload {
36+
37+
/** Maven coordinate without version: {@code groupId:artifactId}. */
38+
private final String mavenCoordinate;
39+
40+
/** Version resolved when the caller does not request a specific one. */
41+
private final String defaultVersion;
42+
43+
/**
44+
* ASF license category: {@code A}/{@code B} may be bundled, {@code X} must never be bundled and
45+
* is only downloaded at the user's explicit request after license acceptance.
46+
*/
47+
private final String licenseCategory;
48+
49+
private final String licenseName;
50+
private final String licenseUrl;
51+
private final String vendor;
52+
private final String vendorUrl;
53+
private final String notes;
54+
55+
/**
56+
* Maven coordinates ({@code groupId:artifactId}, artifactId may be {@code *}) to exclude from the
57+
* download. Use this to drop transitive jars that Hop already ships in lib/core (so the driver
58+
* uses the shared one via its parent classloader), e.g. {@code
59+
* com.google.protobuf:protobuf-java}. Optional and test dependencies are skipped automatically
60+
* and do not need to be listed here.
61+
*/
62+
@Builder.Default private final List<String> excludes = List.of();
63+
64+
/**
65+
* Optional extra Maven repository base URL to resolve this driver from, for drivers that are not
66+
* on Maven Central (e.g. MonetDB on Clojars). It is searched in addition to Maven Central / the
67+
* user-supplied repository. Null means Maven Central only.
68+
*/
69+
private final String repositoryUrl;
70+
71+
/**
72+
* @return true when this is a restricted (Category X) driver that must not be bundled.
73+
*/
74+
public boolean isRestricted() {
75+
return "X".equalsIgnoreCase(licenseCategory);
76+
}
77+
78+
/**
79+
* @return the full Maven coordinate {@code groupId:artifactId:version}, using {@code version}
80+
* when supplied, otherwise {@link #defaultVersion}.
81+
*/
82+
public String toCoordinate(String version) {
83+
String v = (version == null || version.isBlank()) ? defaultVersion : version;
84+
return mavenCoordinate + ":" + v;
85+
}
86+
87+
/**
88+
* @return the artifact id portion of {@link #mavenCoordinate}, used for install detection.
89+
*/
90+
public String getArtifactId() {
91+
if (mavenCoordinate == null) {
92+
return null;
93+
}
94+
String[] parts = mavenCoordinate.split(":");
95+
return parts.length >= 2 ? parts[1] : null;
96+
}
97+
}

‎core/src/main/java/org/apache/hop/core/database/IDatabase.java‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -263,6 +263,18 @@ String getFieldDefinition(
263263
*/
264264
String getDriverClass();
265265

266+
/**
267+
* Optional descriptor that lets Hop download this database's JDBC driver on demand. The default
268+
* is {@code null}, meaning there is no downloadable driver. Database plugins - including
269+
* external, third-party ones - override this to point Hop at their driver's Maven coordinate and
270+
* license, so the driver definition lives next to the rest of the database metadata.
271+
*
272+
* @return the driver download descriptor, or {@code null} when none is available
273+
*/
274+
default DriverDownload getDriverDownload() {
275+
return null;
276+
}
277+
266278
/**
267279
* @param hostname the hostname
268280
* @param port the port as a string

‎core/src/main/java/org/apache/hop/core/plugins/HopURLClassLoader.java‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -46,6 +46,16 @@ public <T> T computeIfAbsent(final Class<T> key, Supplier<T> provider) {
4646
return key.cast(cache.computeIfAbsent(key, k -> provider));
4747
}
4848

49+
/**
50+
* Public entry point to add a jar to this classloader at runtime, e.g. a JDBC driver downloaded
51+
* while Hop is running. After this the classes in the jar become loadable without a restart.
52+
*
53+
* @param url the jar URL to add
54+
*/
55+
public void addJar(URL url) {
56+
addURL(url);
57+
}
58+
4959
@Override
5060
protected void addURL(URL url) {
5161
super.addURL(url);

‎core/src/test/java/org/apache/hop/core/database/DriverDownloadTest.java‎

Lines changed: 112 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,112 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one or more
3+
* contributor license agreements. See the NOTICE file distributed with
4+
* this work for additional information regarding copyright ownership.
5+
* The ASF licenses this file to You under the Apache License, Version 2.0
6+
* (the "License"); you may not use this file except in compliance with
7+
* the License. You may obtain a copy of the License at
8+
*
9+
* http://www.apache.org/licenses/LICENSE-2.0
10+
*
11+
* Unless required by applicable law or agreed to in writing, software
12+
* distributed under the License is distributed on an "AS IS" BASIS,
13+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
* See the License for the specific language governing permissions and
15+
* limitations under the License.
16+
*/
17+
18+
package org.apache.hop.core.database;
19+
20+
import static org.junit.jupiter.api.Assertions.assertEquals;
21+
import static org.junit.jupiter.api.Assertions.assertFalse;
22+
import static org.junit.jupiter.api.Assertions.assertNull;
23+
import static org.junit.jupiter.api.Assertions.assertTrue;
24+
25+
import org.junit.jupiter.api.Test;
26+
27+
class DriverDownloadTest {
28+
29+
@Test
30+
void coordinateUsesDefaultVersionAndHonoursOverride() {
31+
DriverDownload download =
32+
DriverDownload.builder()
33+
.mavenCoordinate("com.oracle.database.jdbc:ojdbc11")
34+
.defaultVersion("23.5.0.24.07")
35+
.licenseCategory("X")
36+
.build();
37+
38+
assertEquals("com.oracle.database.jdbc:ojdbc11:23.5.0.24.07", download.toCoordinate(null));
39+
assertEquals("com.oracle.database.jdbc:ojdbc11:21.0.0", download.toCoordinate("21.0.0"));
40+
assertEquals("ojdbc11", download.getArtifactId());
41+
}
42+
43+
@Test
44+
void coordinateKeepsAnExplicitClassifier() {
45+
// Some drivers ship a self-contained classified artifact, e.g. ClickHouse's "all" jar. The
46+
// version is appended after the group:artifact:packaging:classifier, and the artifactId is
47+
// still the second segment.
48+
DriverDownload download =
49+
DriverDownload.builder()
50+
.mavenCoordinate("com.clickhouse:clickhouse-jdbc:jar:all")
51+
.defaultVersion("0.9.8")
52+
.licenseCategory("A")
53+
.build();
54+
55+
assertEquals("com.clickhouse:clickhouse-jdbc:jar:all:0.9.8", download.toCoordinate(null));
56+
assertEquals("clickhouse-jdbc", download.getArtifactId());
57+
}
58+
59+
@Test
60+
void coordinateUsesDefaultVersionWhenOverrideIsBlank() {
61+
DriverDownload download =
62+
DriverDownload.builder()
63+
.mavenCoordinate("org.postgresql:postgresql")
64+
.defaultVersion("42.7.4")
65+
.build();
66+
67+
assertEquals("org.postgresql:postgresql:42.7.4", download.toCoordinate(" "));
68+
}
69+
70+
@Test
71+
void artifactIdIsNullForAnIncompleteCoordinate() {
72+
assertNull(DriverDownload.builder().build().getArtifactId());
73+
assertNull(DriverDownload.builder().mavenCoordinate("single-segment").build().getArtifactId());
74+
}
75+
76+
@Test
77+
void restrictedIsBasedOnCategoryX() {
78+
assertTrue(DriverDownload.builder().licenseCategory("X").build().isRestricted());
79+
assertTrue(DriverDownload.builder().licenseCategory("x").build().isRestricted());
80+
assertFalse(DriverDownload.builder().licenseCategory("A").build().isRestricted());
81+
assertFalse(DriverDownload.builder().build().isRestricted());
82+
}
83+
84+
@Test
85+
void databaseWithoutDownloadReturnsNullByDefault() {
86+
IDatabase database = new NoneDatabaseMeta();
87+
assertNull(database.getDriverDownload());
88+
}
89+
90+
@Test
91+
void excludesDefaultToEmptyAndCarryValues() {
92+
assertTrue(DriverDownload.builder().build().getExcludes().isEmpty());
93+
assertEquals(
94+
java.util.List.of("com.google.protobuf:protobuf-java"),
95+
DriverDownload.builder()
96+
.excludes(java.util.List.of("com.google.protobuf:protobuf-java"))
97+
.build()
98+
.getExcludes());
99+
}
100+
101+
@Test
102+
void repositoryUrlDefaultsToNullAndCarriesValue() {
103+
org.junit.jupiter.api.Assertions.assertNull(
104+
DriverDownload.builder().build().getRepositoryUrl());
105+
assertEquals(
106+
"https://clojars.org/repo/",
107+
DriverDownload.builder()
108+
.repositoryUrl("https://clojars.org/repo/")
109+
.build()
110+
.getRepositoryUrl());
111+
}
112+
}

‎core/src/test/java/org/apache/hop/core/plugins/HopURLClassLoaderTest.java‎

Lines changed: 68 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,68 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one or more
3+
* contributor license agreements. See the NOTICE file distributed with
4+
* this work for additional information regarding copyright ownership.
5+
* The ASF licenses this file to You under the Apache License, Version 2.0
6+
* (the "License"); you may not use this file except in compliance with
7+
* the License. You may obtain a copy of the License at
8+
*
9+
* http://www.apache.org/licenses/LICENSE-2.0
10+
*
11+
* Unless required by applicable law or agreed to in writing, software
12+
* distributed under the License is distributed on an "AS IS" BASIS,
13+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
* See the License for the specific language governing permissions and
15+
* limitations under the License.
16+
*/
17+
18+
package org.apache.hop.core.plugins;
19+
20+
import static org.junit.jupiter.api.Assertions.assertEquals;
21+
import static org.junit.jupiter.api.Assertions.assertNotNull;
22+
import static org.junit.jupiter.api.Assertions.assertNull;
23+
import static org.junit.jupiter.api.Assertions.assertTrue;
24+
25+
import java.net.URL;
26+
import java.nio.charset.StandardCharsets;
27+
import java.nio.file.Files;
28+
import java.nio.file.Path;
29+
import java.util.Arrays;
30+
import java.util.jar.JarEntry;
31+
import java.util.jar.JarOutputStream;
32+
import org.junit.jupiter.api.Test;
33+
import org.junit.jupiter.api.io.TempDir;
34+
35+
/**
36+
* Unit test for {@link HopURLClassLoader#addJar(URL)}, the hook the JDBC driver download uses to
37+
* hot-load a freshly installed driver into a database plugin's classloader without a restart.
38+
*/
39+
class HopURLClassLoaderTest {
40+
41+
@Test
42+
void addJarWiresTheJarIntoTheClasspath(@TempDir Path tempDir) throws Exception {
43+
Path jar = tempDir.resolve("probe.jar");
44+
try (JarOutputStream jos = new JarOutputStream(Files.newOutputStream(jar))) {
45+
jos.putNextEntry(new JarEntry("probe/marker.txt"));
46+
jos.write("hello".getBytes(StandardCharsets.UTF_8));
47+
jos.closeEntry();
48+
}
49+
URL jarUrl = jar.toUri().toURL();
50+
51+
try (HopURLClassLoader loader =
52+
new HopURLClassLoader(new URL[0], getClass().getClassLoader())) {
53+
assertEquals(0, loader.getURLs().length, "the loader starts with no jars of its own");
54+
assertNull(
55+
loader.getResource("probe/marker.txt"),
56+
"the jar's resource must not be reachable before it is added");
57+
58+
loader.addJar(jarUrl);
59+
60+
assertTrue(
61+
Arrays.asList(loader.getURLs()).contains(jarUrl),
62+
"addJar must add the jar to the loader's search path");
63+
assertNotNull(
64+
loader.getResource("probe/marker.txt"),
65+
"the jar's resource must be reachable after addJar");
66+
}
67+
}
68+
}

‎docker/integration-tests/integration-tests-database.yaml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -35,6 +35,7 @@ services:
3535
- SSH_TUNNEL_PORT=22
3636
- SSH_TUNNEL_USER=hop
3737
- SSH_TUNNEL_PASSWORD=hop_ssh_password
38+
- HOP_DRIVERS_DOWNLOAD=mysql
3839
volumes:
3940
- ./resource/mssql/mssql_bulkload.csv:/tmp/mssql_bulkload.csv
4041

‎docker/integration-tests/integration-tests-duckdb.yaml‎

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,27 @@
1+
# Licensed to the Apache Software Foundation (ASF) under one
2+
# or more contributor license agreements. See the NOTICE file
3+
# distributed with this work for additional information
4+
# regarding copyright ownership. The ASF licenses this file
5+
# to you under the Apache License, Version 2.0 (the
6+
# "License"); you may not use this file except in compliance
7+
# with the License. You may obtain a copy of the License at
8+
#
9+
# http://www.apache.org/licenses/LICENSE-2.0
10+
#
11+
# Unless required by applicable law or agreed to in writing,
12+
# software distributed under the License is distributed on an
13+
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+
# KIND, either express or implied. See the License for the
15+
# specific language governing permissions and limitations
16+
# under the License.
17+
18+
# DuckDB is an embedded database, so there is no database service to start. We only need the test
19+
# runner with the DuckDB JDBC driver, which is downloaded on demand by run-tests.sh (it is no longer
20+
# bundled with Hop).
21+
services:
22+
integration_test_duckdb:
23+
extends:
24+
file: integration-tests-base.yaml
25+
service: integration_test
26+
environment:
27+
- HOP_DRIVERS_DOWNLOAD=duckdb

‎docker/integration-tests/integration-tests-vertica.yaml‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -20,8 +20,10 @@ services:
2020
extends:
2121
file: integration-tests-base.yaml
2222
service: integration_test
23+
environment:
24+
- HOP_DRIVERS_DOWNLOAD=vertica
2325
links:
24-
- vertica
26+
- vertica
2527

2628
vertica:
2729
image: vertica/vertica-ce:latest

‎docker/integration-tests/unit-tests.Dockerfile‎

Lines changed: 0 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -86,11 +86,6 @@ RUN chown -R ${JENKINS_USER}:${JENKINS_GROUP} ${DEPLOYMENT_PATH}/hop \
8686
&& cd ${DEPLOYMENT_PATH}/hop \
8787
&& ./hop-conf.sh --generate-fat-jar=/tmp/hop-fatjar.jar
8888

89-
90-
# Download Additional drivers/dependencies
91-
ADD --chown=${JENKINS_USER}:${JENKINS_GROUP} https://repo1.maven.org/maven2/com/vertica/jdbc/vertica-jdbc/23.4.0-0/vertica-jdbc-23.4.0-0.jar /opt/hop/lib/jdbc/vertica-jdbc-23.4.0-0.jar
92-
ADD --chown=${JENKINS_USER}:${JENKINS_GROUP} https://repo1.maven.org/maven2/com/mysql/mysql-connector-j/9.2.0/mysql-connector-j-9.2.0.jar /opt/hop/lib/jdbc/mysql-connector-j-9.2.0.jar
93-
9489
# make volume available so that hop pipeline and workflow files can be provided easily
9590
VOLUME ["/files"]
9691
USER ${JENKINS_USER}

0 commit comments

Comments
 (0)