Skip to content

Commit 1018526

Browse files
aditya-gupta36aditya-gupta36
committed
ATLAS-5190: Fixing IT failures for webapp & addons: hive, falcon, hbase, sqoop, storm bridges module
1 parent 5612e76 commit 1018526

15 files changed

Lines changed: 241 additions & 92 deletions

File tree

‎addons/falcon-bridge/pom.xml‎

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -54,6 +54,10 @@
5454
<groupId>com.sun.jersey</groupId>
5555
<artifactId>*</artifactId>
5656
</exclusion>
57+
<exclusion>
58+
<groupId>org.apache.atlas</groupId>
59+
<artifactId>atlas-client-v2-shaded</artifactId>
60+
</exclusion>
5761
</exclusions>
5862
</dependency>
5963
<!-- Logging -->
@@ -111,6 +115,12 @@
111115
</exclusion>
112116
</exclusions>
113117
</dependency>
118+
119+
<dependency>
120+
<groupId>org.apache.atlas</groupId>
121+
<artifactId>atlas-client-v2</artifactId>
122+
<scope>test</scope>
123+
</dependency>
114124
<dependency>
115125
<groupId>org.apache.atlas</groupId>
116126
<artifactId>atlas-graphdb-impls</artifactId>
@@ -126,6 +136,22 @@
126136

127137
<build>
128138
<plugins>
139+
<!-- Parent disables Failsafe forking (forkCount=0). Exclude shaded v2 client from IT classpath
140+
to avoid Jersey NoSuchMethodError on org.apache.atlas.shaded.javax.ws.rs.RuntimeType. -->
141+
<plugin>
142+
<groupId>org.apache.maven.plugins</groupId>
143+
<artifactId>maven-failsafe-plugin</artifactId>
144+
<configuration combine.self="merge">
145+
<forkCount>1</forkCount>
146+
<reuseForks>false</reuseForks>
147+
<useSystemClassLoader>false</useSystemClassLoader>
148+
<classpathDependencyExcludes>
149+
<classpathDependencyExclude>org.apache.atlas:atlas-client-v2-shaded</classpathDependencyExclude>
150+
<classpathDependencyExclude>org.glassfish.jersey.media:jersey-media-json-jackson</classpathDependencyExclude>
151+
</classpathDependencyExcludes>
152+
</configuration>
153+
</plugin>
154+
129155
<plugin>
130156
<groupId>org.eclipse.jetty</groupId>
131157
<artifactId>jetty-maven-plugin</artifactId>

‎addons/hbase-bridge/pom.xml‎

Lines changed: 41 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -298,13 +298,40 @@
298298
</exclusions>
299299
</dependency>
300300

301+
<!-- Hadoop MiniDFS WebHDFS uses Jersey 1 ServletContainer; Glassfish Jersey 2 cannot replace it.
302+
Jersey 1 is excluded from hadoop-* deps above; add back for IT classpath only. -->
303+
<dependency>
304+
<groupId>com.sun.jersey</groupId>
305+
<artifactId>jersey-core</artifactId>
306+
<version>${jersey.version}</version>
307+
<scope>test</scope>
308+
</dependency>
309+
<dependency>
310+
<groupId>com.sun.jersey</groupId>
311+
<artifactId>jersey-server</artifactId>
312+
<version>${jersey.version}</version>
313+
<scope>test</scope>
314+
</dependency>
315+
<dependency>
316+
<groupId>com.sun.jersey</groupId>
317+
<artifactId>jersey-servlet</artifactId>
318+
<version>${jersey.version}</version>
319+
<scope>test</scope>
320+
</dependency>
321+
301322
<!-- to bring up atlas server for integration tests -->
302323
<dependency>
303324
<groupId>javax.ws.rs</groupId>
304325
<artifactId>javax.ws.rs-api</artifactId>
305326
<version>${javax.ws.rs-api.version}</version>
306327
<scope>test</scope>
307328
</dependency>
329+
<dependency>
330+
<groupId>javax.ws.rs</groupId>
331+
<artifactId>jsr311-api</artifactId>
332+
<version>1.1.1</version>
333+
<scope>test</scope>
334+
</dependency>
308335
<dependency>
309336
<groupId>junit</groupId>
310337
<artifactId>junit</artifactId>
@@ -387,6 +414,20 @@
387414

388415
<build>
389416
<plugins>
417+
<!-- Parent disables Failsafe forking (forkCount=0). Fork IT JVM for stable Atlas client + Hadoop minicluster. -->
418+
<plugin>
419+
<groupId>org.apache.maven.plugins</groupId>
420+
<artifactId>maven-failsafe-plugin</artifactId>
421+
<configuration combine.self="merge">
422+
<forkCount>1</forkCount>
423+
<reuseForks>false</reuseForks>
424+
<useSystemClassLoader>false</useSystemClassLoader>
425+
<classpathDependencyExcludes>
426+
<classpathDependencyExclude>org.glassfish.jersey.media:jersey-media-json-jackson</classpathDependencyExclude>
427+
</classpathDependencyExcludes>
428+
</configuration>
429+
</plugin>
430+
390431
<plugin>
391432
<groupId>org.eclipse.jetty</groupId>
392433
<artifactId>jetty-maven-plugin</artifactId>

‎addons/hbase-testing-util/pom.xml‎

Lines changed: 2 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -36,10 +36,8 @@
3636
<apache.log4j.version>2.17.1</apache.log4j.version>
3737
<checkstyle.failOnViolation>true</checkstyle.failOnViolation>
3838
<checkstyle.skip>false</checkstyle.skip>
39-
<!-- Align minicluster deps with the rest of Atlas (parent hadoop.version = 3.4.2). -->
40-
<hadoop.version>3.4.2</hadoop.version>
41-
<!-- Hadoop 3.4.x MiniDFS WebHDFS uses Jersey 1 ServletContainer; Glassfish Jersey 2 cannot replace it.
42-
Scope test only. Do not override jetty.version: parent Jetty 9.4.x must match Hadoop 3.4. -->
39+
<hadoop.version>3.0.3</hadoop.version>
40+
<jetty.version>9.3.14.v20161028</jetty.version>
4341
</properties>
4442

4543
<dependencies>

‎addons/hbase-testing-util/src/test/java/org/apache/atlas/hbase/TestHBaseTestingUtilSpinup.java‎

Lines changed: 0 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -19,8 +19,6 @@
1919

2020
import org.apache.hadoop.hbase.HBaseTestingUtility;
2121
import org.apache.hadoop.hbase.MiniHBaseCluster;
22-
import org.apache.hadoop.hbase.wal.FSHLogProvider;
23-
import org.apache.hadoop.hbase.wal.WALFactory;
2422
import org.testng.annotations.Test;
2523

2624
import java.io.IOException;
@@ -43,11 +41,6 @@ public TestHBaseTestingUtilSpinup() throws Exception {
4341
hBaseTestingUtility.getConfiguration().set("hbase.regionserver.info.port", String.valueOf(getFreePort()));
4442
hBaseTestingUtility.getConfiguration().set("zookeeper.znode.parent", "/hbase-unsecure");
4543
hBaseTestingUtility.getConfiguration().set("hbase.table.sanity.checks", "false");
46-
// Default AsyncFSWAL hits async HDFS output that is not binary-compatible with Hadoop 3.4.x here
47-
// (HdfsFileStatus class vs interface). FSHLog is sufficient for this util smoke test on JDK 8.
48-
String fsWal = FSHLogProvider.class.getName();
49-
hBaseTestingUtility.getConfiguration().set(WALFactory.WAL_PROVIDER, fsWal);
50-
hBaseTestingUtility.getConfiguration().set(WALFactory.META_WAL_PROVIDER, fsWal);
5144
}
5245

5346
@Test

‎addons/hive-bridge/pom.xml‎

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -247,6 +247,12 @@
247247
<scope>test</scope>
248248
</dependency>
249249

250+
<dependency>
251+
<groupId>org.apache.atlas</groupId>
252+
<artifactId>atlas-client-v2</artifactId>
253+
<scope>test</scope>
254+
</dependency>
255+
250256
<dependency>
251257
<groupId>org.apache.atlas</groupId>
252258
<artifactId>atlas-graphdb-impls</artifactId>
@@ -335,6 +341,22 @@
335341
</dependencies>
336342
<build>
337343
<plugins>
344+
<!-- Parent disables Failsafe forking (forkCount=0). Hive IT runs in Maven JVM + Hive TCCL
345+
breaks JAX-RS ClientBuilder; fork + unshaded v2 client on IT classpath fix LinkageError. -->
346+
<plugin>
347+
<groupId>org.apache.maven.plugins</groupId>
348+
<artifactId>maven-failsafe-plugin</artifactId>
349+
<configuration combine.self="merge">
350+
<forkCount>1</forkCount>
351+
<reuseForks>false</reuseForks>
352+
<useSystemClassLoader>false</useSystemClassLoader>
353+
<classpathDependencyExcludes>
354+
<classpathDependencyExclude>org.apache.atlas:atlas-client-v2-shaded</classpathDependencyExclude>
355+
<classpathDependencyExclude>org.glassfish.jersey.media:jersey-media-json-jackson</classpathDependencyExclude>
356+
</classpathDependencyExcludes>
357+
</configuration>
358+
</plugin>
359+
338360
<plugin>
339361
<groupId>org.eclipse.jetty</groupId>
340362
<artifactId>jetty-maven-plugin</artifactId>

‎addons/hive-bridge/src/test/java/org/apache/atlas/hive/HiveITBase.java‎

Lines changed: 23 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -118,23 +118,33 @@ public void setUp() throws Exception {
118118

119119
SessionState.setCurrentSessionState(ss);
120120

121-
Configuration configuration = ApplicationProperties.get();
121+
// SessionState.start swaps TCCL; align it with the Atlas client class loader for Jersey init.
122+
ClassLoader hiveTccl = Thread.currentThread().getContextClassLoader();
123+
ClassLoader atlasLoader = AtlasClientV2.class.getClassLoader();
122124

123-
String[] atlasEndPoint = configuration.getStringArray(HiveMetaStoreBridge.ATLAS_ENDPOINT);
125+
Thread.currentThread().setContextClassLoader(atlasLoader);
124126

125-
if (atlasEndPoint == null || atlasEndPoint.length == 0) {
126-
atlasEndPoint = new String[] { DGI_URL };
127-
}
127+
try {
128+
Configuration configuration = ApplicationProperties.get();
128129

129-
if (!AuthenticationUtil.isKerberosAuthenticationEnabled()) {
130-
atlasClientV2 = new AtlasClientV2(atlasEndPoint, new String[]{"admin", "admin"});
131-
atlasClient = new AtlasClient(atlasEndPoint, new String[]{"admin", "admin"});
132-
} else {
133-
atlasClientV2 = new AtlasClientV2(atlasEndPoint);
134-
atlasClient = new AtlasClient(atlasEndPoint);
135-
}
130+
String[] atlasEndPoint = configuration.getStringArray(HiveMetaStoreBridge.ATLAS_ENDPOINT);
131+
132+
if (atlasEndPoint == null || atlasEndPoint.length == 0) {
133+
atlasEndPoint = new String[] { DGI_URL };
134+
}
136135

137-
hiveMetaStoreBridge = new HiveMetaStoreBridge(configuration, conf, atlasClientV2);
136+
if (!AuthenticationUtil.isKerberosAuthenticationEnabled()) {
137+
atlasClientV2 = new AtlasClientV2(atlasEndPoint, new String[]{"admin", "admin"});
138+
atlasClient = new AtlasClient(atlasEndPoint, new String[]{"admin", "admin"});
139+
} else {
140+
atlasClientV2 = new AtlasClientV2(atlasEndPoint);
141+
atlasClient = new AtlasClient(atlasEndPoint);
142+
}
143+
144+
hiveMetaStoreBridge = new HiveMetaStoreBridge(configuration, conf, atlasClientV2);
145+
} finally {
146+
Thread.currentThread().setContextClassLoader(hiveTccl);
147+
}
138148

139149
HiveConf conf = new HiveConf();
140150

‎addons/sqoop-bridge/pom.xml‎

Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -53,6 +53,12 @@
5353
<dependency>
5454
<groupId>org.apache.atlas</groupId>
5555
<artifactId>hive-bridge</artifactId>
56+
<exclusions>
57+
<exclusion>
58+
<groupId>org.apache.atlas</groupId>
59+
<artifactId>atlas-client-v2-shaded</artifactId>
60+
</exclusion>
61+
</exclusions>
5662
</dependency>
5763

5864
<dependency>
@@ -209,6 +215,12 @@
209215
</exclusions>
210216
</dependency>
211217

218+
<dependency>
219+
<groupId>org.apache.atlas</groupId>
220+
<artifactId>atlas-client-v2</artifactId>
221+
<scope>test</scope>
222+
</dependency>
223+
212224
<dependency>
213225
<groupId>org.apache.atlas</groupId>
214226
<artifactId>atlas-graphdb-impls</artifactId>
@@ -265,6 +277,22 @@
265277
</dependencies>
266278
<build>
267279
<plugins>
280+
<!-- Parent disables Failsafe forking (forkCount=0). Exclude shaded v2 client from IT classpath
281+
to avoid Jersey NoSuchMethodError in AtlasBaseClient.getClient(). -->
282+
<plugin>
283+
<groupId>org.apache.maven.plugins</groupId>
284+
<artifactId>maven-failsafe-plugin</artifactId>
285+
<configuration combine.self="merge">
286+
<forkCount>1</forkCount>
287+
<reuseForks>false</reuseForks>
288+
<useSystemClassLoader>false</useSystemClassLoader>
289+
<classpathDependencyExcludes>
290+
<classpathDependencyExclude>org.apache.atlas:atlas-client-v2-shaded</classpathDependencyExclude>
291+
<classpathDependencyExclude>org.glassfish.jersey.media:jersey-media-json-jackson</classpathDependencyExclude>
292+
</classpathDependencyExcludes>
293+
</configuration>
294+
</plugin>
295+
268296
<plugin>
269297
<groupId>org.eclipse.jetty</groupId>
270298
<artifactId>jetty-maven-plugin</artifactId>

‎addons/storm-bridge/pom.xml‎

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -74,6 +74,12 @@
7474
<dependency>
7575
<groupId>org.apache.atlas</groupId>
7676
<artifactId>hive-bridge</artifactId>
77+
<exclusions>
78+
<exclusion>
79+
<groupId>org.apache.atlas</groupId>
80+
<artifactId>atlas-client-v2-shaded</artifactId>
81+
</exclusion>
82+
</exclusions>
7783
</dependency>
7884

7985
<dependency>
@@ -208,6 +214,22 @@
208214

209215
<build>
210216
<plugins>
217+
<!-- Parent disables Failsafe forking (forkCount=0). Exclude shaded v2 client from IT classpath
218+
to avoid Jersey NoSuchMethodError on org.apache.atlas.shaded.javax.ws.rs.RuntimeType. -->
219+
<plugin>
220+
<groupId>org.apache.maven.plugins</groupId>
221+
<artifactId>maven-failsafe-plugin</artifactId>
222+
<configuration combine.self="merge">
223+
<forkCount>1</forkCount>
224+
<reuseForks>false</reuseForks>
225+
<useSystemClassLoader>false</useSystemClassLoader>
226+
<classpathDependencyExcludes>
227+
<classpathDependencyExclude>org.apache.atlas:atlas-client-v2-shaded</classpathDependencyExclude>
228+
<classpathDependencyExclude>org.glassfish.jersey.media:jersey-media-json-jackson</classpathDependencyExclude>
229+
</classpathDependencyExcludes>
230+
</configuration>
231+
</plugin>
232+
211233
<plugin>
212234
<groupId>org.eclipse.jetty</groupId>
213235
<artifactId>jetty-maven-plugin</artifactId>

‎client/client-v1/src/main/java/org/apache/atlas/AtlasClient.java‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,7 @@
2626
import org.apache.atlas.type.AtlasType;
2727
import org.apache.atlas.typesystem.types.DataTypes;
2828
import org.apache.atlas.utils.AtlasJson;
29+
import org.apache.atlas.utils.ParamChecker;
2930
import org.apache.atlas.v1.model.instance.Referenceable;
3031
import org.apache.atlas.v1.model.instance.Struct;
3132
import org.apache.atlas.v1.model.typedef.AttributeDefinition;
@@ -412,6 +413,8 @@ public EntityResult updateEntities(Collection<Referenceable> entities) throws At
412413
public EntityResult updateEntityAttribute(final String guid, final String attribute, String value) throws AtlasServiceException {
413414
LOG.debug("Updating entity id: {}, attribute name: {}, attribute value: {}", guid, attribute, value);
414415

416+
ParamChecker.notNull(attribute, "Entity property cannot be null");
417+
415418
final API api = API_V1.UPDATE_ENTITY_PARTIAL;
416419
ObjectNode response = callAPIWithRetries(api, value, () -> {
417420
WebTarget resource = getResource(api, guid);

0 commit comments

Comments
 (0)