RANGER-5520:Audit Server refactoring to segregate audit ingestion and dispatching functionality - Fix merge conflict

Author: Ramesh Mani

.github/workflows/ci.yml

@@ -33,7 +33,7 @@ env:
   MAVEN_OPTS: -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.http.retryHandler.class=standard -Dmaven.wagon.http.retryHandler.count=3
 
 jobs:
-  build-8:
+  build-17:
     runs-on: ubuntu-22.04
     timeout-minutes: 60
     steps:
@@ -47,12 +47,12 @@ jobs:
           key: maven-repo-${{ hashFiles('**/pom.xml') }}
           restore-keys: |
             maven-repo-
-      - name: Set up JDK 8
+      - name: Set up JDK 17
         uses: actions/setup-java@v5
         with:
-          java-version: '8'
+          java-version: '17'
           distribution: 'temurin'
-      - name: build (8)
+      - name: build (17)
         run: mvn -T 8 clean verify --no-transfer-progress -B -V
 
       - name: run code coverage
@@ -69,49 +69,20 @@ jobs:
       - name: Upload artifacts
         uses: actions/upload-artifact@v7
         with:
-          name: target-8
-          path: target/*
-
-  build-11:
-    needs:
-      - build-8
-    runs-on: ubuntu-22.04
-    timeout-minutes: 60
-    steps:
-      - uses: actions/checkout@v6
-      - name: Cache for maven dependencies
-        uses: actions/cache/restore@v5
-        with:
-          path: |
-            ~/.m2/repository/*/*/*
-            !~/.m2/repository/org/apache/ranger
-          key: maven-repo-${{ hashFiles('**/pom.xml') }}
-          restore-keys: |
-            maven-repo-
-      - name: Set up JDK 11
-        uses: actions/setup-java@v5
-        with:
-          java-version: '11'
-          distribution: 'temurin'
-      - name: build (11)
-        run: mvn -T 8 clean verify -pl '!knox-agent' --no-transfer-progress -B -V
-      - name: Upload artifacts
-        uses: actions/upload-artifact@v7
-        with:
-          name: target-11
+          name: target-17
           path: target/*
 
   services-docker-build:
     needs:
-      - build-8
+      - build-17
     runs-on: ubuntu-24.04 # bumping os version for more disk space
     timeout-minutes: 60 # bumping timeout to allow containers to come up
     steps:
       - uses: actions/checkout@v6
-      - name: Download build-8 artifacts
+      - name: Download build-17 artifacts
         uses: actions/download-artifact@v8
         with:
-          name: target-8
+          name: target-17
 
       - name: Copy artifacts for docker build
         run: |
@@ -177,15 +148,15 @@ jobs:
 
   plugins-docker-build:
     needs:
-      - build-8
+      - build-17
     runs-on: ubuntu-24.04 # bumping os version for more disk space
     timeout-minutes: 60 # bumping timeout to allow containers to come up
     steps:
       - uses: actions/checkout@v6
-      - name: Download build-8 artifacts
+      - name: Download build-17 artifacts
         uses: actions/download-artifact@v8
         with:
-          name: target-8
+          name: target-17
 
       - name: Copy artifacts for docker build
         run: |

.github/workflows/docs.yml

@@ -18,7 +18,7 @@ name: docs
 
 on:
   push:
-    branches: [ ranger_5353, master ]
+    branches: [ dev, master ]
 
 permissions:
   contents: write

README.md

@@ -54,7 +54,7 @@ Ranger is built using [Apache Maven](https://maven.apache.org/). To run Ranger:
 
 ## Regular Build Process
 
-1. Check out the code from GIT repository
+1. Check out the code from GIT repository and make sure JAVA_HOME & PATH environment variables are properly set (JDK17).
 
 2. On the root folder, please execute the following Maven command:
 
@@ -120,6 +120,10 @@ Ranger is built using [Apache Maven](https://maven.apache.org/). To run Ranger:
 
 4. Modify the install.properties file with appropriate variables 
 
-5. - If the module has setup.sh, execute ./setup.sh
+5. - export/ set JAVA_OPTS environment variable.
+   ```
+   JAVA_OPTS="--add-opens=java.base/java.nio=ALL-UNNAMED --add-exports=java.base/sun.net.dns=ALL-UNNAMED --add-exports=java.base/sun.net.util=ALL-UNNAMED --add-opens=java.base/java.lang=ALL-UNNAMED --add-exports=java.xml.crypto/com.sun.org.apache.xml.internal.security.utils=ALL-UNNAMED"
+   ```
+   - If the module has setup.sh, execute ./setup.sh
    - If the install.sh file does not exists, execute ./enable-<component>-plugin.sh
 

agents-audit/core/pom.xml

@@ -106,10 +106,5 @@
             <artifactId>mockito-core</artifactId>
             <scope>test</scope>
         </dependency>
-        <dependency>
-            <groupId>org.testng</groupId>
-            <artifactId>testng</artifactId>
-            <scope>test</scope>
-        </dependency>
     </dependencies>
 </project>

agents-audit/core/src/test/java/org/apache/ranger/audit/utils/RangerJSONAuditWriterTest.java

@@ -101,9 +101,13 @@ public void verifyFileRolloverWithAppend() throws Exception {
 
         reset(jsonAuditWriter);
 
-        assertTrue(jsonAuditWriter.logJSON(Collections.singleton("Last log file will be opened in append mode and this event will be written")));
-        assertTrue(jsonAuditWriter.logJSON(Collections.singleton("This event will also be written in append mode")));
-        Path auditPath1 = jsonAuditWriter.auditPath;
+        final Path auditPath1;
+        synchronized (jsonAuditWriter) {
+            assertTrue(jsonAuditWriter.logJSON(Collections.singleton("Last log file will be opened in append mode and this event will be written")));
+            assertTrue(jsonAuditWriter.logJSON(Collections.singleton("This event will also be written in append mode")));
+            auditPath1 = jsonAuditWriter.auditPath;
+        }
+        assertNotNull(auditPath1);
 
         Thread.sleep(6000);
 
@@ -118,6 +122,7 @@ public void verifyFileRolloverWithAppend() throws Exception {
 
         // ensure the same rolled over file is not used for append
         assertNotEquals(auditPath1, jsonAuditWriter.auditPath);
+        assertNotNull(jsonAuditWriter.auditPath, "second write should create a new audit file path");
 
         // cleanup
         jsonAuditWriter.fileSystem.deleteOnExit(auditPath1);
@@ -165,14 +170,19 @@ public void verifyFileRolloverAfterThreshold() throws Exception {
         jsonAuditWriter.fileRolloverSec = 5; // in seconds
         jsonAuditWriter.init(props, "test", "localfs", auditConfigs);
 
-        assertTrue(jsonAuditWriter.logJSON(Collections.singleton("First file created and added this line!")));
-        Path auditPath1 = jsonAuditWriter.auditPath;
+        final Path auditPath1;
+        synchronized (jsonAuditWriter) {
+            assertTrue(jsonAuditWriter.logJSON(Collections.singleton("First file created and added this line!")));
+            auditPath1 = jsonAuditWriter.auditPath;
+        }
+        assertNotNull(auditPath1);
 
         Thread.sleep(6000);
 
         assertNull(jsonAuditWriter.ostream);
         assertNull(jsonAuditWriter.logWriter);
         assertTrue(jsonAuditWriter.logJSON(Collections.singleton("Second file created since rollover happened!")));
+        assertNotNull(jsonAuditWriter.auditPath, "second write should create a new audit file path");
 
         // cleanup
         jsonAuditWriter.fileSystem.deleteOnExit(auditPath1);

agents-audit/dest-auditserver/pom.xml

@@ -37,6 +37,11 @@
             <artifactId>jackson-jaxrs-json-provider</artifactId>
             <version>${fasterxml.jackson.version}</version>
         </dependency>
+        <dependency>
+            <groupId>javax.ws.rs</groupId>
+            <artifactId>javax.ws.rs-api</artifactId>
+            <version>${javax.ws.rs-api.version}</version>
+        </dependency>
         <dependency>
             <groupId>org.apache.ranger</groupId>
             <artifactId>ranger-audit-core</artifactId>
@@ -47,6 +52,21 @@
             <artifactId>ranger-plugins-common</artifactId>
             <version>${project.version}</version>
         </dependency>
+        <dependency>
+            <groupId>org.glassfish.jersey.core</groupId>
+            <artifactId>jersey-client</artifactId>
+            <version>${jersey-client.version}</version>
+        </dependency>
+        <dependency>
+            <groupId>org.glassfish.jersey.inject</groupId>
+            <artifactId>jersey-hk2</artifactId>
+            <version>${jersey-client.version}</version>
+        </dependency>
+        <dependency>
+            <groupId>org.glassfish.jersey.media</groupId>
+            <artifactId>jersey-media-json-jackson</artifactId>
+            <version>${jersey-client.version}</version>
+        </dependency>
         <dependency>
             <groupId>org.slf4j</groupId>
             <artifactId>slf4j-api</artifactId>
@@ -60,10 +80,5 @@
             <version>${slf4j.version}</version>
             <scope>test</scope>
         </dependency>
-        <dependency>
-            <groupId>org.testng</groupId>
-            <artifactId>testng</artifactId>
-            <scope>test</scope>
-        </dependency>
     </dependencies>
 </project>

agents-audit/dest-auditserver/src/main/java/org/apache/ranger/audit/destination/RangerAuditServerDestination.java

@@ -19,19 +19,23 @@
 
 package org.apache.ranger.audit.destination;
 
-import com.sun.jersey.api.client.ClientResponse;
-import com.sun.jersey.api.client.WebResource;
 import org.apache.commons.lang3.StringUtils;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.security.UserGroupInformation;
 import org.apache.http.HttpStatus;
 import org.apache.ranger.audit.model.AuditEventBase;
 import org.apache.ranger.audit.model.AuthzAuditEvent;
 import org.apache.ranger.audit.provider.MiscUtil;
+import org.apache.ranger.plugin.authn.DefaultJwtProvider;
 import org.apache.ranger.plugin.util.RangerRESTClient;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+import javax.ws.rs.client.Entity;
+import javax.ws.rs.client.WebTarget;
+import javax.ws.rs.core.MediaType;
+import javax.ws.rs.core.Response;
+
 import java.security.PrivilegedExceptionAction;
 import java.util.Collection;
 import java.util.HashMap;
@@ -85,6 +89,10 @@ public void init(Properties props, String propPrefix) {
 
         this.restClient = new RangerRESTClient(url, sslConfigFileName, config);
 
+        if (AUTH_TYPE_JWT.equalsIgnoreCase(authType)) {
+            this.restClient.setJwtProvider(new DefaultJwtProvider("ranger.plugin.policy.rest.client", config));
+        }
+
         this.restClient.setRestClientConnTimeOutMs(connTimeoutMs);
         this.restClient.setRestClientReadTimeOutMs(readTimeoutMs);
         this.restClient.setMaxRetryAttempts(maxRetryAttempts);
@@ -180,6 +188,8 @@ private boolean sendBatch(Collection<AuditEventBase> events, RangerRESTClient re
             queryParams.put(QUERY_PARAM_APP_ID, appId);
         }
 
+        Response response = null;
+
         try {
             final UserGroupInformation user         = MiscUtil.getUGILoginUser();
             final boolean              isSecureMode = isKerberosAuthenticated();
@@ -190,10 +200,8 @@ private boolean sendBatch(Collection<AuditEventBase> events, RangerRESTClient re
                 LOG.debug("Sending audit batch of {} events. SecureMode: {}, User: {}", events.size(), isSecureMode, user != null ? user.getUserName() : "null");
             }
 
-            final ClientResponse response;
-
             if (isSecureMode) {
-                response = MiscUtil.executePrivilegedAction((PrivilegedExceptionAction<ClientResponse>) () -> {
+                response = MiscUtil.executePrivilegedAction((PrivilegedExceptionAction<Response>) () -> {
                     try {
                         return postAuditEvents(restClient, queryParams, events);
                     } catch (Exception e) {
@@ -210,7 +218,7 @@ private boolean sendBatch(Collection<AuditEventBase> events, RangerRESTClient re
 
                 if (status == HttpStatus.SC_OK) {
                     if (LOG.isDebugEnabled()) {
-                        LOG.debug("Audit batch sent successfully. {} events delivered. Response: {}", events.size(), response.getEntity(String.class));
+                        LOG.debug("Audit batch sent successfully. {} events delivered. Response: {}", events.size(), response.readEntity(String.class));
                     }
 
                     ret = true;
@@ -219,7 +227,7 @@ private boolean sendBatch(Collection<AuditEventBase> events, RangerRESTClient re
 
                     try {
                         if (response.hasEntity()) {
-                            errorBody = response.getEntity(String.class);
+                            errorBody = response.readEntity(String.class);
                         }
                     } catch (Exception e) {
                         LOG.debug("Failed to read error response body", e);
@@ -242,27 +250,33 @@ private boolean sendBatch(Collection<AuditEventBase> events, RangerRESTClient re
             LOG.error("Failed to send audit batch of {} events. Error: {}", events.size(), e.getMessage(), e);
 
             ret = false;
+        } finally {
+            if (response != null) {
+                try {
+                    response.close();
+                } catch (Exception e) {
+                    LOG.debug("Error closing HTTP response", e);
+                }
+            }
         }
 
         return ret;
     }
 
-    private ClientResponse postAuditEvents(RangerRESTClient restClient, Map<String, String> params, Collection<AuditEventBase> events) {
+    private Response postAuditEvents(RangerRESTClient restClient, Map<String, String> params, Collection<AuditEventBase> events) {
         LOG.debug("Posting {} audit events to {}", events.size(), REST_RELATIVE_PATH_POST);
 
-        WebResource webResource = restClient.getResource(REST_RELATIVE_PATH_POST);
+        WebTarget target = restClient.getResource(REST_RELATIVE_PATH_POST);
 
         if (params != null && !params.isEmpty()) {
             for (Map.Entry<String, String> entry : params.entrySet()) {
-                webResource = webResource.queryParam(entry.getKey(), entry.getValue());
+                target = target.queryParam(entry.getKey(), entry.getValue());
             }
         }
 
-        return webResource
-                .accept("application/json")
-                .type("application/json")
-                .entity(events)
-                .post(ClientResponse.class);
+        return target.request(MediaType.APPLICATION_JSON_TYPE)
+                .accept(MediaType.APPLICATION_JSON_TYPE)
+                .post(Entity.entity(events, MediaType.APPLICATION_JSON_TYPE));
     }
 
     private static Configuration createRESTClientConfiguration(Properties props, String propPrefix, String authType) {

agents-audit/dest-cloudwatch/pom.xml

@@ -52,12 +52,5 @@
             <artifactId>slf4j-api</artifactId>
             <version>${slf4j.version}</version>
         </dependency>
-
-        <!-- Test -->
-        <dependency>
-            <groupId>org.testng</groupId>
-            <artifactId>testng</artifactId>
-            <scope>test</scope>
-        </dependency>
     </dependencies>
 </project>

agents-audit/dest-es/pom.xml

@@ -207,12 +207,5 @@
             <artifactId>slf4j-api</artifactId>
             <version>${slf4j.version}</version>
         </dependency>
-
-        <!-- Test -->
-        <dependency>
-            <groupId>org.testng</groupId>
-            <artifactId>testng</artifactId>
-            <scope>test</scope>
-        </dependency>
     </dependencies>
 </project>