Make split size 1000 to resolve api response timeout issue of more than 60 sec

Author: prince-cs

pom.xml

@@ -20,7 +20,7 @@
 
   <groupId>io.cdap.plugin</groupId>
   <artifactId>servicenow-plugins</artifactId>
-  <version>1.2.0-SNAPSHOT</version>
+  <version>1.2.4-SNAPSHOT</version>
   <name>ServiceNow Plugins</name>
   <packaging>jar</packaging>
   <description>Plugins for ServiceNow</description>

src/main/java/io/cdap/plugin/servicenow/apiclient/ServiceNowTableAPIClientImpl.java

@@ -127,22 +127,30 @@ public List<Map<String, String>> fetchTableRecords(
       int offset,
       int limit)
       throws ServiceNowAPIException {
-    ServiceNowTableAPIRequestBuilder requestBuilder = new ServiceNowTableAPIRequestBuilder(
-      this.conf.getRestApiEndpoint(), tableName, false)
-      .setExcludeReferenceLink(true)
-      .setDisplayValue(valueType)
-      .setLimit(limit);
+    List<Map<String, String>> combinedResults = new ArrayList<>();
+    String accessToken = getAccessToken();
+    int pageSize = limit / 5;
+    LOG.info("pageSize:::: " + pageSize);
+    LOG.info("offset:::: " + offset);
+    for (int i = 0; i < 5; i++) {
+      int currentOffset = offset + (i * pageSize);
+      ServiceNowTableAPIRequestBuilder requestBuilder = new ServiceNowTableAPIRequestBuilder(
+        this.conf.getRestApiEndpoint(), tableName, false)
+        .setExcludeReferenceLink(true)
+        .setDisplayValue(valueType)
+        .setLimit(pageSize)
+        .setOffset(currentOffset);
+      LOG.info("currentOffset:::: " +  currentOffset);
+
+      applyDateRangeToRequest(requestBuilder, startDate, endDate);
 
-    if (offset > 0) {
-      requestBuilder.setOffset(offset);
+      requestBuilder.setAuthHeader(accessToken);
+      RestAPIResponse apiResponse = executeGetWithRetries(requestBuilder.build());
+      List<Map<String, String>> batch = parseResponseToResultListOfMap(apiResponse.getResponseBody());
+      combinedResults.addAll(batch);
     }
-
-    applyDateRangeToRequest(requestBuilder, startDate, endDate);
-
-    String accessToken = getAccessToken();
-    requestBuilder.setAuthHeader(accessToken);
-    RestAPIResponse apiResponse = executeGetWithRetries(requestBuilder.build());
-    return parseResponseToResultListOfMap(apiResponse.getResponseBody());
+    LOG.info("combinedResults size:::: " + combinedResults.size());
+    return combinedResults;
   }
 
   private void applyDateRangeToRequest(ServiceNowTableAPIRequestBuilder requestBuilder, String startDate,