Implement logging for pagination

Author: kylstevenson

src/main/java/io/github/jpmorganchase/fusion/Fusion.java

@@ -41,10 +41,12 @@
 import java.util.Map;
 import java.util.Objects;
 import lombok.Builder;
+import lombok.extern.slf4j.Slf4j;
 
 /**
  * Class representing the Fusion API, providing methods that correspond to available API endpoints
  */
+@Slf4j
 public class Fusion {
 
     private static final DateTimeFormatter dateTimeFormatter = DateTimeFormatter.ofPattern("yyyy-MM-dd");
@@ -210,19 +212,27 @@ private Map<String, Map<String, Object>> callForMap(String url) {
      * @return aggregated JSON response containing all pages of data
      */
     private String callAPIWithPagination(String url, Integer pageSize) {
+        log.debug("Starting paginated request to URL: {}", url);
+
         Map<String, String> headers = new HashMap<>();
         headers.put("x-jpmc-paginate", "true");
         if (pageSize != null) {
+            log.debug("Using page size: {}", pageSize);
             headers.put("x-jpmc-page-size", String.valueOf(pageSize));
         }
 
         Gson gson = new Gson();
         JsonArray aggregatedResources = new JsonArray();
         String nextToken = null;
+        int pageCount = 0;
 
         do {
+            pageCount++;
             if (nextToken != null) {
                 headers.put("x-next-token", nextToken);
+                log.debug("Fetching page {} with next token", pageCount);
+            } else {
+                log.debug("Fetching page {}", pageCount);
             }
 
             HttpResponse<String> response = this.api.callAPIWithResponse(url, headers);
@@ -231,13 +241,24 @@ private String callAPIWithPagination(String url, Integer pageSize) {
             JsonObject pageObject = JsonParser.parseString(pageJson).getAsJsonObject();
             if (pageObject.has("resources") && pageObject.get("resources").isJsonArray()) {
                 JsonArray pageResources = pageObject.getAsJsonArray("resources");
+                int pageResourceCount = pageResources.size();
                 pageResources.forEach(aggregatedResources::add);
+                log.debug("Retrieved {} resources from page {}", pageResourceCount, pageCount);
             }
 
             nextToken = getHeaderValue(response.getHeaders(), "x-next-token");
 
+            if (nextToken != null && !nextToken.isEmpty()) {
+                log.debug("Next token received, more pages available");
+            }
+
         } while (nextToken != null && !nextToken.isEmpty());
 
+        log.debug(
+                "Pagination complete. Total pages fetched: {}, Total resources: {}",
+                pageCount,
+                aggregatedResources.size());
+
         JsonObject result = new JsonObject();
         result.add("resources", aggregatedResources);
         return gson.toJson(result);
@@ -348,7 +369,7 @@ public Map<String, DataProduct> listProducts() {
      */
     public Map<String, Dataset> listDatasets(String catalogName, String contains, boolean idContains) {
         String url = String.format("%1scatalogs/%2s/datasets", this.rootURL, catalogName);
-        String json = callAPIWithPagination(url, null);
+        String json = callAPIWithPagination(url, 500);
         return filterDatasets(responseParser.parseDatasetResponse(json, catalogName), contains, idContains);
     }
 

src/test/java/io/github/jpmorganchase/fusion/FusionTest.java

@@ -750,32 +750,4 @@ public void testListAttributesWithPagination() throws Exception {
         // Verify pagination occurred
         verify(apiManager, times(2)).callAPIWithResponse(anyString(), anyMap());
     }
-
-    @Test
-    public void testPaginationDoesNotSendPageSizeHeaderWhenNull() throws Exception {
-        Fusion f = stubFusion();
-
-        // Setup single page response
-        io.github.jpmorganchase.fusion.http.HttpResponse<String> httpResponse =
-                io.github.jpmorganchase.fusion.http.HttpResponse.<String>builder()
-                        .statusCode(200)
-                        .body("{\"resources\":[{\"identifier\":\"dataset1\"}]}")
-                        .headers(new HashMap<>())
-                        .build();
-
-        when(apiManager.callAPIWithResponse(anyString(), anyMap())).thenReturn(httpResponse);
-
-        Map<String, Dataset> stubResponse = new HashMap<>();
-        stubResponse.put("dataset1", Dataset.builder().identifier("dataset1").build());
-        when(responseParser.parseDatasetResponse(anyString(), eq("common"))).thenReturn(stubResponse);
-
-        f.listDatasets("common");
-
-        // Verify that x-jpmc-page-size header is NOT present (only x-jpmc-paginate should be)
-        verify(apiManager, times(1))
-                .callAPIWithResponse(
-                        anyString(),
-                        argThat(headers ->
-                                headers.containsKey("x-jpmc-paginate") && !headers.containsKey("x-jpmc-page-size")));
-    }
 }