[BUG] 🐛 Export should return less than maxSizeOutput (#506)

Signed-off-by: CChemin <cecile.chemin@insee.fr>

Author: CChemin

docs/configuration.md

@@ -10,6 +10,7 @@
       - [Password configuration](#password-configuration)
     - [WebHooks configuration](#webhooks-configuration)
     - [Spring actuator configuration](#spring-actuator-configuration)
+    - [Export configuration](#export-configuration)
     - [Other info configuration](#other-info-configuration)
     - [Old endpoints configuration](#old-endpoints-configuration)
 
@@ -186,6 +187,15 @@ fr.insee.sugoi.security.monitor-user-password=monitor
 
 This user only has rights on `/actuator` endpoints.
 
+### Export configuration
+
+Configure the /export.csv endpoint with :
+
+| Properties                                   | Description | Default value |
+| ---------------------------------------------| :---------: | ------------: |
+| fr.insee.sugoi.export.maxSizeOutput          | The export function will stop when the number of entities reach this limit   | 10000 |
+| fr.insee.sugoi.export.pagesize               | The maximum size of pages to request on the store provider when exporting. With a pagesize of 10 the store will be requested for pages 10 by 10 before aggregating the results   | 2000  |
+
 ### Other info configuration
 
 You can add all other spring properties for example :

sugoi-api-ldap-store-provider/src/test/java/fr/insee/sugoi/ldap/LdapReaderStoreTest.java

@@ -197,6 +197,18 @@ public void testSearchAllUsers() {
         users.stream().anyMatch(user -> user.getUsername().equals("nogroup")));
   }
 
+  @Test
+  @DisplayName(
+      "When the size of the pagerequest is less than the total result, the pageresult should have isHasMoreResult")
+  public void testAllUsersHasMoreResult() {
+    PageableResult pageableResult = new PageableResult();
+    pageableResult.setSize(1);
+    PageResult<User> result = ldapReaderStore.searchUsers(new User(), pageableResult, "AND");
+    assertThat("Should get one result", 1, is(result.getResults().size()));
+    assertThat("Page size should still be 1", 1, is(result.getResults().size()));
+    assertThat("Should be tagged with more results", result.isHasMoreResult());
+  }
+
   @Test
   public void testSearchUserWithMatchingMail() {
     PageableResult pageableResult = new PageableResult();

sugoi-api-rest-services/src/main/java/fr/insee/sugoi/services/controller/ExportController.java

@@ -34,7 +34,9 @@
 import io.swagger.v3.oas.annotations.security.SecurityRequirements;
 import io.swagger.v3.oas.annotations.tags.Tag;
 import java.io.IOException;
+import java.lang.reflect.Field;
 import java.util.ArrayList;
+import java.util.Arrays;
 import java.util.List;
 import java.util.stream.Collectors;
 import javax.servlet.http.HttpServletResponse;
@@ -193,6 +195,7 @@ private void getExportUsersWithStorages(
       String realm,
       SearchType typeRecherche,
       CSVPrinter csvPrinter) {
+    int allResultsSize = 0;
     try {
 
       // set the page to maintain the search request pagination
@@ -230,7 +233,10 @@ private void getExportUsersWithStorages(
         }
 
         if (foundUsers.isHasMoreResult()) {
-          csvPrinter.close();
+          if (allResultsSize + pageSize >= maxSizeOutput) {
+            break;
+          }
+          allResultsSize += foundUsers.getResults().size();
           pageable = new PageableResult(pageSize, 0, foundUsers.getSearchToken());
         } else {
           break;

sugoi-api-rest-services/src/test/java/fr/insee/sugoi/services/controller/ExportControllerTest.java

@@ -14,21 +14,36 @@
 package fr.insee.sugoi.services.controller;
 
 import static org.hamcrest.MatcherAssert.assertThat;
+import static org.hamcrest.Matchers.greaterThanOrEqualTo;
 import static org.hamcrest.Matchers.is;
+import static org.hamcrest.Matchers.lessThan;
 import static org.mockito.ArgumentMatchers.any;
 import static org.mockito.ArgumentMatchers.eq;
 
+import com.fasterxml.jackson.databind.ObjectMapper;
 import fr.insee.sugoi.commons.services.controller.technics.SugoiAdviceController;
 import fr.insee.sugoi.core.service.ConfigService;
 import fr.insee.sugoi.core.service.UserService;
-import fr.insee.sugoi.model.*;
+import fr.insee.sugoi.model.Group;
+import fr.insee.sugoi.model.Habilitation;
+import fr.insee.sugoi.model.Organization;
+import fr.insee.sugoi.model.PostalAddress;
+import fr.insee.sugoi.model.Realm;
+import fr.insee.sugoi.model.User;
+import fr.insee.sugoi.model.UserStorage;
 import fr.insee.sugoi.model.paging.PageResult;
+import fr.insee.sugoi.model.paging.PageableResult;
+import fr.insee.sugoi.model.paging.SearchType;
 import fr.insee.sugoi.model.technics.ModelType;
 import fr.insee.sugoi.model.technics.StoreMapping;
+import java.io.UnsupportedEncodingException;
+import java.util.ArrayList;
+import java.util.Arrays;
 import java.util.List;
 import java.util.Map;
 import java.util.stream.Collectors;
 import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.DisplayName;
 import org.junit.jupiter.api.Test;
 import org.mockito.Mockito;
 import org.springframework.beans.factory.annotation.Autowired;
@@ -56,6 +71,8 @@ public class ExportControllerTest {
 
   @MockBean private ConfigService configService;
 
+  ObjectMapper objectMapper = new ObjectMapper();
+
   private Realm reamlOneUS;
   private Realm realmTwoUS;
 
@@ -148,6 +165,7 @@ public void headerTestMultipleUserStorages() throws Exception {
   @Test
   @WithMockUser
   public void bodyTestString() throws Exception {
+    String response = getOneUsResponse().getContentAsString();
     List<String> lines =
         getOneUsResponse().getContentAsString().lines().collect(Collectors.toList());
     assertThat(
@@ -236,6 +254,120 @@ private String retrieveAttribute(String header, String userLine, String attribut
     return s[attributePosition];
   }
 
+  @Test
+  @WithMockUser
+  @DisplayName("When there are only two users in userstorage all users should be returned")
+  public void getAllUsersWhenTotalIs2() throws Exception {
+    Mockito.when(
+            userService.findByProperties(
+                Mockito.eq("realmOneUS"),
+                Mockito.eq("storage"),
+                Mockito.isA(User.class),
+                Mockito.isA(PageableResult.class),
+                Mockito.isA(SearchType.class)))
+        .thenReturn(createPageResult("toto", "tutu", false));
+
+    RequestBuilder requestBuilder =
+        MockMvcRequestBuilders.get("/realms/realmOneUS/storages/storage/export/users/export.csv");
+    MockHttpServletResponse response = mockMvc.perform(requestBuilder).andReturn().getResponse();
+
+    assertThat(
+        "HttpServletResponse type should be a csv",
+        "text/csv;charset=UTF-8",
+        is(response.getContentType()));
+    List<String> usernames = convertCsvToUsernames(response.getContentAsString());
+    assertThat("Should return 2 users", 2, is(usernames.size()));
+    assertThat(
+        "Should have a user tutu",
+        usernames.stream().anyMatch(username -> username.equals("tutu")));
+  }
+
+  @Test
+  @WithMockUser
+  @DisplayName(
+      "When there are more users than the pagesize, "
+          + "but less than the maximum value then "
+          + "users should be requested several times and all users should be returned")
+  public void getTotalUsersWhenMoreThanPageSize() throws Exception {
+
+    Mockito.when(
+            userService.findByProperties(
+                Mockito.eq("realmOneUS"),
+                Mockito.eq("storage"),
+                Mockito.isA(User.class),
+                Mockito.isA(PageableResult.class),
+                Mockito.isA(SearchType.class)))
+        .thenReturn(createPageResult("toto", "tata", true))
+        .thenReturn(createPageResult("tutu", null, false));
+
+    RequestBuilder requestBuilder =
+        MockMvcRequestBuilders.get("/realms/realmOneUS/storages/storage/export/users/export.csv");
+    List<String> usernames =
+        convertCsvToUsernames(
+            mockMvc.perform(requestBuilder).andReturn().getResponse().getContentAsString());
+
+    assertThat("Should return 3 users even if page size is 2", 3, is(usernames.size()));
+  }
+
+  @Test
+  @WithMockUser
+  @DisplayName(
+      "When there are more users than the maximum a user can fetch, max exception should be returned")
+  public void exceptionIsReturnedWhenMoreThanMax() throws Exception {
+
+    Mockito.when(
+            userService.findByProperties(
+                Mockito.eq("realmOneUS"),
+                Mockito.eq("storage"),
+                Mockito.isA(User.class),
+                Mockito.isA(PageableResult.class),
+                Mockito.isA(SearchType.class)))
+        .thenReturn(createPageResult("toto", "tata", true))
+        .thenReturn(createPageResult("tutu", "toto2", true))
+        .thenReturn(createPageResult("toto4", "toto4", true))
+        .thenReturn(createPageResult("toto5", "toto6", true))
+        .thenReturn(createPageResult("toto7", "toto8", true))
+        .thenReturn(createPageResult("toto9", "toto10", true))
+        .thenReturn(createPageResult("toto11", "toto12", false));
+
+    RequestBuilder requestBuilder =
+        MockMvcRequestBuilders.get("/realms/realmOneUS/storages/storage/export/users/export.csv");
+    List<String> usernames =
+        convertCsvToUsernames(
+            mockMvc.perform(requestBuilder).andReturn().getResponse().getContentAsString());
+
+    assertThat("Should return a value under the max", 10, greaterThanOrEqualTo(usernames.size()));
+    assertThat(
+        "Should return a value above the max plus the pagesize",
+        10 - 2,
+        lessThan(usernames.size()));
+  }
+
+  private List<String> convertCsvToUsernames(String usersCsv) throws UnsupportedEncodingException {
+    List<String> usernames =
+        Arrays.stream(usersCsv.split("\n"))
+            .skip(1)
+            .map(line -> line.split(",")[0])
+            .collect(Collectors.toList());
+    return usernames;
+  }
+
+  private PageResult<User> createPageResult(
+      String firstName, String secondName, boolean hasMoreResult) {
+    PageResult<User> usersResult = new PageResult<>();
+    usersResult.setPageSize(2);
+    usersResult.setHasMoreResult(hasMoreResult);
+    List<User> users = new ArrayList<>();
+    if (firstName != null) {
+      users.add(new User(firstName));
+    }
+    if (secondName != null) {
+      users.add(new User(secondName));
+    }
+    usersResult.setResults(users);
+    return usersResult;
+  }
+
   private MockHttpServletResponse getOneUsResponse() throws Exception {
     return getResponse("/realms/realmOneUS/storages/storage/export/users/export.csv");
   }
@@ -247,6 +379,7 @@ private MockHttpServletResponse getTwoUsResponse() throws Exception {
   private MockHttpServletResponse getResponse(String url) throws Exception {
     RequestBuilder requestBuilder =
         MockMvcRequestBuilders.get(url).accept(MediaType.APPLICATION_JSON);
-    return mockMvc.perform(requestBuilder).andReturn().getResponse();
+    MockHttpServletResponse response = mockMvc.perform(requestBuilder).andReturn().getResponse();
+    return response;
   }
 }

sugoi-api-rest-services/src/test/resources/controller/application.properties

@@ -1,2 +1,3 @@
-
-logging.level.fr.insee.sugoi=debug
+logging.level.fr.insee.sugoi=debug
+fr.insee.sugoi.export.maxSizeOutput=10
+fr.insee.sugoi.export.pagesize=2