fix: bug for horizontal scaling

src/main/java/fr/insee/genesis/controller/rest/responses/InterrogationController.java

@@ -58,11 +58,9 @@ public ResponseEntity<Long> countAllInterrogationIdsByQuestionnaire(
     public ResponseEntity<List<InterrogationId>> getPaginatedInterrogationIdsByQuestionnaire(
             @Parameter(description = "questionnaireId", required = true) @PathVariable("questionnaireId") String questionnaireId,
             @Parameter(description = "if totalSize is 0, a count query is made to get the real totalSize to process", required = false) @RequestParam(defaultValue = "0") long totalSize,
-            @Parameter(description = "workersNumbers", required = false) @RequestParam(defaultValue = "1") int workersNumbers,
-            @Parameter(description = "workerId", required = false) @RequestParam(defaultValue = "1") int workerId,
             @Parameter(description = "blockSize", required = false) @RequestParam(defaultValue = "1000") long blockSize,
-            @Parameter(description = "page number", required = false) @RequestParam(defaultValue = "0") long page) {
-        List<InterrogationId> responses = surveyUnitService.findDistinctPageableInterrogationIdsByQuestionnaireId(questionnaireId, totalSize, workersNumbers, workerId, blockSize, page);
+            @Parameter(description = "page number / block index", required = false) @RequestParam(defaultValue = "0") long page) {
+        List<InterrogationId> responses = surveyUnitService.findDistinctPageableInterrogationIdsByQuestionnaireId(questionnaireId, totalSize, blockSize, page);
         return ResponseEntity.ok(responses);
     }
     //======== OPTIMISATIONS PERFS (END) ===========

src/main/java/fr/insee/genesis/controller/rest/responses/ModeController.java

@@ -41,4 +41,20 @@ public ResponseEntity<List<Mode>> getModesByCampaign(@RequestParam("campaignId")
         return ResponseEntity.ok(modes);
     }
 
+    //========= OPTIMISATIONS PERFS (START) ==========
+    @Operation(summary = "List sources/modes used for a given questionnaire")
+    @GetMapping(path = "/by-questionnaireV2")
+    public ResponseEntity<List<Mode>> getModesByQuestionnaireV2(@RequestParam("questionnaireId") String questionnaireId) {
+        List<Mode> modes = surveyUnitService.findModesByQuestionnaireIdV2(questionnaireId);
+        return ResponseEntity.ok(modes);
+    }
+
+    @Operation(summary = "List sources/modes used for a given campaign")
+    @GetMapping(path = "/by-campaignV2")
+    public ResponseEntity<List<Mode>> getModesByCampaignV2(@RequestParam("campaignId") String campaignId) {
+        List<Mode> modes = surveyUnitService.findModesByCampaignIdV2(campaignId);
+        return ResponseEntity.ok(modes);
+    }
+    //========= OPTIMISATIONS PERFS (END) ==========
+
 }

src/main/java/fr/insee/genesis/controller/rest/responses/QuestionnaireController.java

@@ -53,6 +53,18 @@ public ResponseEntity<Set<String>> getQuestionnairesByCampaign(@RequestParam("ca
         return ResponseEntity.ok(questionnaires);
     }
 
+    //========= OPTIMISATIONS PERFS (START) ==========
+    /**
+     * @author Adrien Marchal
+     */
+    @Operation(summary = "List questionnaires used for a given campaign (using a DISTINCT query)")
+    @GetMapping(path = "/by-campaignV2")
+    public ResponseEntity<Set<String>> getQuestionnairesByCampaignV2(@RequestParam("campaignId") String campaignId) {
+        Set<String> questionnaires = surveyUnitService.findQuestionnaireIdsByCampaignIdV2(campaignId);
+        return ResponseEntity.ok(questionnaires);
+    }
+    //========= OPTIMISATIONS PERFS (END) ==========
+
     @Operation(summary = "Get the questionnaireId corresponding to an interrogationId")
     @GetMapping(path = "/by-interrogation")
     @ApiResponse(responseCode = "200", description = "Successfully retrieved the questionnaireId")

src/main/java/fr/insee/genesis/domain/ports/api/SurveyUnitApiPort.java

@@ -41,19 +41,32 @@ public interface SurveyUnitApiPort {
     long countInterrogationIdsByQuestionnaireId(String questionnaireId);
 
     List<InterrogationId> findDistinctPageableInterrogationIdsByQuestionnaireId(String questionnaireId,
-                                                                                long totalSize, int workersNumbers, int workerId, long blockSize, long page);
+                                                                                long totalSize, long blockSize, long page);
+
+    List<Mode> findModesByQuestionnaireIdV2(String questionnaireId);
     //========= OPTIMISATIONS PERFS (END) ==========
 
     List<Mode> findModesByQuestionnaireId(String questionnaireId);
 
     List<Mode> findModesByCampaignId(String campaignId);
 
+    //========= OPTIMISATIONS PERFS (START) ==========
+    List<Mode> findModesByCampaignIdV2(String campaignId);
+    //========= OPTIMISATIONS PERFS (END) ==========
+
     Long deleteByQuestionnaireId(String questionnaireId);
 
     long countResponses();
 
     Set<String> findQuestionnaireIdsByCampaignId(String campaignId);
 
+    //========= OPTIMISATIONS PERFS (START) ==========
+    /**
+     * @author Adrien Marchal
+     */
+    Set<String> findQuestionnaireIdsByCampaignIdV2(String campaignId);
+    //========= OPTIMISATIONS PERFS (END) ==========
+
     Set<String> findDistinctCampaignIds();
 
     long countResponsesByCampaignId(String campaignId);

src/main/java/fr/insee/genesis/domain/ports/spi/SurveyUnitPersistencePort.java

@@ -31,6 +31,10 @@ public interface SurveyUnitPersistencePort {
     long countInterrogationIdsByQuestionnaireId(String questionnaireId);
 
     List<SurveyUnitModel> findPageableInterrogationIdsByQuestionnaireId(String questionnaireId, Long skip, Long limit);
+
+    List<SurveyUnitModel> findModesByCampaignIdV2(String campaignId);
+
+    List<SurveyUnitModel> findModesByQuestionnaireIdV2(String questionnaireId);
     //======= OPTIMISATIONS PERFS (END) =========
 
     List<SurveyUnitModel> findInterrogationIdsByCampaignId(String campaignId);
@@ -41,6 +45,13 @@ public interface SurveyUnitPersistencePort {
 
     Set<String> findQuestionnaireIdsByCampaignId(String campaignId);
 
+    //========= OPTIMISATIONS PERFS (START) ==========
+    /**
+     * @author Adrien Marchal
+     */
+    Set<String> findQuestionnaireIdsByCampaignIdV2(String campaignId);
+    //========= OPTIMISATIONS PERFS (END) ==========
+
     Set<String> findDistinctCampaignIds();
 
     long countByCampaignId(String campaignId);

src/main/java/fr/insee/genesis/domain/service/surveyunit/SurveyUnitService.java

@@ -264,23 +264,22 @@ public List<InterrogationId> findDistinctInterrogationIdsByQuestionnaireId(Strin
      * (needed for distributed process / horizontal scaling)
      */
     @Override
-    public List<InterrogationId> findDistinctPageableInterrogationIdsByQuestionnaireId(String questionnaireId,
-                                                                                       long totalSize, int workersNumbers, int workerId,
+    public List<InterrogationId> findDistinctPageableInterrogationIdsByQuestionnaireId(String questionnaireId, long totalSize,
                                                                                        long blockSize, long page) {
         long calculatedTotalSize;
         if(totalSize == 0) {
             calculatedTotalSize = countInterrogationIdsByQuestionnaireId(questionnaireId);
         } else {
             calculatedTotalSize = totalSize;
         }
-        long workerSize = calculatedTotalSize / workersNumbers;
-        long workerOffset = (workerId - 1) * workerSize;
-        long skip = workerOffset + blockSize * page;
-        long calculatedBlockSize = (skip + blockSize) > (workerOffset + workerSize) ? (workerOffset + workerSize) - (blockSize * page) : blockSize;
-        // processing of the last element on the last worker if division result is a decimal number (modulo operator)
-        if(workerId == workersNumbers && calculatedTotalSize % workersNumbers != 0) {
-            calculatedBlockSize = calculatedBlockSize + 1;
+
+        //Check arguments
+        long skip = page * blockSize;
+        if(page < 0 || skip > calculatedTotalSize) {
+            //return empty list
+            return List.of();
         }
+        long calculatedBlockSize = (skip + blockSize) > calculatedTotalSize ? calculatedTotalSize - skip : blockSize;
 
         List<SurveyUnitModel> surveyUnitModels = surveyUnitPersistencePort.findPageableInterrogationIdsByQuestionnaireId(questionnaireId, skip, calculatedBlockSize);
         List<InterrogationId> suIds = new ArrayList<>();
@@ -321,6 +320,24 @@ public List<Mode> findModesByCampaignId(String campaignId) {
         return sources.stream().distinct().toList();
     }
 
+    //========= OPTIMISATIONS PERFS (START) ==========
+    @Override
+    public List<Mode> findModesByQuestionnaireIdV2(String questionnaireId) {
+        List<SurveyUnitModel> surveyUnitModels = surveyUnitPersistencePort.findModesByQuestionnaireIdV2(questionnaireId);
+        List<Mode> sources = new ArrayList<>();
+        surveyUnitModels.forEach(surveyUnitModel -> sources.add(surveyUnitModel.getMode()));
+        return sources.stream().distinct().toList();
+    }
+
+    @Override
+    public List<Mode> findModesByCampaignIdV2(String campaignId) {
+        List<SurveyUnitModel> surveyUnitModels = surveyUnitPersistencePort.findModesByCampaignIdV2(campaignId);
+        List<Mode> sources = new ArrayList<>();
+        surveyUnitModels.forEach(surveyUnitModel -> sources.add(surveyUnitModel.getMode()));
+        return sources.stream().distinct().toList();
+    }
+    //========= OPTIMISATIONS PERFS (END) ==========
+
     @Override
     public Long deleteByQuestionnaireId(String questionnaireId) {
         return surveyUnitPersistencePort.deleteByQuestionnaireId(questionnaireId);
@@ -336,6 +353,16 @@ public Set<String> findQuestionnaireIdsByCampaignId(String campaignId) {
             return surveyUnitPersistencePort.findQuestionnaireIdsByCampaignId(campaignId);
     }
 
+    //========= OPTIMISATIONS PERFS (START) ==========
+    /**
+     * @author Adrien Marchal
+     */
+    @Override
+    public Set<String> findQuestionnaireIdsByCampaignIdV2(String campaignId) {
+        return surveyUnitPersistencePort.findQuestionnaireIdsByCampaignIdV2(campaignId);
+    }
+    //========= OPTIMISATIONS PERFS (END) ==========
+
     @Override
     public Set<String> findDistinctCampaignIds() {
         return surveyUnitPersistencePort.findDistinctCampaignIds();

src/main/java/fr/insee/genesis/infrastructure/adapter/SurveyUnitMongoAdapter.java

@@ -113,6 +113,31 @@ public Set<String> findQuestionnaireIdsByCampaignId(String campaignId){
 		return questionnaireIds;
 	}
 
+	//========= OPTIMISATIONS PERFS (START) ==========
+	/**
+	 * @author Adrien Marchal
+	 */
+	@Override
+	public Set<String> findQuestionnaireIdsByCampaignIdV2(String campaignId){
+		Set<String> mongoResponse =
+				mongoRepository.findQuestionnaireIdsByCampaignIdV2(campaignId);
+
+		//Extract questionnaireIds from JSON response
+		Set<String> questionnaireIds = new HashSet<>();
+		for(String line : mongoResponse){
+			ObjectMapper objectMapper = new ObjectMapper().findAndRegisterModules();
+			try{
+				JsonNode jsonNode = objectMapper.readTree(line);
+				questionnaireIds.add(jsonNode.get("questionnaireId").asText());
+			}catch (JsonProcessingException e){
+				log.error(e.getMessage());
+			}
+		}
+
+		return questionnaireIds;
+	}
+	//========= OPTIMISATIONS PERFS (END) ==========
+
 	@Override
 	public Set<String> findDistinctCampaignIds() {
 		Set<String> campaignIds = new HashSet<>();
@@ -140,6 +165,19 @@ public List<SurveyUnitModel> findPageableInterrogationIdsByQuestionnaireId(Strin
 		List<SurveyUnitDocument> surveyUnits = mongoRepository.findPageableInterrogationIdsByQuestionnaireId(questionnaireId, skip, limit);
 		return surveyUnits.isEmpty() ? Collections.emptyList() : SurveyUnitDocumentMapper.INSTANCE.listDocumentToListModel(surveyUnits);
 	}
+
+	@Override
+	public List<SurveyUnitModel> findModesByCampaignIdV2(String campaignId) {
+		List<SurveyUnitDocument> surveyUnits = mongoRepository.findModesByCampaignIdV2(campaignId);
+		return surveyUnits.isEmpty() ? Collections.emptyList() : SurveyUnitDocumentMapper.INSTANCE.listDocumentToListModel(surveyUnits);
+
+	}
+
+	@Override
+	public List<SurveyUnitModel> findModesByQuestionnaireIdV2(String questionnaireId) {
+		List<SurveyUnitDocument> surveyUnits = mongoRepository.findModesByQuestionnaireIdV2(questionnaireId);
+		return surveyUnits.isEmpty() ? Collections.emptyList() : SurveyUnitDocumentMapper.INSTANCE.listDocumentToListModel(surveyUnits);
+	}
 	//========== OPTIMISATIONS PERFS (END) ============
 
 	@Override

src/main/java/fr/insee/genesis/infrastructure/document/surveyunit/SurveyUnitDocument.java

@@ -3,6 +3,7 @@
 import fr.insee.genesis.Constants;
 import lombok.Data;
 import org.springframework.data.mongodb.core.index.CompoundIndex;
+import org.springframework.data.mongodb.core.index.Indexed;
 import org.springframework.data.mongodb.core.mapping.Document;
 
 import java.time.LocalDateTime;
@@ -13,11 +14,14 @@
 @CompoundIndex(name = "campaignId_1_questionnaireId_1", def = "{'campaignId': 1, 'questionnaireId': 1}") //1 = ascending, -1 = descending
 @CompoundIndex(name = "questionnaireId_1_campaignId_1", def = "{'questionnaireId': 1, 'campaignId': 1}")
 @CompoundIndex(name = "questionnaireId_1_interrogationId_1", def = "{'questionnaireId': 1, 'interrogationId': 1}")
+@CompoundIndex(name = "interrogationId_1_questionnaireId_1", def = "{'interrogationId': 1, 'questionnaireId': 1}")
 public class SurveyUnitDocument {
 	private String campaignId;
+	@Indexed
 	private String interrogationId;
 	private String questionnaireId;
 	private String state;
+	@Indexed
 	private String mode;
 	private LocalDateTime recordDate;
 	private LocalDateTime fileDate;

src/main/java/fr/insee/genesis/infrastructure/repository/SurveyUnitMongoDBRepository.java

@@ -47,6 +47,20 @@ public interface SurveyUnitMongoDBRepository extends MongoRepository<SurveyUnitD
 			"{ '$limit' : ?2 }"
 	})
 	List<SurveyUnitDocument> findPageableInterrogationIdsByQuestionnaireId(String questionnaireId, Long skip, Long limit);
+
+	@Aggregation(pipeline = {
+			"{ '$match': { 'campaignId' : ?0 } }",
+			"{ '$group': { '_id': '$mode' } }",
+			"{ '$set': { 'mode': '$_id', '_id': '$$REMOVE' } }"
+	})
+	List<SurveyUnitDocument> findModesByCampaignIdV2(String campaignId);
+
+	@Aggregation(pipeline = {
+			"{ '$match': { 'questionnaireId' : ?0 } }",
+			"{ '$group': { '_id': '$mode' } }",
+			"{ '$set': { 'mode': '$_id', '_id': '$$REMOVE' } }"
+	})
+	List<SurveyUnitDocument> findModesByQuestionnaireIdV2(String campaignId);
 	//========= OPTIMISATIONS PERFS (END) ==========
 
 	@Query(value = "{ 'campaignId' : ?0 }", fields = "{ 'interrogationId' : 1, 'mode' :  1 }")
@@ -62,6 +76,19 @@ public interface SurveyUnitMongoDBRepository extends MongoRepository<SurveyUnitD
 	@Query(value = "{ 'campaignId' : ?0 }", fields = "{ _id : 0, 'questionnaireId' : 1 }")
 	Set<String> findQuestionnaireIdsByCampaignId(String campaignId);
 
+	//========= OPTIMISATIONS PERFS (START) ==========
+	/**
+	 * @author Adrien Marchal
+	 * Here we make a "DISTINCT" query
+	 */
+	@Aggregation(pipeline = {
+			"{ '$match': { 'campaignId' : ?0 } }",
+			"{ '$group': { '_id': { 'questionnaireId' : '$questionnaireId'} } }",
+			"{ '$set': { 'questionnaireId': '$_id', '_id': '$$REMOVE' } }"
+	})
+	Set<String> findQuestionnaireIdsByCampaignIdV2(String campaignId);
+	//========= OPTIMISATIONS PERFS (END) ==========
+
 	long countByCampaignId(String campaignId);
 
 	@Query(value = "{ 'questionnaireId' : ?0 }", fields = "{ _id : 0, 'campaignId' : 1 }")

src/test/java/fr/insee/genesis/controller/rest/responses/ModeControllerTest.java

@@ -54,4 +54,24 @@ void getModesByCampaignTest() {
         Assertions.assertThat(response.getBody().getFirst()).isEqualTo(Mode.WEB);
     }
 
+    //========= OPTIMISATIONS PERFS (START) ==========
+    @Test
+    void getModesByQuestionnaireV2Test() {
+        ResponseEntity<List<Mode>> response = modeControllerStatic.getModesByQuestionnaireV2(DEFAULT_QUESTIONNAIRE_ID);
+
+        Assertions.assertThat(response.getStatusCode().is2xxSuccessful()).isTrue();
+        Assertions.assertThat(response.getBody()).isNotNull().isNotEmpty().hasSize(1);
+        Assertions.assertThat(response.getBody().getFirst()).isEqualTo(Mode.WEB);
+    }
+
+    @Test
+    void getModesByCampaignV2Test() {
+        ResponseEntity<List<Mode>> response = modeControllerStatic.getModesByCampaignV2("TESTCAMPAIGNID");
+
+        Assertions.assertThat(response.getStatusCode().is2xxSuccessful()).isTrue();
+        Assertions.assertThat(response.getBody()).isNotNull().isNotEmpty().hasSize(1);
+        Assertions.assertThat(response.getBody().getFirst()).isEqualTo(Mode.WEB);
+    }
+    //========= OPTIMISATIONS PERFS (END) ==========
+
 }

src/test/java/fr/insee/genesis/domain/service/SurveyUnitServiceTest.java

@@ -243,6 +243,15 @@ void findInterrogationIdsByQuestionnaireIdTest(){
         ).isNotEmpty();
     }
 
+    //========= OPTIMISATIONS PERFS (START) ==========
+    @Test
+    void findModesByQuestionnaireIdV2Test(){
+        Assertions.assertThat(surveyUnitServiceStatic.findModesByQuestionnaireIdV2(DEFAULT_QUESTIONNAIRE_ID)).filteredOn(
+                mode -> mode.equals(Mode.WEB)
+        ).isNotEmpty();
+    }
+    //========= OPTIMISATIONS PERFS (END) ==========
+
     @Test
     void getQuestionnairesByCampaignTest() {
         addAdditionnalSurveyUnitModelToMongoStub("TESTQUESTIONNAIRE2");

src/test/java/fr/insee/genesis/stubs/SurveyUnitPersistencePortStub.java

@@ -108,6 +108,33 @@ public List<SurveyUnitModel> findPageableInterrogationIdsByQuestionnaireId(Strin
         }
         return surveyUnitModelList;
     }
+
+
+    @Override
+    public List<SurveyUnitModel> findModesByCampaignIdV2(String campaignId) {
+        List<SurveyUnitModel> surveyUnitModelList = new ArrayList<>();
+        for(SurveyUnitModel SurveyUnitModel : mongoStub){
+            if(SurveyUnitModel.getCampaignId().equals(campaignId))
+                surveyUnitModelList.add(
+                        new SurveyUnitModel(SurveyUnitModel.getInterrogationId(), SurveyUnitModel.getMode())
+                );
+        }
+
+        return surveyUnitModelList;
+    }
+
+    @Override
+    public List<SurveyUnitModel> findModesByQuestionnaireIdV2(String questionnaireId) {
+        List<SurveyUnitModel> surveyUnitModelList = new ArrayList<>();
+        for(SurveyUnitModel SurveyUnitModel : mongoStub){
+            if(SurveyUnitModel.getQuestionnaireId().equals(questionnaireId))
+                surveyUnitModelList.add(
+                        new SurveyUnitModel(SurveyUnitModel.getInterrogationId(), SurveyUnitModel.getMode())
+                );
+        }
+
+        return surveyUnitModelList;
+    }
     //======= OPTIMISATIONS PERFS (END) =========
 
 
@@ -145,6 +172,14 @@ public Set<String> findQuestionnaireIdsByCampaignId(String campaignId) {
         return questionnaireIdSet;
     }
 
+    //========= OPTIMISATIONS PERFS (START) ==========
+    @Override
+    public Set<String> findQuestionnaireIdsByCampaignIdV2(String campaignId) {
+        //This stub is explicitally the same as method "findQuestionnaireIdsByCampaignId()"
+        return findQuestionnaireIdsByCampaignId(campaignId);
+    }
+    //========= OPTIMISATIONS PERFS (END) ==========
+
     @Override
     public Set<String> findDistinctCampaignIds() {
         Set<String> campaignIds = new HashSet<>();