Skip to content

Commit e0ab5f9

Browse files
alexisszmundyalexisszmundy
authored
Merge pull request #233 from InseeFr/horizontalScaling
fix: bug for horizontal scaling
2 parents 150f60d + aeb5c0d commit e0ab5f9

12 files changed

Lines changed: 221 additions & 14 deletions

File tree

‎src/main/java/fr/insee/genesis/controller/rest/responses/InterrogationController.java‎

Lines changed: 2 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -58,11 +58,9 @@ public ResponseEntity<Long> countAllInterrogationIdsByQuestionnaire(
5858
public ResponseEntity<List<InterrogationId>> getPaginatedInterrogationIdsByQuestionnaire(
5959
@Parameter(description = "questionnaireId", required = true) @PathVariable("questionnaireId") String questionnaireId,
6060
@Parameter(description = "if totalSize is 0, a count query is made to get the real totalSize to process", required = false) @RequestParam(defaultValue = "0") long totalSize,
61-
@Parameter(description = "workersNumbers", required = false) @RequestParam(defaultValue = "1") int workersNumbers,
62-
@Parameter(description = "workerId", required = false) @RequestParam(defaultValue = "1") int workerId,
6361
@Parameter(description = "blockSize", required = false) @RequestParam(defaultValue = "1000") long blockSize,
64-
@Parameter(description = "page number", required = false) @RequestParam(defaultValue = "0") long page) {
65-
List<InterrogationId> responses = surveyUnitService.findDistinctPageableInterrogationIdsByQuestionnaireId(questionnaireId, totalSize, workersNumbers, workerId, blockSize, page);
62+
@Parameter(description = "page number / block index", required = false) @RequestParam(defaultValue = "0") long page) {
63+
List<InterrogationId> responses = surveyUnitService.findDistinctPageableInterrogationIdsByQuestionnaireId(questionnaireId, totalSize, blockSize, page);
6664
return ResponseEntity.ok(responses);
6765
}
6866
//======== OPTIMISATIONS PERFS (END) ===========

‎src/main/java/fr/insee/genesis/controller/rest/responses/ModeController.java‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -41,4 +41,20 @@ public ResponseEntity<List<Mode>> getModesByCampaign(@RequestParam("campaignId")
4141
return ResponseEntity.ok(modes);
4242
}
4343

44+
//========= OPTIMISATIONS PERFS (START) ==========
45+
@Operation(summary = "List sources/modes used for a given questionnaire")
46+
@GetMapping(path = "/by-questionnaireV2")
47+
public ResponseEntity<List<Mode>> getModesByQuestionnaireV2(@RequestParam("questionnaireId") String questionnaireId) {
48+
List<Mode> modes = surveyUnitService.findModesByQuestionnaireIdV2(questionnaireId);
49+
return ResponseEntity.ok(modes);
50+
}
51+
52+
@Operation(summary = "List sources/modes used for a given campaign")
53+
@GetMapping(path = "/by-campaignV2")
54+
public ResponseEntity<List<Mode>> getModesByCampaignV2(@RequestParam("campaignId") String campaignId) {
55+
List<Mode> modes = surveyUnitService.findModesByCampaignIdV2(campaignId);
56+
return ResponseEntity.ok(modes);
57+
}
58+
//========= OPTIMISATIONS PERFS (END) ==========
59+
4460
}

‎src/main/java/fr/insee/genesis/controller/rest/responses/QuestionnaireController.java‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -53,6 +53,18 @@ public ResponseEntity<Set<String>> getQuestionnairesByCampaign(@RequestParam("ca
5353
return ResponseEntity.ok(questionnaires);
5454
}
5555

56+
//========= OPTIMISATIONS PERFS (START) ==========
57+
/**
58+
* @author Adrien Marchal
59+
*/
60+
@Operation(summary = "List questionnaires used for a given campaign (using a DISTINCT query)")
61+
@GetMapping(path = "/by-campaignV2")
62+
public ResponseEntity<Set<String>> getQuestionnairesByCampaignV2(@RequestParam("campaignId") String campaignId) {
63+
Set<String> questionnaires = surveyUnitService.findQuestionnaireIdsByCampaignIdV2(campaignId);
64+
return ResponseEntity.ok(questionnaires);
65+
}
66+
//========= OPTIMISATIONS PERFS (END) ==========
67+
5668
@Operation(summary = "Get the questionnaireId corresponding to an interrogationId")
5769
@GetMapping(path = "/by-interrogation")
5870
@ApiResponse(responseCode = "200", description = "Successfully retrieved the questionnaireId")

‎src/main/java/fr/insee/genesis/domain/ports/api/SurveyUnitApiPort.java‎

Lines changed: 14 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -41,19 +41,32 @@ public interface SurveyUnitApiPort {
4141
long countInterrogationIdsByQuestionnaireId(String questionnaireId);
4242

4343
List<InterrogationId> findDistinctPageableInterrogationIdsByQuestionnaireId(String questionnaireId,
44-
long totalSize, int workersNumbers, int workerId, long blockSize, long page);
44+
long totalSize, long blockSize, long page);
45+
46+
List<Mode> findModesByQuestionnaireIdV2(String questionnaireId);
4547
//========= OPTIMISATIONS PERFS (END) ==========
4648

4749
List<Mode> findModesByQuestionnaireId(String questionnaireId);
4850

4951
List<Mode> findModesByCampaignId(String campaignId);
5052

53+
//========= OPTIMISATIONS PERFS (START) ==========
54+
List<Mode> findModesByCampaignIdV2(String campaignId);
55+
//========= OPTIMISATIONS PERFS (END) ==========
56+
5157
Long deleteByQuestionnaireId(String questionnaireId);
5258

5359
long countResponses();
5460

5561
Set<String> findQuestionnaireIdsByCampaignId(String campaignId);
5662

63+
//========= OPTIMISATIONS PERFS (START) ==========
64+
/**
65+
* @author Adrien Marchal
66+
*/
67+
Set<String> findQuestionnaireIdsByCampaignIdV2(String campaignId);
68+
//========= OPTIMISATIONS PERFS (END) ==========
69+
5770
Set<String> findDistinctCampaignIds();
5871

5972
long countResponsesByCampaignId(String campaignId);

‎src/main/java/fr/insee/genesis/domain/ports/spi/SurveyUnitPersistencePort.java‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -31,6 +31,10 @@ public interface SurveyUnitPersistencePort {
3131
long countInterrogationIdsByQuestionnaireId(String questionnaireId);
3232

3333
List<SurveyUnitModel> findPageableInterrogationIdsByQuestionnaireId(String questionnaireId, Long skip, Long limit);
34+
35+
List<SurveyUnitModel> findModesByCampaignIdV2(String campaignId);
36+
37+
List<SurveyUnitModel> findModesByQuestionnaireIdV2(String questionnaireId);
3438
//======= OPTIMISATIONS PERFS (END) =========
3539

3640
List<SurveyUnitModel> findInterrogationIdsByCampaignId(String campaignId);
@@ -41,6 +45,13 @@ public interface SurveyUnitPersistencePort {
4145

4246
Set<String> findQuestionnaireIdsByCampaignId(String campaignId);
4347

48+
//========= OPTIMISATIONS PERFS (START) ==========
49+
/**
50+
* @author Adrien Marchal
51+
*/
52+
Set<String> findQuestionnaireIdsByCampaignIdV2(String campaignId);
53+
//========= OPTIMISATIONS PERFS (END) ==========
54+
4455
Set<String> findDistinctCampaignIds();
4556

4657
long countByCampaignId(String campaignId);

‎src/main/java/fr/insee/genesis/domain/service/surveyunit/SurveyUnitService.java‎

Lines changed: 36 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -280,23 +280,22 @@ public List<InterrogationId> findDistinctInterrogationIdsByQuestionnaireId(Strin
280280
* (needed for distributed process / horizontal scaling)
281281
*/
282282
@Override
283-
public List<InterrogationId> findDistinctPageableInterrogationIdsByQuestionnaireId(String questionnaireId,
284-
long totalSize, int workersNumbers, int workerId,
283+
public List<InterrogationId> findDistinctPageableInterrogationIdsByQuestionnaireId(String questionnaireId, long totalSize,
285284
long blockSize, long page) {
286285
long calculatedTotalSize;
287286
if(totalSize == 0) {
288287
calculatedTotalSize = countInterrogationIdsByQuestionnaireId(questionnaireId);
289288
} else {
290289
calculatedTotalSize = totalSize;
291290
}
292-
long workerSize = calculatedTotalSize / workersNumbers;
293-
long workerOffset = (workerId - 1) * workerSize;
294-
long skip = workerOffset + blockSize * page;
295-
long calculatedBlockSize = (skip + blockSize) > (workerOffset + workerSize) ? (workerOffset + workerSize) - (blockSize * page) : blockSize;
296-
// processing of the last element on the last worker if division result is a decimal number (modulo operator)
297-
if(workerId == workersNumbers && calculatedTotalSize % workersNumbers != 0) {
298-
calculatedBlockSize = calculatedBlockSize + 1;
291+
292+
//Check arguments
293+
long skip = page * blockSize;
294+
if(page < 0 || skip > calculatedTotalSize) {
295+
//return empty list
296+
return List.of();
299297
}
298+
long calculatedBlockSize = (skip + blockSize) > calculatedTotalSize ? calculatedTotalSize - skip : blockSize;
300299

301300
List<SurveyUnitModel> surveyUnitModels = surveyUnitPersistencePort.findPageableInterrogationIdsByQuestionnaireId(questionnaireId, skip, calculatedBlockSize);
302301
List<InterrogationId> suIds = new ArrayList<>();
@@ -337,6 +336,24 @@ public List<Mode> findModesByCampaignId(String campaignId) {
337336
return sources.stream().distinct().toList();
338337
}
339338

339+
//========= OPTIMISATIONS PERFS (START) ==========
340+
@Override
341+
public List<Mode> findModesByQuestionnaireIdV2(String questionnaireId) {
342+
List<SurveyUnitModel> surveyUnitModels = surveyUnitPersistencePort.findModesByQuestionnaireIdV2(questionnaireId);
343+
List<Mode> sources = new ArrayList<>();
344+
surveyUnitModels.forEach(surveyUnitModel -> sources.add(surveyUnitModel.getMode()));
345+
return sources.stream().distinct().toList();
346+
}
347+
348+
@Override
349+
public List<Mode> findModesByCampaignIdV2(String campaignId) {
350+
List<SurveyUnitModel> surveyUnitModels = surveyUnitPersistencePort.findModesByCampaignIdV2(campaignId);
351+
List<Mode> sources = new ArrayList<>();
352+
surveyUnitModels.forEach(surveyUnitModel -> sources.add(surveyUnitModel.getMode()));
353+
return sources.stream().distinct().toList();
354+
}
355+
//========= OPTIMISATIONS PERFS (END) ==========
356+
340357
@Override
341358
public Long deleteByQuestionnaireId(String questionnaireId) {
342359
return surveyUnitPersistencePort.deleteByQuestionnaireId(questionnaireId);
@@ -352,6 +369,16 @@ public Set<String> findQuestionnaireIdsByCampaignId(String campaignId) {
352369
return surveyUnitPersistencePort.findQuestionnaireIdsByCampaignId(campaignId);
353370
}
354371

372+
//========= OPTIMISATIONS PERFS (START) ==========
373+
/**
374+
* @author Adrien Marchal
375+
*/
376+
@Override
377+
public Set<String> findQuestionnaireIdsByCampaignIdV2(String campaignId) {
378+
return surveyUnitPersistencePort.findQuestionnaireIdsByCampaignIdV2(campaignId);
379+
}
380+
//========= OPTIMISATIONS PERFS (END) ==========
381+
355382
@Override
356383
public Set<String> findDistinctCampaignIds() {
357384
return surveyUnitPersistencePort.findDistinctCampaignIds();

‎src/main/java/fr/insee/genesis/infrastructure/adapter/SurveyUnitMongoAdapter.java‎

Lines changed: 38 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -113,6 +113,31 @@ public Set<String> findQuestionnaireIdsByCampaignId(String campaignId){
113113
return questionnaireIds;
114114
}
115115

116+
//========= OPTIMISATIONS PERFS (START) ==========
117+
/**
118+
* @author Adrien Marchal
119+
*/
120+
@Override
121+
public Set<String> findQuestionnaireIdsByCampaignIdV2(String campaignId){
122+
Set<String> mongoResponse =
123+
mongoRepository.findQuestionnaireIdsByCampaignIdV2(campaignId);
124+
125+
//Extract questionnaireIds from JSON response
126+
Set<String> questionnaireIds = new HashSet<>();
127+
for(String line : mongoResponse){
128+
ObjectMapper objectMapper = new ObjectMapper().findAndRegisterModules();
129+
try{
130+
JsonNode jsonNode = objectMapper.readTree(line);
131+
questionnaireIds.add(jsonNode.get("questionnaireId").asText());
132+
}catch (JsonProcessingException e){
133+
log.error(e.getMessage());
134+
}
135+
}
136+
137+
return questionnaireIds;
138+
}
139+
//========= OPTIMISATIONS PERFS (END) ==========
140+
116141
@Override
117142
public Set<String> findDistinctCampaignIds() {
118143
Set<String> campaignIds = new HashSet<>();
@@ -140,6 +165,19 @@ public List<SurveyUnitModel> findPageableInterrogationIdsByQuestionnaireId(Strin
140165
List<SurveyUnitDocument> surveyUnits = mongoRepository.findPageableInterrogationIdsByQuestionnaireId(questionnaireId, skip, limit);
141166
return surveyUnits.isEmpty() ? Collections.emptyList() : SurveyUnitDocumentMapper.INSTANCE.listDocumentToListModel(surveyUnits);
142167
}
168+
169+
@Override
170+
public List<SurveyUnitModel> findModesByCampaignIdV2(String campaignId) {
171+
List<SurveyUnitDocument> surveyUnits = mongoRepository.findModesByCampaignIdV2(campaignId);
172+
return surveyUnits.isEmpty() ? Collections.emptyList() : SurveyUnitDocumentMapper.INSTANCE.listDocumentToListModel(surveyUnits);
173+
174+
}
175+
176+
@Override
177+
public List<SurveyUnitModel> findModesByQuestionnaireIdV2(String questionnaireId) {
178+
List<SurveyUnitDocument> surveyUnits = mongoRepository.findModesByQuestionnaireIdV2(questionnaireId);
179+
return surveyUnits.isEmpty() ? Collections.emptyList() : SurveyUnitDocumentMapper.INSTANCE.listDocumentToListModel(surveyUnits);
180+
}
143181
//========== OPTIMISATIONS PERFS (END) ============
144182

145183
@Override

‎src/main/java/fr/insee/genesis/infrastructure/document/surveyunit/SurveyUnitDocument.java‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,7 @@ public class SurveyUnitDocument {
2121
private String interrogationId;
2222
private String questionnaireId;
2323
private String state;
24+
@Indexed
2425
private String mode;
2526
private LocalDateTime recordDate;
2627
private LocalDateTime fileDate;

‎src/main/java/fr/insee/genesis/infrastructure/repository/SurveyUnitMongoDBRepository.java‎

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -47,6 +47,20 @@ public interface SurveyUnitMongoDBRepository extends MongoRepository<SurveyUnitD
4747
"{ '$limit' : ?2 }"
4848
})
4949
List<SurveyUnitDocument> findPageableInterrogationIdsByQuestionnaireId(String questionnaireId, Long skip, Long limit);
50+
51+
@Aggregation(pipeline = {
52+
"{ '$match': { 'campaignId' : ?0 } }",
53+
"{ '$group': { '_id': '$mode' } }",
54+
"{ '$set': { 'mode': '$_id', '_id': '$$REMOVE' } }"
55+
})
56+
List<SurveyUnitDocument> findModesByCampaignIdV2(String campaignId);
57+
58+
@Aggregation(pipeline = {
59+
"{ '$match': { 'questionnaireId' : ?0 } }",
60+
"{ '$group': { '_id': '$mode' } }",
61+
"{ '$set': { 'mode': '$_id', '_id': '$$REMOVE' } }"
62+
})
63+
List<SurveyUnitDocument> findModesByQuestionnaireIdV2(String campaignId);
5064
//========= OPTIMISATIONS PERFS (END) ==========
5165

5266
@Query(value = "{ 'campaignId' : ?0 }", fields = "{ 'interrogationId' : 1, 'mode' : 1 }")
@@ -62,6 +76,19 @@ public interface SurveyUnitMongoDBRepository extends MongoRepository<SurveyUnitD
6276
@Query(value = "{ 'campaignId' : ?0 }", fields = "{ _id : 0, 'questionnaireId' : 1 }")
6377
Set<String> findQuestionnaireIdsByCampaignId(String campaignId);
6478

79+
//========= OPTIMISATIONS PERFS (START) ==========
80+
/**
81+
* @author Adrien Marchal
82+
* Here we make a "DISTINCT" query
83+
*/
84+
@Aggregation(pipeline = {
85+
"{ '$match': { 'campaignId' : ?0 } }",
86+
"{ '$group': { '_id': { 'questionnaireId' : '$questionnaireId'} } }",
87+
"{ '$set': { 'questionnaireId': '$_id', '_id': '$$REMOVE' } }"
88+
})
89+
Set<String> findQuestionnaireIdsByCampaignIdV2(String campaignId);
90+
//========= OPTIMISATIONS PERFS (END) ==========
91+
6592
long countByCampaignId(String campaignId);
6693

6794
@Query(value = "{ 'questionnaireId' : ?0 }", fields = "{ _id : 0, 'campaignId' : 1 }")

‎src/test/java/fr/insee/genesis/controller/rest/responses/ModeControllerTest.java‎

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -61,4 +61,24 @@ void getModesByCampaignTest() {
6161
Assertions.assertThat(response.getBody().getFirst()).isEqualTo(Mode.WEB);
6262
}
6363

64+
//========= OPTIMISATIONS PERFS (START) ==========
65+
@Test
66+
void getModesByQuestionnaireV2Test() {
67+
ResponseEntity<List<Mode>> response = modeControllerStatic.getModesByQuestionnaireV2(DEFAULT_QUESTIONNAIRE_ID);
68+
69+
Assertions.assertThat(response.getStatusCode().is2xxSuccessful()).isTrue();
70+
Assertions.assertThat(response.getBody()).isNotNull().isNotEmpty().hasSize(1);
71+
Assertions.assertThat(response.getBody().getFirst()).isEqualTo(Mode.WEB);
72+
}
73+
74+
@Test
75+
void getModesByCampaignV2Test() {
76+
ResponseEntity<List<Mode>> response = modeControllerStatic.getModesByCampaignV2("TESTCAMPAIGNID");
77+
78+
Assertions.assertThat(response.getStatusCode().is2xxSuccessful()).isTrue();
79+
Assertions.assertThat(response.getBody()).isNotNull().isNotEmpty().hasSize(1);
80+
Assertions.assertThat(response.getBody().getFirst()).isEqualTo(Mode.WEB);
81+
}
82+
//========= OPTIMISATIONS PERFS (END) ==========
83+
6484
}

0 commit comments

Comments
 (0)