Skip to content

Commit 35126ad

Browse files
alexisszmundyalexisszmundy
authored
Merge pull request #275 from InseeFr/hotFixUnprocessed
Hotfix : no more getting data when we list unprocessed ids
2 parents 41cf8b3 + b4340d1 commit 35126ad

7 files changed

Lines changed: 164 additions & 7 deletions

File tree

‎src/main/java/fr/insee/genesis/domain/ports/spi/LunaticJsonRawDataPersistencePort.java‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -21,4 +21,5 @@ public interface LunaticJsonRawDataPersistencePort {
2121
Page<LunaticJsonRawDataModel> findByCampaignIdAndDate(String campaignId, Instant startDt, Instant endDt, Pageable pageable);
2222
long countResponsesByQuestionnaireId(String questionnaireId);
2323
List<GroupedInterrogation> findProcessedIdsGroupedByQuestionnaireSince(LocalDateTime since);
24+
List<GroupedInterrogation> findUnprocessedIds();
2425
}

‎src/main/java/fr/insee/genesis/domain/service/rawdata/LunaticJsonRawDataService.java‎

Lines changed: 9 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -247,13 +247,15 @@ public List<SurveyUnitModel> convertRawData(List<LunaticJsonRawDataModel> rawDat
247247
public List<LunaticJsonRawDataUnprocessedDto> getUnprocessedDataIds() {
248248
List<LunaticJsonRawDataUnprocessedDto> dtos = new ArrayList<>();
249249

250-
for (LunaticJsonRawDataModel dataModel : lunaticJsonRawDataPersistencePort.getAllUnprocessedData()) {
251-
dtos.add(LunaticJsonRawDataUnprocessedDto.builder()
252-
.campaignId(dataModel.campaignId())
253-
.questionnaireId(dataModel.questionnaireId())
254-
.interrogationId(dataModel.interrogationId())
255-
.build()
256-
);
250+
for (GroupedInterrogation groupedInterrogation : lunaticJsonRawDataPersistencePort.findUnprocessedIds()) {
251+
for (String interrogationId : groupedInterrogation.interrogationIds()){
252+
dtos.add(LunaticJsonRawDataUnprocessedDto.builder()
253+
.campaignId(groupedInterrogation.partitionOrCampaignId())
254+
.questionnaireId(groupedInterrogation.questionnaireId())
255+
.interrogationId(interrogationId)
256+
.build()
257+
);
258+
}
257259
}
258260
return dtos;
259261
}

‎src/main/java/fr/insee/genesis/infrastructure/adapter/LunaticJsonRawDataMongoAdapter.java‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -91,5 +91,9 @@ public List<GroupedInterrogation> findProcessedIdsGroupedByQuestionnaireSince(Lo
9191
return GroupedInterrogationDocumentMapper.INSTANCE.listDocumentToListModel(repository.aggregateRawGrouped(since));
9292
}
9393

94+
@Override
95+
public List<GroupedInterrogation> findUnprocessedIds() {
96+
return GroupedInterrogationDocumentMapper.INSTANCE.listDocumentToListModel(repository.aggregateRawGroupedWithNullProcessDate());
97+
}
9498

9599
}

‎src/main/java/fr/insee/genesis/infrastructure/repository/LunaticJsonMongoDBRepository.java‎

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -46,4 +46,21 @@ public interface LunaticJsonMongoDBRepository extends MongoRepository<LunaticJso
4646
})
4747
List<GroupedInterrogationDocument> aggregateRawGrouped(LocalDateTime since);
4848

49+
@Aggregation(pipeline = {
50+
"{ '$match': { 'processDate': null } }",
51+
"{ '$group': { " +
52+
"'_id': { " +
53+
"'questionnaireId': '$questionnaireId', " +
54+
"'partitionOrCampaignId': { '$ifNull': ['$partitionId', '$campaignId'] } " +
55+
"}, " +
56+
"'interrogationIds': { '$addToSet': '$interrogationId' } " +
57+
"} }",
58+
"{ '$project': { " +
59+
"'questionnaireId': '$_id.questionnaireId', " +
60+
"'partitionOrCampaignId': '$_id.partitionOrCampaignId', " +
61+
"'interrogationIds': 1, " +
62+
"'_id': 0 " +
63+
"} }"
64+
})
65+
List<GroupedInterrogationDocument> aggregateRawGroupedWithNullProcessDate();
4966
}

‎src/test/java/fr/insee/genesis/domain/service/rawdata/LunaticJsonRawDataServiceTest.java‎

Lines changed: 63 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,7 @@
2828
import org.junit.jupiter.params.ParameterizedTest;
2929
import org.junit.jupiter.params.provider.ValueSource;
3030

31+
import java.time.LocalDateTime;
3132
import java.util.ArrayList;
3233
import java.util.List;
3334
import java.util.Map;
@@ -479,6 +480,68 @@ void convertRawData_review_desactivated() throws Exception {
479480
Assertions.assertThat(surveyUnitQualityToolPerretAdapterStub.getReceivedMaps()).isEmpty();
480481
}
481482

483+
@Test
484+
void getUnprocessedDataIdsTest_only_processed_data() throws JsonProcessingException {
485+
surveyUnitPersistencePortStub.getMongoStub().clear();
486+
String campaignId = "SAMPLETEST-PARADATA-v1";
487+
String questionnaireId = "TESTIDQUEST";
488+
String interrogationId = "TESTinterrogationId";
489+
String json = "{\"EXTERNAL\": {\"TESTVAR_EXT\": \"test_ext\"}, " +
490+
"\"COLLECTED\": {\"TESTVAR\": {\"COLLECTED\": [\"test\"], \"EDITED\": [\"test_ed\"]}}}";
491+
492+
LunaticJsonRawDataModel rawDataModel = LunaticJsonRawDataModel.builder()
493+
.campaignId(campaignId)
494+
.questionnaireId(questionnaireId)
495+
.interrogationId(interrogationId)
496+
.data(JsonUtils.jsonToMap(json))
497+
.mode(Mode.WEB)
498+
.processDate(LocalDateTime.now())
499+
.build();
500+
501+
//WHEN
502+
lunaticJsonRawDataService.save(rawDataModel);
503+
504+
Assertions.assertThat(lunaticJsonRawDataService.getUnprocessedDataIds()).isEmpty();
505+
}
506+
@Test
507+
void getUnprocessedDataIdsTest_unprocessed_data() throws JsonProcessingException {
508+
surveyUnitPersistencePortStub.getMongoStub().clear();
509+
String campaignId = "SAMPLETEST-PARADATA-v1";
510+
String questionnaireId = "TESTIDQUEST";
511+
String interrogationId = "TESTinterrogationId";
512+
String json = "{\"EXTERNAL\": {\"TESTVAR_EXT\": \"test_ext\"}, " +
513+
"\"COLLECTED\": {\"TESTVAR\": {\"COLLECTED\": [\"test\"], \"EDITED\": [\"test_ed\"]}}}";
514+
515+
LunaticJsonRawDataModel rawDataModel = LunaticJsonRawDataModel.builder()
516+
.campaignId(campaignId)
517+
.questionnaireId(questionnaireId)
518+
.interrogationId(interrogationId)
519+
.data(JsonUtils.jsonToMap(json))
520+
.mode(Mode.WEB)
521+
.processDate(LocalDateTime.now())
522+
.build();
523+
524+
String interrogationId2 = "TESTinterrogationId2";
525+
String json2 = "{\"EXTERNAL\": {\"TESTVAR_EXT\": \"test_ext2\"}, " +
526+
"\"COLLECTED\": {\"TESTVAR\": {\"COLLECTED\": [\"test2\"], \"EDITED\": [\"test_ed2\"]}}}";
527+
528+
LunaticJsonRawDataModel rawDataModel2 = LunaticJsonRawDataModel.builder()
529+
.campaignId(campaignId)
530+
.questionnaireId(questionnaireId)
531+
.interrogationId(interrogationId2)
532+
.data(JsonUtils.jsonToMap(json2))
533+
.mode(Mode.WEB)
534+
.build();
535+
//WHEN
536+
lunaticJsonRawDataService.save(rawDataModel);
537+
lunaticJsonRawDataService.save(rawDataModel2);
538+
539+
Assertions.assertThat(lunaticJsonRawDataService.getUnprocessedDataIds()).isNotEmpty();
540+
Assertions.assertThat(lunaticJsonRawDataService.getUnprocessedDataIds()).hasSize(1);
541+
Assertions.assertThat(lunaticJsonRawDataService.getUnprocessedDataIds().getFirst().interrogationId()).isEqualTo("TESTinterrogationId2");
542+
}
543+
544+
482545
private List<String> prepareConvertTest(int rawDataSize, String campaignId, String questionnaireId) throws JsonProcessingException {
483546
//CLEAN
484547
surveyUnitPersistencePortStub.getMongoStub().clear();

‎src/test/java/fr/insee/genesis/stubs/LunaticJsonMongoDBRepositoryStub.java‎

Lines changed: 35 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -101,6 +101,41 @@ public List<GroupedInterrogationDocument> aggregateRawGrouped(LocalDateTime sinc
101101
return result;
102102
}
103103

104+
@Override
105+
public List<GroupedInterrogationDocument> aggregateRawGroupedWithNullProcessDate() {
106+
List<LunaticJsonRawDataDocument> recentDocs = documents.stream().filter(
107+
rawData -> rawData.processDate() == null).toList();
108+
109+
// Aggregation map: key = (questionnaireId, partitionOrCampaignId), value = Set of interrogationId
110+
Map<String, Map<String, Set<String>>> groupedMap = new HashMap<>();
111+
112+
for (LunaticJsonRawDataDocument doc : recentDocs) {
113+
String questionnaireId = doc.questionnaireId();
114+
String partitionOrCampaignId = doc.campaignId();
115+
String interrogationId = doc.interrogationId();
116+
groupedMap
117+
.computeIfAbsent(questionnaireId, q -> new HashMap<>())
118+
.computeIfAbsent(partitionOrCampaignId, p -> new HashSet<>())
119+
.add(interrogationId);
120+
}
121+
122+
// Conversion to a list of GroupedInterrogationDocument
123+
List<GroupedInterrogationDocument> result = new ArrayList<>();
124+
for (Map.Entry<String, Map<String, Set<String>>> entry1 : groupedMap.entrySet()) {
125+
String questionnaireId = entry1.getKey();
126+
Map<String, Set<String>> innerMap = entry1.getValue();
127+
128+
for (Map.Entry<String, Set<String>> entry2 : innerMap.entrySet()) {
129+
GroupedInterrogationDocument grouped = new GroupedInterrogationDocument();
130+
grouped.setQuestionnaireId(questionnaireId);
131+
grouped.setPartitionOrCampaignId(entry2.getKey());
132+
grouped.setInterrogationIds(new ArrayList<>(entry2.getValue()));
133+
result.add(grouped);
134+
}
135+
}
136+
return result;
137+
}
138+
104139
// Implémentations vides requises par MongoRepository
105140
@Override public <S extends LunaticJsonRawDataDocument> S save(S entity) { return null; }
106141
@Override public Optional<LunaticJsonRawDataDocument> findById(String s) { return Optional.empty(); }

‎src/test/java/fr/insee/genesis/stubs/LunaticJsonRawDataPersistanceStub.java‎

Lines changed: 35 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -141,4 +141,39 @@ public List<GroupedInterrogation> findProcessedIdsGroupedByQuestionnaireSince(Lo
141141
return GroupedInterrogationDocumentMapper.INSTANCE.listDocumentToListModel(result);
142142
}
143143

144+
@Override
145+
public List<GroupedInterrogation> findUnprocessedIds() {
146+
List<LunaticJsonRawDataDocument> recentDocs = mongoStub.stream().filter(
147+
rawData -> rawData.processDate() == null).toList();
148+
149+
// Aggregation map: key = (questionnaireId, partitionOrCampaignId), value = Set of interrogationId
150+
Map<String, Map<String, Set<String>>> groupedMap = new HashMap<>();
151+
152+
for (LunaticJsonRawDataDocument doc : recentDocs) {
153+
String questionnaireId = doc.questionnaireId();
154+
String partitionOrCampaignId = doc.campaignId();
155+
String interrogationId = doc.interrogationId();
156+
groupedMap
157+
.computeIfAbsent(questionnaireId, q -> new HashMap<>())
158+
.computeIfAbsent(partitionOrCampaignId, p -> new HashSet<>())
159+
.add(interrogationId);
160+
}
161+
162+
// Conversion to a list of GroupedInterrogationDocument
163+
List<GroupedInterrogationDocument> result = new ArrayList<>();
164+
for (Map.Entry<String, Map<String, Set<String>>> entry1 : groupedMap.entrySet()) {
165+
String questionnaireId = entry1.getKey();
166+
Map<String, Set<String>> innerMap = entry1.getValue();
167+
168+
for (Map.Entry<String, Set<String>> entry2 : innerMap.entrySet()) {
169+
GroupedInterrogationDocument grouped = new GroupedInterrogationDocument();
170+
grouped.setQuestionnaireId(questionnaireId);
171+
grouped.setPartitionOrCampaignId(entry2.getKey());
172+
grouped.setInterrogationIds(new ArrayList<>(entry2.getValue()));
173+
result.add(grouped);
174+
}
175+
}
176+
return GroupedInterrogationDocumentMapper.INSTANCE.listDocumentToListModel(result);
177+
}
178+
144179
}

0 commit comments

Comments
 (0)