Skip to content

Commit eb254da

Browse files
alexisszmundyalexisszmundy
committed
feat(lunaticjsondata): keep null values
1 parent 7b3c7a0 commit eb254da

4 files changed

Lines changed: 20 additions & 17 deletions

File tree

‎src/main/java/fr/insee/genesis/domain/converter/rawdata/RawDataConverter.java‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -160,8 +160,9 @@ private static void fillRawDataMapWithIterations(
160160
rawDataMap.put(variableName, rawValuesList);
161161
return;
162162
}
163-
Map<String, Object> stateMap = (Map<String, Object>) rawDataMap.get(variableName);
163+
Map<String, Object> stateMap = new HashMap<>((Map<String, Object>) rawDataMap.get(variableName));
164164
stateMap.put(dataState.toString(), rawValuesList);
165+
rawDataMap.put(variableName, stateMap);
165166
}
166167

167168
/**

‎src/main/java/fr/insee/genesis/domain/utils/DataVerifier.java‎

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -45,7 +45,7 @@ public class DataVerifier {
4545
* @param variablesMap VariablesMap containing definitions of each variable
4646
*/
4747
public static void verifySurveyUnits(List<SurveyUnitModel> surveyUnitModelsList, VariablesMap variablesMap){
48-
List<SurveyUnitModel> surveyUnitModelsListFormatted = new ArrayList<>(); // Created FORCED SU models
48+
List<SurveyUnitModel> surveyUnitModelsListFormatted = new ArrayList<>(); // Created FORMATTED SU models
4949

5050
for(String interrogationId : getInterrogationIds(surveyUnitModelsList)) { // For each id of the list
5151
List<SurveyUnitModel> srcSurveyUnitModelsOfInterrogationId = surveyUnitModelsList.stream().filter(element -> element.getInterrogationId().equals(interrogationId)).toList();
@@ -56,7 +56,7 @@ public static void verifySurveyUnits(List<SurveyUnitModel> surveyUnitModelsList,
5656
collectedVariablesManagement(srcSurveyUnitModelsOfInterrogationId, variablesMap, correctedCollectedVariables);
5757
externalVariablesManagement(srcSurveyUnitModelsOfInterrogationId, variablesMap, correctedExternalVariables);
5858

59-
//Create FORCED if any corrected variable
59+
//Create FORMATTED if any corrected variable
6060
if(!correctedCollectedVariables.isEmpty() || !correctedExternalVariables.isEmpty()){
6161
SurveyUnitModel newFormattedSurveyUnitModel = createFormattedSurveyUnitModel(surveyUnitModelsList, interrogationId, correctedCollectedVariables, correctedExternalVariables);
6262
surveyUnitModelsListFormatted.add(newFormattedSurveyUnitModel);
@@ -194,6 +194,10 @@ private static VariableModel verifyVariable(
194194
fr.insee.bpm.metadata.model.Variable variableDefinition,
195195
DataState dataState
196196
) {
197+
//null values are OK
198+
if(variableModel.value() == null){
199+
return null;
200+
}
197201
if(isParseError(variableModel.value(), variableDefinition.getType(),dataState)){
198202
return VariableModel.builder()
199203
.varId(variableModel.varId())

‎src/test/java/fr/insee/genesis/controller/rest/responses/RawResponseControllerIT.java‎

Lines changed: 6 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -508,7 +508,7 @@ void process_raw_response_keep_null_test(boolean isNullSurveyUnitValues) {
508508
Assertions.assertThat(savedDocuments).isNotNull().hasSize(interrogationIds.size());
509509
SurveyUnitDocument savedDocument = savedDocuments.stream().filter(
510510
surveyUnitDocument ->
511-
surveyUnitDocument.getInterrogationId().equals("interrogationId"))
511+
surveyUnitDocument.getInterrogationId().equals(interrogationIds.getFirst()))
512512
.toList().getFirst();
513513
Assertions.assertThat(savedDocument.getCollectedVariables()).isNotNull().hasSize(1);
514514
VariableDocument variableDocument = savedDocument.getCollectedVariables().getFirst();
@@ -577,7 +577,7 @@ void process_raw_response_null_on_absent_test() {
577577
Assertions.assertThat(savedDocuments).isNotNull().hasSize(interrogationIds.size());
578578
SurveyUnitDocument savedDocument = savedDocuments.stream().filter(
579579
surveyUnitDocument ->
580-
surveyUnitDocument.getInterrogationId().equals("interrogationId"))
580+
surveyUnitDocument.getInterrogationId().equals(interrogationIds.getFirst()))
581581
.toList().getFirst();
582582
Assertions.assertThat(savedDocument.getCollectedVariables()).isNotNull().hasSize(1);
583583
VariableDocument variableDocument = savedDocument.getCollectedVariables().getFirst();
@@ -891,7 +891,7 @@ void processLunaticJsonRawData_keep_null_test(boolean isNullSurveyUnitValues){
891891
Assertions.assertThat(savedDocuments).isNotNull().hasSize(interrogationIds.size());
892892
SurveyUnitDocument savedDocument = savedDocuments.stream().filter(
893893
surveyUnitDocument ->
894-
surveyUnitDocument.getInterrogationId().equals("interrogationId"))
894+
surveyUnitDocument.getInterrogationId().equals(interrogationIds.getFirst()))
895895
.toList().getFirst();
896896
Assertions.assertThat(savedDocument.getCollectedVariables()).isNotNull().hasSize(1);
897897
VariableDocument variableDocument = savedDocument.getCollectedVariables().getFirst();
@@ -961,7 +961,7 @@ void processLunaticJsonRawData_null_on_absent_test(){
961961
Assertions.assertThat(savedDocuments).isNotNull().hasSize(interrogationIds.size());
962962
SurveyUnitDocument savedDocument = savedDocuments.stream().filter(
963963
surveyUnitDocument ->
964-
surveyUnitDocument.getInterrogationId().equals("interrogationId"))
964+
surveyUnitDocument.getInterrogationId().equals(interrogationIds.getFirst()))
965965
.toList().getFirst();
966966
Assertions.assertThat(savedDocument.getCollectedVariables()).isNotNull().hasSize(1);
967967
VariableDocument variableDocument = savedDocument.getCollectedVariables().getFirst();
@@ -1269,6 +1269,7 @@ private SurveyUnitDocument getSurveyUnitDocument(
12691269
alreadyPresentSurveyUnitDocument.setCollectionInstrumentId(collectionInstrumentId);
12701270
alreadyPresentSurveyUnitDocument.setInterrogationId(interrogationId);
12711271
alreadyPresentSurveyUnitDocument.setMode(Mode.WEB.getModeName());
1272+
alreadyPresentSurveyUnitDocument.setState("COLLECTED");
12721273

12731274
alreadyPresentSurveyUnitDocument.setCollectedVariables(new ArrayList<>());
12741275
VariableDocument oldVariable = new VariableDocument();
@@ -1284,6 +1285,7 @@ private SurveyUnitDocument getSurveyUnitDocument(
12841285
oldVariable.setIteration(1);
12851286
oldVariable.setValue(externalValue);
12861287
oldVariable.setScope(Constants.ROOT_GROUP_NAME);
1288+
alreadyPresentSurveyUnitDocument.getExternalVariables().add(oldVariable);
12871289
return alreadyPresentSurveyUnitDocument;
12881290
}
12891291

‎src/test/java/fr/insee/genesis/domain/converter/rawdata/LunaticJsonRawDataConverterTest.java‎

Lines changed: 6 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -40,6 +40,7 @@ class LunaticJsonRawDataConverterTest {
4040

4141
private static final LocalDateTime DATE_TIME = LocalDateTime.parse("2025-01-01T10:00:00");
4242
private static final String QUESTIONNAIRE_ID = "testQuestionnaire";
43+
private static final String INTERROGATION_ID = "testInterrogation";
4344

4445
@Mock
4546
private SurveyUnitService surveyUnitService;
@@ -229,13 +230,12 @@ void shouldAssignRootScopeWhenExternalVariableIsMissingFromMetadata() {
229230
.extracting(VariableModel::scope)
230231
.isEqualTo(Constants.ROOT_GROUP_NAME);
231232
}
232-
//FIXME Faire les devs de feat(rawresponses): keep null values if variable present in last survey... dans lunaticjson converter
233+
233234
@Nested
234235
@DisplayName("Null cases tests")
235236
class NullVariablesTests {
236237

237238
//NullVariablesTests constants
238-
private static final String INTERROGATION_ID = "testInterrogation";
239239
private static final String COLLECTED_VARIABLE_NAME = "VAR1";
240240
private static final String COLLECTED_VARIABLE_VALUE = "test";
241241
private static final String EXTERNAL_VARIABLE_NAME = "EXTVAR1";
@@ -471,14 +471,14 @@ void shouldKeepNull(boolean isNewVariablesPresent){
471471
SurveyUnitModel surveyUnitModel = SurveyUnitModel.builder()
472472
.collectionInstrumentId(QUESTIONNAIRE_ID)
473473
.interrogationId(INTERROGATION_ID)
474+
.state(DataState.COLLECTED)
474475
.collectedVariables(new ArrayList<>())
475476
.externalVariables(new ArrayList<>())
476477
.build();
477478
surveyUnitModel.getCollectedVariables().add(
478479
VariableModel.builder()
479480
.varId(COLLECTED_VARIABLE_NAME)
480481
.value(null)
481-
.state(DataState.COLLECTED)
482482
.scope(Constants.ROOT_GROUP_NAME)
483483
.iteration(1)
484484
.build()
@@ -487,7 +487,6 @@ void shouldKeepNull(boolean isNewVariablesPresent){
487487
VariableModel.builder()
488488
.varId(EXTERNAL_VARIABLE_NAME)
489489
.value(null)
490-
.state(DataState.COLLECTED)
491490
.scope(Constants.ROOT_GROUP_NAME)
492491
.iteration(1)
493492
.build()
@@ -540,7 +539,6 @@ void shouldKeepNullIteration(boolean isNewVariablesPresent){
540539
VariableModel.builder()
541540
.varId(COLLECTED_VARIABLE_NAME)
542541
.value(null)
543-
.state(DataState.COLLECTED)
544542
.scope(Constants.ROOT_GROUP_NAME)
545543
.iteration(2)
546544
.build()
@@ -549,7 +547,6 @@ void shouldKeepNullIteration(boolean isNewVariablesPresent){
549547
VariableModel.builder()
550548
.varId(EXTERNAL_VARIABLE_NAME)
551549
.value(null)
552-
.state(DataState.COLLECTED)
553550
.scope(Constants.ROOT_GROUP_NAME)
554551
.iteration(2)
555552
.build()
@@ -603,6 +600,7 @@ private SurveyUnitModel getSurveyUnitModel() {
603600
SurveyUnitModel surveyUnitModel = SurveyUnitModel.builder()
604601
.collectionInstrumentId(QUESTIONNAIRE_ID)
605602
.interrogationId(INTERROGATION_ID)
603+
.state(DataState.COLLECTED)
606604
.collectedVariables(new ArrayList<>())
607605
.externalVariables(new ArrayList<>())
608606
.build();
@@ -611,7 +609,6 @@ private SurveyUnitModel getSurveyUnitModel() {
611609
VariableModel.builder()
612610
.varId(COLLECTED_VARIABLE_NAME)
613611
.value(COLLECTED_VARIABLE_VALUE)
614-
.state(DataState.COLLECTED)
615612
.scope(Constants.ROOT_GROUP_NAME)
616613
.iteration(1)
617614
.build()
@@ -620,7 +617,6 @@ private SurveyUnitModel getSurveyUnitModel() {
620617
VariableModel.builder()
621618
.varId(EXTERNAL_VARIABLE_NAME)
622619
.value(EXTERNAL_VARIABLE_VALUE)
623-
.state(DataState.COLLECTED)
624620
.scope(Constants.ROOT_GROUP_NAME)
625621
.iteration(1)
626622
.build()
@@ -683,9 +679,9 @@ private void assertSecondIterationNull(List<SurveyUnitModel> surveyUnitModels) {
683679

684680
private LunaticJsonRawDataModel rawData(Map<String, Object> data) {
685681
LunaticJsonRawDataModel rawData = LunaticJsonRawDataModel.builder()
686-
.questionnaireId("questionnaire-id")
682+
.questionnaireId(QUESTIONNAIRE_ID)
687683
.mode(Mode.WEB)
688-
.interrogationId("interrogation-id")
684+
.interrogationId(INTERROGATION_ID)
689685
.idUE("survey-unit-id")
690686
.recordDate(DATE_TIME)
691687
.data(data)

0 commit comments

Comments
 (0)