delete fileDate from new surveyrUnitModels and replace them with rawRecordDate (#482)

Author: Hajarel-moukh

src/main/java/fr/insee/genesis/controller/adapter/LunaticXmlAdapter.java

@@ -72,8 +72,7 @@ private static SurveyUnitModel getStateDataFromSurveyUnit(LunaticXmlSurveyUnit s
                 .state(dataState)
                 .mode(mode)
                 .recordDate(Instant.now())
-                .fileDate(su.getFileDate())
-                .rawRecordDate(su.getFileDate())
+                .rawRecordDate(su.getRawRecordDate())
                 .build();
 
         return getCollectedDataFromSurveyUnit(su, surveyUnitModel, variablesMap, dataState);

src/main/java/fr/insee/genesis/controller/sources/xml/LunaticXmlDataParser.java

@@ -47,7 +47,7 @@ private Document readXmlFile(Path filePath) throws IOException, SAXException, Ge
         return document;
     }
 
-    private LocalDateTime getFileDate(Path filePath) throws IOException {
+    private LocalDateTime getRawRecordDate(Path filePath) throws IOException {
         BasicFileAttributes attr = Files.readAttributes(filePath, BasicFileAttributes.class);
         return LocalDateTime.ofInstant(attr.lastModifiedTime().toInstant(), ZoneId.of("Europe/Paris"));
     }
@@ -68,9 +68,8 @@ public LunaticXmlCampaign parseDataFile(Path filePath) throws GenesisException,
             if (surveyUnit.getNodeType() == Node.ELEMENT_NODE) {
                 Element surveyUnitElement = (Element) surveyUnit;
                 LunaticXmlSurveyUnit lunaticXmlSurveyUnit = new LunaticXmlSurveyUnit();
-                LocalDateTime rawRecordDate = getFileDate(filePath);
+                LocalDateTime rawRecordDate = getRawRecordDate(filePath);
                 lunaticXmlSurveyUnit.setRawRecordDate(rawRecordDate);
-                lunaticXmlSurveyUnit.setFileDate(rawRecordDate);
                 lunaticXmlSurveyUnit.setId(surveyUnitElement.getElementsByTagName("Id").item(0).getFirstChild().getNodeValue());
                 lunaticXmlSurveyUnit.setQuestionnaireModelId(surveyUnitElement.getElementsByTagName("QuestionnaireModelId").item(0).getFirstChild().getNodeValue());
                 Node data = surveyUnitElement.getElementsByTagName("Data").item(0);

src/main/java/fr/insee/genesis/controller/sources/xml/LunaticXmlDataSequentialParser.java

@@ -29,7 +29,7 @@ public class LunaticXmlDataSequentialParser{
 
 
      public LunaticXmlDataSequentialParser(final Path filePath, final InputStream stream) throws IOException, XMLStreamException {
-         this.rawRecordDate = getFileDate(filePath);
+         this.rawRecordDate = getRawRecordDate(filePath);
 
          XMLInputFactory factory = XMLInputFactory.newInstance();
          factory.setProperty(XMLInputFactory.SUPPORT_DTD, false);
@@ -90,7 +90,6 @@ public LunaticXmlSurveyUnit readNextSurveyUnit() throws XMLStreamException {
      */
     private LunaticXmlSurveyUnit parseSurveyUnit(final XMLEventReader reader) throws XMLStreamException {
         LunaticXmlSurveyUnit xmlSurveyUnit = new LunaticXmlSurveyUnit();
-        xmlSurveyUnit.setFileDate(this.rawRecordDate);
         xmlSurveyUnit.setRawRecordDate(this.rawRecordDate);
 
         LunaticXmlData data = new LunaticXmlData();
@@ -276,7 +275,7 @@ private List<LunaticXmlOtherData> readCalculatedOrExternal(XMLEventReader reader
         return lunaticXmlOtherDataList;
     }
 
-    private LocalDateTime getFileDate(Path filePath) throws IOException {
+    private LocalDateTime getRawRecordDate(Path filePath) throws IOException {
         BasicFileAttributes attr = Files.readAttributes(filePath, BasicFileAttributes.class);
         return LocalDateTime.ofInstant(attr.lastModifiedTime().toInstant(), ZoneId.of("Europe/Paris"));
     }

src/main/java/fr/insee/genesis/domain/converter/rawdata/LunaticJsonRawDataConverter.java

@@ -53,7 +53,6 @@ public List<SurveyUnitModel> convertRawDataAndCollectEmptyModels(
                         .validationDate(payloadParser.getValidationDate(rawData))
                         .isCapturedIndirectly(payloadParser.getIsCapturedIndirectly(rawData))
                         .state(dataState)
-                        .fileDate(rawData.recordDate())
                         .rawRecordDate(rawData.recordDate())
                         .recordDate(Instant.now())
                         .collectedVariables(new ArrayList<>())

src/main/java/fr/insee/genesis/domain/converter/rawdata/RawResponseConverter.java

@@ -97,7 +97,7 @@ private SurveyUnitModel buildSurveyUnitModel(RawResponseModel rawResponseModel,
                 .validationDate(rawResponsePayloadParser.getValidationDate(rawResponseModel))
                 .isCapturedIndirectly(rawResponsePayloadParser.getIsCapturedIndirectly(rawResponseModel))
                 .state(dataState)
-                .fileDate(rawResponseModel.recordDate())
+                .rawRecordDate(rawResponseModel.recordDate())
                 .recordDate(Instant.now())
                 .collectedVariables(new ArrayList<>())
                 .externalVariables(new ArrayList<>())

src/main/java/fr/insee/genesis/domain/utils/DataVerifier.java

@@ -79,7 +79,6 @@ private static SurveyUnitModel createFormattedSurveyUnitModel(
                 .state(DataState.FORMATTED)
                 .mode(sampleSurveyUnitModel.getMode())
                 .recordDate(Instant.now().plusSeconds(1)) // Add 1 second to avoid same recordDate as COLLECTED
-                .fileDate(sampleSurveyUnitModel.getFileDate())
                 .rawRecordDate(sampleSurveyUnitModel.getRawRecordDate())
                 .collectedVariables(new ArrayList<>())
                 .externalVariables(new ArrayList<>())

src/main/java/fr/insee/genesis/infrastructure/mappers/SurveyUnitDocumentMapper.java

@@ -19,6 +19,7 @@ public interface SurveyUnitDocumentMapper {
 	@Mapping(target = "mode", source = "mode", qualifiedByName = "mapMode")
 	SurveyUnitModel documentToModel(SurveyUnitDocument surveyUnit);
 
+    @Mapping(target = "fileDate", ignore = true)
 	SurveyUnitDocument modelToDocument(SurveyUnitModel surveyUnitModel);
 
 	@Mapping(target = "mode", source = "mode", qualifiedByName = "mapMode")
@@ -39,6 +40,10 @@ default void handleDeprecatedFields(SurveyUnitDocument doc,
 			model.setCollectionInstrumentId(doc.getQuestionnaireId());
 		}
 
+        if (model.getRawRecordDate() == null) {
+            model.setRawRecordDate(doc.getFileDate());
+        }
+
 	}
 
 	@Named("mapMode")

src/test/java/fr/insee/genesis/controller/adapter/LunaticXmlAdapterTest.java

@@ -255,14 +255,12 @@ void convert_shouldMapRawRecordDate() {
             // GIVEN
             LocalDateTime rawRecordDate = LocalDateTime.of(2024, 1, 15, 10, 0);
             LunaticXmlSurveyUnit su = buildSurveyUnit(List.of(collectedDataWithValue("VAR1", "val")));
-            su.setFileDate(rawRecordDate);
             su.setRawRecordDate(rawRecordDate);
 
             // WHEN
             SurveyUnitModel collected = getCollected(LunaticXmlAdapter.convert(su, VARIABLES_MAP, MODE));
 
             // THEN
-            assertThat(collected.getFileDate()).isEqualTo(rawRecordDate);
             assertThat(collected.getRawRecordDate()).isEqualTo(rawRecordDate);
         }
     }

src/test/java/fr/insee/genesis/domain/model/surveyunit/SurveyUnitModelTest.java

@@ -55,7 +55,6 @@ void noArgsConstructor_shouldCreateEmptyModel() {
             assertThat(model.getQuestionnaireState()).isNull();
             assertThat(model.getValidationDate()).isNull();
             assertThat(model.getRecordDate()).isNull();
-            assertThat(model.getFileDate()).isNull();
             assertThat(model.getRawRecordDate()).isNull();
             assertThat(model.getCollectedVariables()).isNull();
             assertThat(model.getExternalVariables()).isNull();
@@ -112,7 +111,6 @@ void allArgsConstructor_shouldSetAllFields() {
             assertThat(model.getQuestionnaireState()).isEqualTo(RawResponseDto.QuestionnaireStateEnum.FINISHED);
             assertThat(model.getValidationDate()).isEqualTo(now);
             assertThat(model.getRecordDate()).isEqualTo(now.toInstant(ZoneOffset.UTC));
-            assertThat(model.getFileDate()).isEqualTo(now);
             assertThat(model.getRawRecordDate()).isEqualTo(now);
             assertThat(model.getCollectedVariables()).isEqualTo(collected);
             assertThat(model.getExternalVariables()).isEqualTo(external);
@@ -164,7 +162,6 @@ void builder_shouldSetAllFields() {
             assertThat(model.getQuestionnaireState()).isEqualTo(RawResponseDto.QuestionnaireStateEnum.FINISHED);
             assertThat(model.getValidationDate()).isEqualTo(now);
             assertThat(model.getRecordDate()).isEqualTo(now.toInstant(ZoneOffset.UTC));
-            assertThat(model.getFileDate()).isEqualTo(now);
             assertThat(model.getRawRecordDate()).isEqualTo(now);
             assertThat(model.getCollectedVariables()).isEqualTo(collected);
             assertThat(model.getExternalVariables()).isEqualTo(external);

src/test/java/fr/insee/genesis/infrastructure/mapper/SurveyUnitDocumentMapperImplTest.java

@@ -51,7 +51,6 @@ static void init(){
         surveyUnitDocumentStatic.setInterrogationId(INTERROGATION_ID);
         surveyUnitDocumentStatic.setState("COLLECTED");
         LocalDateTime rawRecordDate = LocalDateTime.of(2023, 1, 1, 0, 0, 0);
-        surveyUnitDocumentStatic.setFileDate(rawRecordDate);
         surveyUnitDocumentStatic.setRawRecordDate(rawRecordDate);
 
         List<VariableDocument> documentExternalVariableList = new ArrayList<>();
@@ -90,7 +89,6 @@ static void init(){
         deprecatedSurveyUnitDocumentStatic.setIdUE(ID_UE);
         deprecatedSurveyUnitDocumentStatic.setInterrogationId(INTERROGATION_ID);
         deprecatedSurveyUnitDocumentStatic.setState("COLLECTED");
-        deprecatedSurveyUnitDocumentStatic.setFileDate(rawRecordDate);
         deprecatedSurveyUnitDocumentStatic.setRawRecordDate(rawRecordDate);
 
         documentExternalVariableList = new ArrayList<>();
@@ -158,7 +156,6 @@ void shouldReturnModelFromDocument(){
         Assertions.assertThat(surveyUnit.getCollectionInstrumentId()).isEqualTo(COLLECTION_INSTRUMENT_ID);
         Assertions.assertThat(surveyUnit.getUsualSurveyUnitId()).isEqualTo(USUAL_SURVEY_UNIT_ID);
         Assertions.assertThat(surveyUnit.getState()).isEqualTo(DataState.COLLECTED);
-        Assertions.assertThat(surveyUnit.getFileDate()).isEqualTo(rawRecordDate);
         Assertions.assertThat(surveyUnit.getRawRecordDate()).isEqualTo(rawRecordDate);
 
         Assertions.assertThat(surveyUnit.getExternalVariables()).filteredOn(externalVariableModel ->
@@ -184,7 +181,6 @@ void shouldReturnModelFromDeprecatedDocument(){
         Assertions.assertThat(surveyUnit.getCollectionInstrumentId()).isEqualTo(QUESTIONNAIRE_ID);
         Assertions.assertThat(surveyUnit.getUsualSurveyUnitId()).isEqualTo(ID_UE);
         Assertions.assertThat(surveyUnit.getState()).isEqualTo(DataState.COLLECTED);
-        Assertions.assertThat(surveyUnit.getFileDate()).isEqualTo(rawRecordDate);
         Assertions.assertThat(surveyUnit.getRawRecordDate()).isEqualTo(rawRecordDate);
 
         Assertions.assertThat(surveyUnit.getExternalVariables()).filteredOn(externalVariableModel ->
@@ -209,7 +205,6 @@ void shouldReturnDocumentFromModel(){
         Assertions.assertThat(surveyUnitDocument.getInterrogationId()).isEqualTo(INTERROGATION_ID);
         Assertions.assertThat(surveyUnitDocument.getCollectionInstrumentId()).isEqualTo(COLLECTION_INSTRUMENT_ID);
         Assertions.assertThat(surveyUnitDocument.getState()).isEqualTo("COLLECTED");
-        Assertions.assertThat(surveyUnitDocument.getFileDate()).isEqualTo(rawRecordDate);
         Assertions.assertThat(surveyUnitDocument.getRawRecordDate()).isEqualTo(rawRecordDate);
 
         Assertions.assertThat(surveyUnitDocument.getExternalVariables()).filteredOn(externalVariableDocument ->
@@ -239,7 +234,6 @@ void shouldReturnModelListFromDocumentList(){
         Assertions.assertThat(surveyUnitList.getFirst().getCollectionInstrumentId()).isEqualTo(COLLECTION_INSTRUMENT_ID);
         Assertions.assertThat(surveyUnitList.getFirst().getUsualSurveyUnitId()).isEqualTo(USUAL_SURVEY_UNIT_ID);
         Assertions.assertThat(surveyUnitList.getFirst().getState()).isEqualTo(DataState.COLLECTED);
-        Assertions.assertThat(surveyUnitList.getFirst().getFileDate()).isEqualTo(rawRecordDate);
         Assertions.assertThat(surveyUnitList.getFirst().getRawRecordDate()).isEqualTo(rawRecordDate);
 
         Assertions.assertThat(surveyUnitList.getFirst().getExternalVariables()).filteredOn(externalVariableModel ->
@@ -267,7 +261,6 @@ void shouldReturnModelListFromDeprecatedDocumentList(){
         Assertions.assertThat(surveyUnitList.getFirst().getCollectionInstrumentId()).isEqualTo(QUESTIONNAIRE_ID);
         Assertions.assertThat(surveyUnitList.getFirst().getUsualSurveyUnitId()).isEqualTo(ID_UE);
         Assertions.assertThat(surveyUnitList.getFirst().getState()).isEqualTo(DataState.COLLECTED);
-        Assertions.assertThat(surveyUnitList.getFirst().getFileDate()).isEqualTo(rawRecordDate);
         Assertions.assertThat(surveyUnitList.getFirst().getRawRecordDate()).isEqualTo(rawRecordDate);
 
         Assertions.assertThat(surveyUnitList.getFirst().getExternalVariables()).filteredOn(externalVariableModel ->
@@ -293,7 +286,6 @@ void shouldReturnDocumentListFromModelList(){
         Assertions.assertThat(surveyUnitDocumentList.getFirst().getInterrogationId()).isEqualTo(INTERROGATION_ID);
         Assertions.assertThat(surveyUnitDocumentList.getFirst().getCollectionInstrumentId()).isEqualTo(COLLECTION_INSTRUMENT_ID);
         Assertions.assertThat(surveyUnitDocumentList.getFirst().getState()).isEqualTo("COLLECTED");
-        Assertions.assertThat(surveyUnitDocumentList.getFirst().getFileDate()).isEqualTo(rawRecordDate);
         Assertions.assertThat(surveyUnitDocumentList.getFirst().getRawRecordDate()).isEqualTo(rawRecordDate);
 
         Assertions.assertThat(surveyUnitDocumentList.getFirst().getExternalVariables()).filteredOn(externalVariableDocument ->