Merge branch 'devNullVariables' of https://github.com/InseeFr/Genesis-API into devNullVariables

Author: Hajarel-moukh

CHANGELOG.md

@@ -1,4 +1,7 @@
 # Changelog
+## 2.6.9 [2026-06-11]
+### Fixed
+- Delete only expired V2 schedules from dataProcessingContext
 ## 2.6.6 [2026-05-26]
 ### Fixed
 - Compound index names

pom.xml

@@ -4,7 +4,7 @@
     <modelVersion>4.0.0</modelVersion>
     <groupId>fr.insee.genesis</groupId>
     <artifactId>genesis-api</artifactId>
-    <version>2.6.7</version>
+    <version>2.6.9</version>
     <packaging>jar</packaging>
     <name>genesis-api</name>
 
@@ -24,14 +24,14 @@
         <jackson-bom.version>2.21.1</jackson-bom.version>
 
         <!-- Dependency versions not managed by Spring -->
-        <bpm.version>1.1.4</bpm.version>
+        <bpm.version>1.1.5</bpm.version>
         <springdoc.version>2.8.17</springdoc.version>
         <mapstruct.version>1.6.3</mapstruct.version>
-        <pitest.version>1.25.1</pitest.version>
+        <pitest.version>1.25.3</pitest.version>
         <pitest.junit.version>1.2.3</pitest.junit.version>
 
         <!-- Sonar properties -->
-        <jacoco.version>0.8.14</jacoco.version>
+        <jacoco.version>0.8.15</jacoco.version>
         <sonar.core.codeCoveragePlugin>jacoco</sonar.core.codeCoveragePlugin>
         <sonar.language>java</sonar.language>
         <sonar.sourceEncoding>UTF-8</sonar.sourceEncoding>
@@ -113,14 +113,14 @@
         <dependency>
             <groupId>net.sf.saxon</groupId>
             <artifactId>Saxon-HE</artifactId>
-            <version>12.9</version>
+            <version>13.0</version>
         </dependency>
 
         <!-- JSON -->
         <dependency>
             <groupId>com.networknt</groupId>
             <artifactId>json-schema-validator</artifactId>
-            <version>3.0.2</version>
+            <version>3.0.3</version>
         </dependency>
 
         <!-- generate implementation auto -->
@@ -164,7 +164,7 @@
                 <plugin>
                     <groupId>org.sonarsource.scanner.maven</groupId>
                     <artifactId>sonar-maven-plugin</artifactId>
-                    <version>5.6.0.6792</version>
+                    <version>5.7.0.6970</version>
                 </plugin>
                 <plugin>
                     <groupId>org.jacoco</groupId>

src/main/java/fr/insee/genesis/controller/adapter/LunaticXmlAdapter.java

@@ -72,8 +72,7 @@ private static SurveyUnitModel getStateDataFromSurveyUnit(LunaticXmlSurveyUnit s
                 .state(dataState)
                 .mode(mode)
                 .recordDate(Instant.now())
-                .fileDate(su.getFileDate())
-                .rawRecordDate(su.getFileDate())
+                .rawRecordDate(su.getRawRecordDate())
                 .build();
 
         return getCollectedDataFromSurveyUnit(su, surveyUnitModel, variablesMap, dataState);

src/main/java/fr/insee/genesis/controller/rest/responses/ResponseController.java

@@ -34,6 +34,7 @@
 import fr.insee.genesis.exceptions.ReviewDisabledException;
 import fr.insee.genesis.infrastructure.utils.FileUtils;
 import fr.insee.modelefiliere.RawResponseDto;
+import io.swagger.v3.oas.annotations.Hidden;
 import io.swagger.v3.oas.annotations.Operation;
 import io.swagger.v3.oas.annotations.Parameter;
 import io.swagger.v3.oas.annotations.media.Schema;
@@ -333,7 +334,7 @@ public ResponseEntity<SurveyUnitSimplifiedDto> getLatestByInterrogationOneObject
     @Operation(summary = "Returns the response with the latest variables for a collectionInstrument, mode and " +
             "interrogation")
     @GetMapping(path = "/{collectionInstrumentId}/{mode}/{interrogationId}")
-    @PreAuthorize("hasRole('USER_KRAFTWERK')")
+    @PreAuthorize("hasRole('ADMIN')")
     public ResponseEntity<SurveyUnitSimplifiedDto> getResponseByCollectionInstrumentAndInterrogation(
             @PathVariable("collectionInstrumentId") String collectionInstrumentId,
             @PathVariable("interrogationId") String interrogationId,
@@ -407,6 +408,7 @@ public ResponseEntity<List<SurveyUnitSimplifiedDto>> getLatestForInterrogationLi
         return ResponseEntity.ok(results);
     }
 
+    //Kraftwerk uses this
     @Operation(summary = "Retrieve responses for a collection instrument and a list of interrogations",
             description = "Return the latest state for each variable for the given interrogationIds and a given collection instrument (formerly questionnaire).<br>" +
                     "For a given id, the endpoint returns a document by collection mode (if there is more than one)<br>" +
@@ -696,6 +698,7 @@ private static String getSuccessMessage(boolean isAnyDataSaved) {
      */
     //TODO Unused for now, reuse code for optimizations, also move it to service
     @Deprecated
+    @Hidden
     @Operation(summary = "Retrieve all responses for a questionnaire and a list of UE",
             description = "Return the latest state for each variable for the given ids and a given questionnaire.<br>" +
                     "For a given id, the endpoint returns a document by collection mode (if there is more than one).")

src/main/java/fr/insee/genesis/controller/sources/xml/LunaticXmlDataParser.java

@@ -47,7 +47,7 @@ private Document readXmlFile(Path filePath) throws IOException, SAXException, Ge
         return document;
     }
 
-    private LocalDateTime getFileDate(Path filePath) throws IOException {
+    private LocalDateTime getRawRecordDate(Path filePath) throws IOException {
         BasicFileAttributes attr = Files.readAttributes(filePath, BasicFileAttributes.class);
         return LocalDateTime.ofInstant(attr.lastModifiedTime().toInstant(), ZoneId.of("Europe/Paris"));
     }
@@ -68,9 +68,8 @@ public LunaticXmlCampaign parseDataFile(Path filePath) throws GenesisException,
             if (surveyUnit.getNodeType() == Node.ELEMENT_NODE) {
                 Element surveyUnitElement = (Element) surveyUnit;
                 LunaticXmlSurveyUnit lunaticXmlSurveyUnit = new LunaticXmlSurveyUnit();
-                LocalDateTime rawRecordDate = getFileDate(filePath);
+                LocalDateTime rawRecordDate = getRawRecordDate(filePath);
                 lunaticXmlSurveyUnit.setRawRecordDate(rawRecordDate);
-                lunaticXmlSurveyUnit.setFileDate(rawRecordDate);
                 lunaticXmlSurveyUnit.setId(surveyUnitElement.getElementsByTagName("Id").item(0).getFirstChild().getNodeValue());
                 lunaticXmlSurveyUnit.setQuestionnaireModelId(surveyUnitElement.getElementsByTagName("QuestionnaireModelId").item(0).getFirstChild().getNodeValue());
                 Node data = surveyUnitElement.getElementsByTagName("Data").item(0);

src/main/java/fr/insee/genesis/controller/sources/xml/LunaticXmlDataSequentialParser.java

@@ -29,7 +29,7 @@ public class LunaticXmlDataSequentialParser{
 
 
      public LunaticXmlDataSequentialParser(final Path filePath, final InputStream stream) throws IOException, XMLStreamException {
-         this.rawRecordDate = getFileDate(filePath);
+         this.rawRecordDate = getRawRecordDate(filePath);
 
          XMLInputFactory factory = XMLInputFactory.newInstance();
          factory.setProperty(XMLInputFactory.SUPPORT_DTD, false);
@@ -90,7 +90,6 @@ public LunaticXmlSurveyUnit readNextSurveyUnit() throws XMLStreamException {
      */
     private LunaticXmlSurveyUnit parseSurveyUnit(final XMLEventReader reader) throws XMLStreamException {
         LunaticXmlSurveyUnit xmlSurveyUnit = new LunaticXmlSurveyUnit();
-        xmlSurveyUnit.setFileDate(this.rawRecordDate);
         xmlSurveyUnit.setRawRecordDate(this.rawRecordDate);
 
         LunaticXmlData data = new LunaticXmlData();
@@ -276,7 +275,7 @@ private List<LunaticXmlOtherData> readCalculatedOrExternal(XMLEventReader reader
         return lunaticXmlOtherDataList;
     }
 
-    private LocalDateTime getFileDate(Path filePath) throws IOException {
+    private LocalDateTime getRawRecordDate(Path filePath) throws IOException {
         BasicFileAttributes attr = Files.readAttributes(filePath, BasicFileAttributes.class);
         return LocalDateTime.ofInstant(attr.lastModifiedTime().toInstant(), ZoneId.of("Europe/Paris"));
     }

src/main/java/fr/insee/genesis/domain/converter/rawdata/LunaticJsonRawDataConverter.java

@@ -73,7 +73,6 @@ public List<SurveyUnitModel> convertRawDataAndCollectEmptyModels(
                         .validationDate(payloadParser.getValidationDate(rawData))
                         .isCapturedIndirectly(payloadParser.getIsCapturedIndirectly(rawData))
                         .state(dataState)
-                        .fileDate(rawData.recordDate())
                         .rawRecordDate(rawData.recordDate())
                         .recordDate(Instant.now())
                         .collectedVariables(new ArrayList<>())

src/main/java/fr/insee/genesis/domain/converter/rawdata/RawDataConverter.java

@@ -12,8 +12,8 @@
 import fr.insee.genesis.domain.utils.JsonUtils;
 import jakarta.validation.constraints.NotNull;
 import lombok.extern.slf4j.Slf4j;
+import org.jspecify.annotations.Nullable;
 import org.springframework.beans.factory.annotation.Autowired;
-import org.springframework.lang.Nullable;
 import org.springframework.stereotype.Component;
 
 import java.math.BigDecimal;
@@ -268,10 +268,14 @@ protected void convertCollectedVariables(
     ) {
         //Get data map from payload
         Map<String, Object> dataMap = dataOrPayloadMap;
-        if (rawDataModelType == RawDataModelType.FILIERE) {
+        if (rawDataModelType == RawDataModelType.FILIERE && dataOrPayloadMap.containsKey("data")) {
             dataMap = JsonUtils.asMap(dataOrPayloadMap.get("data"));
         }
-        Map<String, Object> collectedMap  = new HashMap<>(JsonUtils.asMap(dataMap.get("COLLECTED")));
+
+        Map<String, Object> collectedMap = new HashMap<>();
+        if(dataMap.containsKey("COLLECTED")) {
+            collectedMap = new HashMap<>(JsonUtils.asMap(dataMap.get("COLLECTED")));
+        }
 
         if(lastSurveyUnitModel != null && lastSurveyUnitModel.getState().equals(dataState)) {
             fillRawDataMapWithAbsentVariables(lastSurveyUnitModel, collectedMap, dataState);
@@ -491,18 +495,18 @@ protected void convertExternalVariables(
     ) {
         //Get data map from payload if filiere model
         Map<String, Object> dataMap = dataOrPayloadMap;
-        if (rawDataModelType == RawDataModelType.FILIERE) {
+        if (rawDataModelType == RawDataModelType.FILIERE && dataOrPayloadMap.containsKey("data")) {
             dataMap = JsonUtils.asMap(dataOrPayloadMap.get("data"));
         }
-        Map<String, Object> externalMap = null;
+        Map<String, Object> externalMap = new HashMap<>();
         if(dataMap.containsKey("EXTERNAL")) {
             externalMap = new HashMap<>(JsonUtils.asMap(dataMap.get("EXTERNAL")));
         }
 
         if(lastSurveyUnitModel != null){
             fillRawDataMapWithAbsentVariables(lastSurveyUnitModel, externalMap, null);
         }
-        if(externalMap == null){
+        if(externalMap.isEmpty()){
             return;
         }
 

src/main/java/fr/insee/genesis/domain/converter/rawdata/RawResponseRawDataConverter.java

@@ -7,7 +7,6 @@
 import fr.insee.genesis.domain.model.surveyunit.rawdata.RawResponseModel;
 import fr.insee.genesis.domain.parser.rawdata.RawResponsePayloadParser;
 import fr.insee.genesis.domain.ports.api.SurveyUnitApiPort;
-import fr.insee.genesis.domain.service.surveyunit.SurveyUnitService;
 import fr.insee.modelefiliere.RawResponseDto;
 import lombok.extern.slf4j.Slf4j;
 import org.springframework.stereotype.Component;
@@ -141,7 +140,7 @@ private SurveyUnitModel buildSurveyUnitModel(RawResponseModel rawResponseModel,
                 .validationDate(rawResponsePayloadParser.getValidationDate(rawResponseModel))
                 .isCapturedIndirectly(rawResponsePayloadParser.getIsCapturedIndirectly(rawResponseModel))
                 .state(dataState)
-                .fileDate(rawResponseModel.recordDate())
+                .rawRecordDate(rawResponseModel.recordDate())
                 .recordDate(Instant.now())
                 .collectedVariables(new ArrayList<>())
                 .externalVariables(new ArrayList<>())

src/main/java/fr/insee/genesis/domain/model/context/schedule/DeletedExpiredSchedules.java

@@ -0,0 +1,12 @@
+package fr.insee.genesis.domain.model.context.schedule;
+
+import java.util.List;
+
+public record DeletedExpiredSchedules(
+        List<KraftwerkExecutionSchedule> v1Schedules,
+        List<KraftwerkExecutionScheduleV2> v2Schedules
+) {
+    public boolean isEmpty() {
+        return v1Schedules.isEmpty() && v2Schedules.isEmpty();
+    }
+}