New endpoint that exposes the list of processed idInterrogations in last hours by questionnaire

src/main/java/fr/insee/genesis/controller/rest/responses/InterrogationController.java

@@ -1,6 +1,6 @@
 package fr.insee.genesis.controller.rest.responses;
 
-import fr.insee.genesis.controller.dto.InterrogationId;
+import fr.insee.genesis.domain.model.surveyunit.InterrogationId;
 import fr.insee.genesis.controller.rest.CommonApiResponse;
 import fr.insee.genesis.domain.ports.api.SurveyUnitApiPort;
 import io.swagger.v3.oas.annotations.Operation;

src/main/java/fr/insee/genesis/controller/rest/responses/RawResponseController.java

@@ -180,6 +180,17 @@ public ResponseEntity<String> processJsonRawData(
         }
     }
 
+    @Operation(summary = "Get processed data ids from last n hours (default 24h)")
+    @GetMapping(path = "/lunatic-json/processed/ids")
+    @PreAuthorize("hasRole('ADMIN')")
+    public ResponseEntity<Map<String, List<String>>> getProcessedDataIdsSinceHours(
+            @RequestParam("questionnaireId") String questionnaireId,
+            @RequestParam(name = "sinceHours", defaultValue = "24") int hours
+    ) {
+        log.info("Retrieve ids of data processed in last {}h", hours);
+        Map<String, List<String>> result = lunaticJsonRawDataApiPort.findProcessedIdsgroupedByQuestionnaireSince(LocalDateTime.now().minusHours(hours).minusMinutes(10));
+        return ResponseEntity.ok(result);
+    }
     @Operation(summary = "Get lunatic JSON data from one campaign in Genesis Database, filtered by optional start and end dates")
     @GetMapping(path = "/lunatic-json/{campaignId}")
     @PreAuthorize("hasRole('USER_KRAFTWERK')")

src/main/java/fr/insee/genesis/controller/rest/responses/ResponseController.java

@@ -6,7 +6,7 @@
 import fr.insee.bpm.metadata.reader.lunatic.LunaticReader;
 import fr.insee.genesis.Constants;
 import fr.insee.genesis.controller.adapter.LunaticXmlAdapter;
-import fr.insee.genesis.controller.dto.InterrogationId;
+import fr.insee.genesis.domain.model.surveyunit.InterrogationId;
 import fr.insee.genesis.controller.dto.SurveyUnitDto;
 import fr.insee.genesis.controller.dto.SurveyUnitInputDto;
 import fr.insee.genesis.controller.dto.SurveyUnitQualityToolDto;

src/main/java/fr/insee/genesis/domain/model/surveyunit/GroupedInterrogation.java

@@ -0,0 +1,13 @@
+package fr.insee.genesis.domain.model.surveyunit;
+
+import lombok.Builder;
+
+import java.util.List;
+
+
+@Builder
+public record GroupedInterrogation(
+        String questionnaireId,
+        String partitionOrCampaignId,
+        List<String> interrogationIds
+){}

src/main/java/fr/insee/genesis/controller/dto/InterrogationId.java → src/main/java/fr/insee/genesis/domain/model/surveyunit/InterrogationId.java

@@ -1,4 +1,4 @@
-package fr.insee.genesis.controller.dto;
+package fr.insee.genesis.domain.model.surveyunit;
 
 import lombok.AllArgsConstructor;
 import lombok.NoArgsConstructor;

src/main/java/fr/insee/genesis/domain/ports/api/DataProcessingContextApiPort.java

@@ -32,4 +32,5 @@ void saveKraftwerkExecutionSchedule(String partitionId,
 
     DataProcessingContextModel getContext(String interrogationId) throws GenesisException;
     DataProcessingContextModel getContextByPartitionId(String partitionId) throws GenesisException;
+
 }

src/main/java/fr/insee/genesis/domain/ports/api/LunaticJsonRawDataApiPort.java

@@ -11,8 +11,10 @@
 import org.springframework.data.domain.Page;
 import org.springframework.data.domain.Pageable;
 
+import java.time.LocalDateTime;
 import java.time.Instant;
 import java.util.List;
+import java.util.Map;
 import java.util.Set;
 
 public interface LunaticJsonRawDataApiPort {
@@ -27,4 +29,5 @@ public interface LunaticJsonRawDataApiPort {
     Page<LunaticJsonRawDataModel> findRawDataByCampaignIdAndDate(String campaignId, Instant  startDt, Instant endDt, Pageable pageable);
 
     DataProcessResult processRawData(String campaignName, List<String> interrogationIdList, List<GenesisError> errors) throws GenesisException;
+    Map<String, List<String>> findProcessedIdsgroupedByQuestionnaireSince(LocalDateTime since);
 }

src/main/java/fr/insee/genesis/domain/ports/api/SurveyUnitApiPort.java

@@ -2,7 +2,7 @@
 
 import fr.insee.bpm.metadata.model.VariablesMap;
 import fr.insee.genesis.controller.dto.CampaignWithQuestionnaire;
-import fr.insee.genesis.controller.dto.InterrogationId;
+import fr.insee.genesis.domain.model.surveyunit.InterrogationId;
 import fr.insee.genesis.controller.dto.QuestionnaireWithCampaign;
 import fr.insee.genesis.controller.dto.SurveyUnitDto;
 import fr.insee.genesis.controller.dto.SurveyUnitInputDto;

src/main/java/fr/insee/genesis/domain/ports/spi/DataProcessingContextPersistancePort.java

@@ -9,6 +9,8 @@
 public interface DataProcessingContextPersistancePort {
     DataProcessingContextDocument findByPartitionId(String partitionId);
 
+    List<DataProcessingContextModel> findByPartitionIds(List<String> partitionIds);
+
     void save(DataProcessingContextDocument dataProcessingContextDocument);
 
     void saveAll(List<DataProcessingContextDocument> dataProcessingContextDocuments);

src/main/java/fr/insee/genesis/domain/ports/spi/LunaticJsonRawDataPersistencePort.java

@@ -1,10 +1,12 @@
 package fr.insee.genesis.domain.ports.spi;
 
+import fr.insee.genesis.domain.model.surveyunit.GroupedInterrogation;
 import fr.insee.genesis.domain.model.surveyunit.Mode;
 import fr.insee.genesis.domain.model.surveyunit.rawdata.LunaticJsonRawDataModel;
 import org.springframework.data.domain.Page;
 import org.springframework.data.domain.Pageable;
 
+import java.time.LocalDateTime;
 import java.time.Instant;
 import java.util.List;
 import java.util.Set;
@@ -18,4 +20,5 @@ public interface LunaticJsonRawDataPersistencePort {
     Set<String> findDistinctQuestionnaireIds();
     Page<LunaticJsonRawDataModel> findByCampaignIdAndDate(String campaignId, Instant startDt, Instant endDt, Pageable pageable);
     long countResponsesByQuestionnaireId(String questionnaireId);
+    List<GroupedInterrogation> findProcessedIdsGroupedByQuestionnaireSince(LocalDateTime since);
 }

src/main/java/fr/insee/genesis/domain/service/context/DataProcessingContextService.java

@@ -172,4 +172,5 @@ public DataProcessingContextModel getContextByPartitionId(String partitionId){
                 dataProcessingContextPersistancePort.findByPartitionId(partitionId)
         );
     }
+
 }

src/main/java/fr/insee/genesis/domain/service/rawdata/LunaticJsonRawDataService.java

@@ -8,13 +8,16 @@
 import fr.insee.genesis.controller.utils.ControllerUtils;
 import fr.insee.genesis.domain.model.context.DataProcessingContextModel;
 import fr.insee.genesis.domain.model.surveyunit.DataState;
+import fr.insee.genesis.domain.model.surveyunit.GroupedInterrogation;
+import fr.insee.genesis.domain.model.surveyunit.InterrogationId;
 import fr.insee.genesis.domain.model.surveyunit.Mode;
 import fr.insee.genesis.domain.model.surveyunit.SurveyUnitModel;
 import fr.insee.genesis.domain.model.surveyunit.VariableModel;
 import fr.insee.genesis.domain.model.surveyunit.rawdata.DataProcessResult;
 import fr.insee.genesis.domain.model.surveyunit.rawdata.LunaticJsonRawDataModel;
 import fr.insee.genesis.domain.model.surveyunit.rawdata.RawDataModelType;
 import fr.insee.genesis.domain.ports.api.LunaticJsonRawDataApiPort;
+import fr.insee.genesis.domain.ports.spi.DataProcessingContextPersistancePort;
 import fr.insee.genesis.domain.ports.spi.LunaticJsonRawDataPersistencePort;
 import fr.insee.genesis.domain.ports.spi.SurveyUnitQualityToolPort;
 import fr.insee.genesis.domain.service.context.DataProcessingContextService;
@@ -42,6 +45,7 @@
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
+import java.util.stream.Collectors;
 
 @Service
 @Slf4j
@@ -58,6 +62,8 @@ public class LunaticJsonRawDataService implements LunaticJsonRawDataApiPort {
 
     @Qualifier("lunaticJsonMongoAdapterNew")
     private final LunaticJsonRawDataPersistencePort lunaticJsonRawDataPersistencePort;
+    @Qualifier("dataProcessingContextMongoAdapter")
+    private final DataProcessingContextPersistancePort dataProcessingContextPersistancePort;
 
     @Autowired
     public LunaticJsonRawDataService(LunaticJsonRawDataPersistencePort lunaticJsonRawDataNewPersistencePort,
@@ -68,14 +74,16 @@ public LunaticJsonRawDataService(LunaticJsonRawDataPersistencePort lunaticJsonRa
                                      FileUtils fileUtils,
                                      DataProcessingContextService dataProcessingContextService,
                                      SurveyUnitQualityToolPort surveyUnitQualityToolPort,
-                                     Config config
+                                     Config config,
+                                     DataProcessingContextPersistancePort dataProcessingContextPersistancePort
     ) {
         this.controllerUtils = controllerUtils;
         this.metadataService = metadataService;
         this.surveyUnitService = surveyUnitService;
         this.surveyUnitQualityService = surveyUnitQualityService;
         this.fileUtils = fileUtils;
         this.lunaticJsonRawDataPersistencePort = lunaticJsonRawDataNewPersistencePort;
+        this.dataProcessingContextPersistancePort = dataProcessingContextPersistancePort;
         this.surveyUnitQualityToolPort = surveyUnitQualityToolPort;
         this.dataProcessingContextService = dataProcessingContextService;
         this.config = config;
@@ -391,6 +399,18 @@ public long countResponsesByQuestionnaireId(String campaignId) {
         return lunaticJsonRawDataPersistencePort.countResponsesByQuestionnaireId(campaignId);
     }
 
+    @Override
+    public Map<String, List<String>> findProcessedIdsgroupedByQuestionnaireSince(LocalDateTime since) {
+        List<GroupedInterrogation> idsByQuestionnaire = lunaticJsonRawDataPersistencePort.findProcessedIdsGroupedByQuestionnaireSince(since);
+        List<String> partitionIds = idsByQuestionnaire.stream().map(GroupedInterrogation::partitionOrCampaignId).toList();
+        List<DataProcessingContextModel> contexts = dataProcessingContextPersistancePort.findByPartitionIds(partitionIds);
+        List<String> partitionIdsWithReview = contexts.stream().filter(DataProcessingContextModel::isWithReview).map(DataProcessingContextModel::getPartitionId).toList();
+        return idsByQuestionnaire.stream().filter(groupedInterrogation -> partitionIdsWithReview.contains(groupedInterrogation.partitionOrCampaignId()))
+                .collect(Collectors.toMap(
+                GroupedInterrogation::questionnaireId,
+                GroupedInterrogation::interrogationIds
+        ));
+    }
 
     @Override
     public Page<LunaticJsonRawDataModel> findRawDataByCampaignIdAndDate(String campaignId, Instant startDt, Instant endDt, Pageable pageable){

src/main/java/fr/insee/genesis/domain/service/surveyunit/SurveyUnitService.java

@@ -2,7 +2,7 @@
 
 import fr.insee.bpm.metadata.model.VariablesMap;
 import fr.insee.genesis.controller.dto.CampaignWithQuestionnaire;
-import fr.insee.genesis.controller.dto.InterrogationId;
+import fr.insee.genesis.domain.model.surveyunit.InterrogationId;
 import fr.insee.genesis.controller.dto.QuestionnaireWithCampaign;
 import fr.insee.genesis.controller.dto.SurveyUnitDto;
 import fr.insee.genesis.controller.dto.SurveyUnitInputDto;

src/main/java/fr/insee/genesis/infrastructure/adapter/DataProcessingContextMongoAdapter.java

@@ -5,6 +5,7 @@
 import fr.insee.genesis.domain.model.context.schedule.KraftwerkExecutionSchedule;
 import fr.insee.genesis.domain.ports.spi.DataProcessingContextPersistancePort;
 import fr.insee.genesis.infrastructure.document.context.DataProcessingContextDocument;
+import fr.insee.genesis.infrastructure.mappers.DataProcessingContextMapper;
 import fr.insee.genesis.infrastructure.repository.DataProcessingContextMongoDBRepository;
 import fr.insee.genesis.infrastructure.utils.context.ContextDedupUtils;
 import lombok.extern.slf4j.Slf4j;
@@ -41,6 +42,13 @@ public DataProcessingContextDocument findByPartitionId(String partitionId) {
         return ContextDedupUtils.deduplicateContexts(partitionId, existingDocuments);
     }
 
+    @Override
+    public List<DataProcessingContextModel> findByPartitionIds(List<String> partitionIds){
+        List<DataProcessingContextDocument> existingDocuments =
+                dataProcessingContextMongoDBRepository.findByPartitionIdList(partitionIds);
+        return DataProcessingContextMapper.INSTANCE.listDocumentToListModel(ContextDedupUtils.deduplicateContexts(existingDocuments));
+    }
+
     @Override
     public void save(DataProcessingContextDocument dataProcessingContextDocument) {
         dataProcessingContextMongoDBRepository.save(dataProcessingContextDocument);

src/main/java/fr/insee/genesis/infrastructure/adapter/LunaticJsonRawDataMongoAdapter.java

@@ -1,10 +1,13 @@
 package fr.insee.genesis.infrastructure.adapter;
 
 import fr.insee.genesis.Constants;
+import fr.insee.genesis.domain.model.surveyunit.GroupedInterrogation;
+import fr.insee.genesis.domain.model.surveyunit.InterrogationId;
 import fr.insee.genesis.domain.model.surveyunit.Mode;
 import fr.insee.genesis.domain.model.surveyunit.rawdata.LunaticJsonRawDataModel;
 import fr.insee.genesis.domain.ports.spi.LunaticJsonRawDataPersistencePort;
 import fr.insee.genesis.infrastructure.document.rawdata.LunaticJsonRawDataDocument;
+import fr.insee.genesis.infrastructure.mappers.GroupedInterrogationDocumentMapper;
 import fr.insee.genesis.infrastructure.mappers.LunaticJsonRawDataDocumentMapper;
 import fr.insee.genesis.infrastructure.repository.LunaticJsonMongoDBRepository;
 import lombok.extern.slf4j.Slf4j;
@@ -83,4 +86,11 @@ public Page<LunaticJsonRawDataModel> findByCampaignIdAndDate(String campaignId,
     public long countResponsesByQuestionnaireId(String questionnaireId) {
         return repository.countByQuestionnaireId(questionnaireId);
     }
+
+    @Override
+    public List<GroupedInterrogation> findProcessedIdsGroupedByQuestionnaireSince(LocalDateTime since){
+        return GroupedInterrogationDocumentMapper.INSTANCE.listDocumentToListModel(repository.aggregateRawGrouped(since));
+    }
+
+
 }

src/main/java/fr/insee/genesis/infrastructure/document/rawdata/LunaticJsonRawDataDocument.java

@@ -4,6 +4,8 @@
 import lombok.Builder;
 import org.bson.types.ObjectId;
 import org.springframework.data.annotation.Id;
+import org.springframework.data.mongodb.core.index.IndexDirection;
+import org.springframework.data.mongodb.core.index.Indexed;
 import org.springframework.data.mongodb.core.mapping.Document;
 
 import java.time.LocalDateTime;
@@ -21,5 +23,6 @@ public record LunaticJsonRawDataDocument(
         Mode mode,
         Map<String,Object> data,
         LocalDateTime recordDate,
+        @Indexed(direction = IndexDirection.DESCENDING)
         LocalDateTime processDate
 ){}

src/main/java/fr/insee/genesis/infrastructure/document/surveyunit/GroupedInterrogationDocument.java

@@ -0,0 +1,13 @@
+package fr.insee.genesis.infrastructure.document.surveyunit;
+
+import lombok.Data;
+
+import java.util.List;
+
+@Data
+public class GroupedInterrogationDocument {
+
+    private String questionnaireId;
+    private String partitionOrCampaignId;
+    private List<String> interrogationIds;
+}

src/main/java/fr/insee/genesis/infrastructure/mappers/GroupedInterrogationDocumentMapper.java

@@ -0,0 +1,23 @@
+package fr.insee.genesis.infrastructure.mappers;
+
+import fr.insee.genesis.domain.model.surveyunit.GroupedInterrogation;
+import fr.insee.genesis.infrastructure.document.surveyunit.GroupedInterrogationDocument;
+import org.mapstruct.Mapper;
+import org.mapstruct.factory.Mappers;
+
+import java.util.List;
+
+@Mapper
+public interface GroupedInterrogationDocumentMapper {
+
+
+    GroupedInterrogationDocumentMapper INSTANCE = Mappers.getMapper(GroupedInterrogationDocumentMapper.class);
+
+    GroupedInterrogation documentToModel(GroupedInterrogationDocument interrogationIdDoc);
+
+    GroupedInterrogationDocument modelToDocument(GroupedInterrogation interrogationId);
+
+    List<GroupedInterrogation> listDocumentToListModel(List<GroupedInterrogationDocument> interrogationIdDocList);
+
+    List<GroupedInterrogationDocument> listModelToListDocument(List<GroupedInterrogation> interrogationIdList);
+}

src/main/java/fr/insee/genesis/infrastructure/repository/LunaticJsonMongoDBRepository.java

@@ -2,12 +2,15 @@
 
 import fr.insee.genesis.domain.model.surveyunit.Mode;
 import fr.insee.genesis.infrastructure.document.rawdata.LunaticJsonRawDataDocument;
+import fr.insee.genesis.infrastructure.document.surveyunit.GroupedInterrogationDocument;
+import org.springframework.data.mongodb.repository.Aggregation;
 import org.springframework.data.domain.Page;
 import org.springframework.data.domain.Pageable;
 import org.springframework.data.mongodb.repository.MongoRepository;
 import org.springframework.data.mongodb.repository.Query;
 import org.springframework.stereotype.Repository;
 
+import java.time.LocalDateTime;
 import java.time.Instant;
 import java.util.List;
 
@@ -25,6 +28,22 @@ public interface LunaticJsonMongoDBRepository extends MongoRepository<LunaticJso
 
     Page<LunaticJsonRawDataDocument> findByCampaignIdAndRecordDateBetween(String campagneId, Instant start, Instant  end, Pageable pageable);
     long countByQuestionnaireId(String questionnaireId);
-
+    @Aggregation(pipeline = {
+            "{ '$match': { 'processDate': { '$gte': ?0 } } }",
+            "{ '$group': { " +
+                    "'_id': { " +
+                    "'questionnaireId': '$questionnaireId', " +
+                    "'partitionOrCampaignId': { '$ifNull': ['$partitionId', '$campaignId'] } " +
+                    "}, " +
+                    "'interrogationIds': { '$addToSet': '$interrogationId' } " +
+                    "} }",
+            "{ '$project': { " +
+                    "'questionnaireId': '$_id.questionnaireId', " +
+                    "'partitionOrCampaignId': '$_id.partitionOrCampaignId', " +
+                    "'interrogationIds': 1, " +
+                    "'_id': 0 " +
+                    "} }"
+    })
+    List<GroupedInterrogationDocument> aggregateRawGrouped(LocalDateTime since);
 
 }

src/test/java/cucumber/functional_tests/RawDataDefinitions.java

@@ -63,6 +63,7 @@ public class RawDataDefinitions {
     LunaticJsonRawDataPersistanceStub lunaticJsonRawDataPersistanceStub = new LunaticJsonRawDataPersistanceStub();
     MetadataService metadataService = new MetadataService();
     SurveyUnitPersistencePortStub surveyUnitPersistencePortStub = new SurveyUnitPersistencePortStub();
+    DataProcessingContextPersistancePortStub contextStub = new DataProcessingContextPersistancePortStub();
     Config config = new ConfigStub();
     FileUtils fileUtils = new FileUtils(config);
     SurveyUnitService surveyUnitService = new SurveyUnitService(surveyUnitPersistencePortStub, metadataService, fileUtils);
@@ -83,7 +84,9 @@ public class RawDataDefinitions {
                             dataProcessingContextPersistancePortStub,
                             surveyUnitPersistencePortStub),
                     surveyUnitQualityToolPerretAdapterStub,
-                    config);
+                    config,
+                    dataProcessingContextPersistancePortStub);
+
     RawResponseController rawResponseController = new RawResponseController(
             lunaticJsonRawDataService
     );

src/test/java/fr/insee/genesis/controller/rest/UtilsControllerTest.java

@@ -2,7 +2,7 @@
 
 import cucumber.TestConstants;
 import fr.insee.genesis.Constants;
-import fr.insee.genesis.controller.dto.InterrogationId;
+import fr.insee.genesis.domain.model.surveyunit.InterrogationId;
 import fr.insee.genesis.controller.services.MetadataService;
 import fr.insee.genesis.controller.utils.ControllerUtils;
 import fr.insee.genesis.domain.model.surveyunit.DataState;
@@ -11,6 +11,7 @@
 import fr.insee.genesis.domain.model.surveyunit.VariableModel;
 import fr.insee.genesis.domain.ports.api.LunaticJsonRawDataApiPort;
 import fr.insee.genesis.domain.ports.api.SurveyUnitApiPort;
+import fr.insee.genesis.domain.ports.spi.DataProcessingContextPersistancePort;
 import fr.insee.genesis.domain.service.context.DataProcessingContextService;
 import fr.insee.genesis.domain.service.rawdata.LunaticJsonRawDataService;
 import fr.insee.genesis.domain.service.surveyunit.SurveyUnitQualityService;
@@ -46,6 +47,7 @@ class UtilsControllerTest {
     static UtilsController utilsControllerStatic;
     static SurveyUnitPersistencePortStub surveyUnitPersistencePortStub;
     static LunaticJsonRawDataPersistanceStub lunaticJsonRawDataPersistencePort;
+    static DataProcessingContextPersistancePort contextStub = new DataProcessingContextPersistancePortStub();
     static SurveyUnitQualityToolPerretAdapterStub surveyUnitQualityToolPerretAdapterStub;
     static List<InterrogationId> interrogationIdList;
     static FileUtils fileUtils = new FileUtils(new ConfigStub());
@@ -73,7 +75,9 @@ static void init() {
                                 new DataProcessingContextPersistancePortStub(),
                                 surveyUnitPersistencePortStub),
                         surveyUnitQualityToolPerretAdapterStub,
-                        new ConfigStub());
+                        new ConfigStub(),
+                        contextStub);
+
 
         utilsControllerStatic = new UtilsController(
                 surveyUnitApiPort