diff --git a/src/main/java/fr/insee/genesis/controller/rest/responses/InterrogationController.java b/src/main/java/fr/insee/genesis/controller/rest/responses/InterrogationController.java index c3eda664a..a95809886 100644 --- a/src/main/java/fr/insee/genesis/controller/rest/responses/InterrogationController.java +++ b/src/main/java/fr/insee/genesis/controller/rest/responses/InterrogationController.java @@ -1,6 +1,6 @@ package fr.insee.genesis.controller.rest.responses; -import fr.insee.genesis.controller.dto.InterrogationId; +import fr.insee.genesis.domain.model.surveyunit.InterrogationId; import fr.insee.genesis.controller.rest.CommonApiResponse; import fr.insee.genesis.domain.ports.api.SurveyUnitApiPort; import io.swagger.v3.oas.annotations.Operation; diff --git a/src/main/java/fr/insee/genesis/controller/rest/responses/RawResponseController.java b/src/main/java/fr/insee/genesis/controller/rest/responses/RawResponseController.java index 16bcb55ed..ce435b89a 100644 --- a/src/main/java/fr/insee/genesis/controller/rest/responses/RawResponseController.java +++ b/src/main/java/fr/insee/genesis/controller/rest/responses/RawResponseController.java @@ -180,6 +180,17 @@ public ResponseEntity processJsonRawData( } } + @Operation(summary = "Get processed data ids from last n hours (default 24h)") + @GetMapping(path = "/lunatic-json/processed/ids") + @PreAuthorize("hasRole('ADMIN')") + public ResponseEntity>> getProcessedDataIdsSinceHours( + @RequestParam("questionnaireId") String questionnaireId, + @RequestParam(name = "sinceHours", defaultValue = "24") int hours + ) { + log.info("Retrieve ids of data processed in last {}h", hours); + Map> result = lunaticJsonRawDataApiPort.findProcessedIdsgroupedByQuestionnaireSince(LocalDateTime.now().minusHours(hours).minusMinutes(10)); + return ResponseEntity.ok(result); + } @Operation(summary = "Get lunatic JSON data from one campaign in Genesis Database, filtered by optional start and end dates") @GetMapping(path = "/lunatic-json/{campaignId}") @PreAuthorize("hasRole('USER_KRAFTWERK')") diff --git a/src/main/java/fr/insee/genesis/controller/rest/responses/ResponseController.java b/src/main/java/fr/insee/genesis/controller/rest/responses/ResponseController.java index 25979c0d3..5a490cca2 100644 --- a/src/main/java/fr/insee/genesis/controller/rest/responses/ResponseController.java +++ b/src/main/java/fr/insee/genesis/controller/rest/responses/ResponseController.java @@ -6,7 +6,7 @@ import fr.insee.bpm.metadata.reader.lunatic.LunaticReader; import fr.insee.genesis.Constants; import fr.insee.genesis.controller.adapter.LunaticXmlAdapter; -import fr.insee.genesis.controller.dto.InterrogationId; +import fr.insee.genesis.domain.model.surveyunit.InterrogationId; import fr.insee.genesis.controller.dto.SurveyUnitDto; import fr.insee.genesis.controller.dto.SurveyUnitInputDto; import fr.insee.genesis.controller.dto.SurveyUnitQualityToolDto; diff --git a/src/main/java/fr/insee/genesis/domain/model/surveyunit/GroupedInterrogation.java b/src/main/java/fr/insee/genesis/domain/model/surveyunit/GroupedInterrogation.java new file mode 100644 index 000000000..8f5535100 --- /dev/null +++ b/src/main/java/fr/insee/genesis/domain/model/surveyunit/GroupedInterrogation.java @@ -0,0 +1,13 @@ +package fr.insee.genesis.domain.model.surveyunit; + +import lombok.Builder; + +import java.util.List; + + +@Builder +public record GroupedInterrogation( + String questionnaireId, + String partitionOrCampaignId, + List interrogationIds +){} diff --git a/src/main/java/fr/insee/genesis/controller/dto/InterrogationId.java b/src/main/java/fr/insee/genesis/domain/model/surveyunit/InterrogationId.java similarity index 93% rename from src/main/java/fr/insee/genesis/controller/dto/InterrogationId.java rename to src/main/java/fr/insee/genesis/domain/model/surveyunit/InterrogationId.java index 33c6e9f69..c280f4933 100644 --- a/src/main/java/fr/insee/genesis/controller/dto/InterrogationId.java +++ b/src/main/java/fr/insee/genesis/domain/model/surveyunit/InterrogationId.java @@ -1,4 +1,4 @@ -package fr.insee.genesis.controller.dto; +package fr.insee.genesis.domain.model.surveyunit; import lombok.AllArgsConstructor; import lombok.NoArgsConstructor; diff --git a/src/main/java/fr/insee/genesis/domain/ports/api/DataProcessingContextApiPort.java b/src/main/java/fr/insee/genesis/domain/ports/api/DataProcessingContextApiPort.java index 50a2be1aa..7b45abf88 100644 --- a/src/main/java/fr/insee/genesis/domain/ports/api/DataProcessingContextApiPort.java +++ b/src/main/java/fr/insee/genesis/domain/ports/api/DataProcessingContextApiPort.java @@ -32,4 +32,5 @@ void saveKraftwerkExecutionSchedule(String partitionId, DataProcessingContextModel getContext(String interrogationId) throws GenesisException; DataProcessingContextModel getContextByPartitionId(String partitionId) throws GenesisException; + } diff --git a/src/main/java/fr/insee/genesis/domain/ports/api/LunaticJsonRawDataApiPort.java b/src/main/java/fr/insee/genesis/domain/ports/api/LunaticJsonRawDataApiPort.java index 6c7fb3088..f52b66b26 100644 --- a/src/main/java/fr/insee/genesis/domain/ports/api/LunaticJsonRawDataApiPort.java +++ b/src/main/java/fr/insee/genesis/domain/ports/api/LunaticJsonRawDataApiPort.java @@ -11,8 +11,10 @@ import org.springframework.data.domain.Page; import org.springframework.data.domain.Pageable; +import java.time.LocalDateTime; import java.time.Instant; import java.util.List; +import java.util.Map; import java.util.Set; public interface LunaticJsonRawDataApiPort { @@ -27,4 +29,5 @@ public interface LunaticJsonRawDataApiPort { Page findRawDataByCampaignIdAndDate(String campaignId, Instant startDt, Instant endDt, Pageable pageable); DataProcessResult processRawData(String campaignName, List interrogationIdList, List errors) throws GenesisException; + Map> findProcessedIdsgroupedByQuestionnaireSince(LocalDateTime since); } diff --git a/src/main/java/fr/insee/genesis/domain/ports/api/SurveyUnitApiPort.java b/src/main/java/fr/insee/genesis/domain/ports/api/SurveyUnitApiPort.java index 74838b2c2..d690ec42a 100644 --- a/src/main/java/fr/insee/genesis/domain/ports/api/SurveyUnitApiPort.java +++ b/src/main/java/fr/insee/genesis/domain/ports/api/SurveyUnitApiPort.java @@ -2,7 +2,7 @@ import fr.insee.bpm.metadata.model.VariablesMap; import fr.insee.genesis.controller.dto.CampaignWithQuestionnaire; -import fr.insee.genesis.controller.dto.InterrogationId; +import fr.insee.genesis.domain.model.surveyunit.InterrogationId; import fr.insee.genesis.controller.dto.QuestionnaireWithCampaign; import fr.insee.genesis.controller.dto.SurveyUnitDto; import fr.insee.genesis.controller.dto.SurveyUnitInputDto; diff --git a/src/main/java/fr/insee/genesis/domain/ports/spi/DataProcessingContextPersistancePort.java b/src/main/java/fr/insee/genesis/domain/ports/spi/DataProcessingContextPersistancePort.java index d7445390a..210d79894 100644 --- a/src/main/java/fr/insee/genesis/domain/ports/spi/DataProcessingContextPersistancePort.java +++ b/src/main/java/fr/insee/genesis/domain/ports/spi/DataProcessingContextPersistancePort.java @@ -9,6 +9,8 @@ public interface DataProcessingContextPersistancePort { DataProcessingContextDocument findByPartitionId(String partitionId); + List findByPartitionIds(List partitionIds); + void save(DataProcessingContextDocument dataProcessingContextDocument); void saveAll(List dataProcessingContextDocuments); diff --git a/src/main/java/fr/insee/genesis/domain/ports/spi/LunaticJsonRawDataPersistencePort.java b/src/main/java/fr/insee/genesis/domain/ports/spi/LunaticJsonRawDataPersistencePort.java index 8f3c8c3ce..b3df207e3 100644 --- a/src/main/java/fr/insee/genesis/domain/ports/spi/LunaticJsonRawDataPersistencePort.java +++ b/src/main/java/fr/insee/genesis/domain/ports/spi/LunaticJsonRawDataPersistencePort.java @@ -1,10 +1,12 @@ package fr.insee.genesis.domain.ports.spi; +import fr.insee.genesis.domain.model.surveyunit.GroupedInterrogation; import fr.insee.genesis.domain.model.surveyunit.Mode; import fr.insee.genesis.domain.model.surveyunit.rawdata.LunaticJsonRawDataModel; import org.springframework.data.domain.Page; import org.springframework.data.domain.Pageable; +import java.time.LocalDateTime; import java.time.Instant; import java.util.List; import java.util.Set; @@ -18,4 +20,5 @@ public interface LunaticJsonRawDataPersistencePort { Set findDistinctQuestionnaireIds(); Page findByCampaignIdAndDate(String campaignId, Instant startDt, Instant endDt, Pageable pageable); long countResponsesByQuestionnaireId(String questionnaireId); + List findProcessedIdsGroupedByQuestionnaireSince(LocalDateTime since); } diff --git a/src/main/java/fr/insee/genesis/domain/service/context/DataProcessingContextService.java b/src/main/java/fr/insee/genesis/domain/service/context/DataProcessingContextService.java index 1a159a3e3..5e1d351d0 100644 --- a/src/main/java/fr/insee/genesis/domain/service/context/DataProcessingContextService.java +++ b/src/main/java/fr/insee/genesis/domain/service/context/DataProcessingContextService.java @@ -172,4 +172,5 @@ public DataProcessingContextModel getContextByPartitionId(String partitionId){ dataProcessingContextPersistancePort.findByPartitionId(partitionId) ); } + } diff --git a/src/main/java/fr/insee/genesis/domain/service/rawdata/LunaticJsonRawDataService.java b/src/main/java/fr/insee/genesis/domain/service/rawdata/LunaticJsonRawDataService.java index bc8e145d7..19ce85522 100644 --- a/src/main/java/fr/insee/genesis/domain/service/rawdata/LunaticJsonRawDataService.java +++ b/src/main/java/fr/insee/genesis/domain/service/rawdata/LunaticJsonRawDataService.java @@ -8,6 +8,8 @@ import fr.insee.genesis.controller.utils.ControllerUtils; import fr.insee.genesis.domain.model.context.DataProcessingContextModel; import fr.insee.genesis.domain.model.surveyunit.DataState; +import fr.insee.genesis.domain.model.surveyunit.GroupedInterrogation; +import fr.insee.genesis.domain.model.surveyunit.InterrogationId; import fr.insee.genesis.domain.model.surveyunit.Mode; import fr.insee.genesis.domain.model.surveyunit.SurveyUnitModel; import fr.insee.genesis.domain.model.surveyunit.VariableModel; @@ -15,6 +17,7 @@ import fr.insee.genesis.domain.model.surveyunit.rawdata.LunaticJsonRawDataModel; import fr.insee.genesis.domain.model.surveyunit.rawdata.RawDataModelType; import fr.insee.genesis.domain.ports.api.LunaticJsonRawDataApiPort; +import fr.insee.genesis.domain.ports.spi.DataProcessingContextPersistancePort; import fr.insee.genesis.domain.ports.spi.LunaticJsonRawDataPersistencePort; import fr.insee.genesis.domain.ports.spi.SurveyUnitQualityToolPort; import fr.insee.genesis.domain.service.context.DataProcessingContextService; @@ -42,6 +45,7 @@ import java.util.List; import java.util.Map; import java.util.Set; +import java.util.stream.Collectors; @Service @Slf4j @@ -58,6 +62,8 @@ public class LunaticJsonRawDataService implements LunaticJsonRawDataApiPort { @Qualifier("lunaticJsonMongoAdapterNew") private final LunaticJsonRawDataPersistencePort lunaticJsonRawDataPersistencePort; + @Qualifier("dataProcessingContextMongoAdapter") + private final DataProcessingContextPersistancePort dataProcessingContextPersistancePort; @Autowired public LunaticJsonRawDataService(LunaticJsonRawDataPersistencePort lunaticJsonRawDataNewPersistencePort, @@ -68,7 +74,8 @@ public LunaticJsonRawDataService(LunaticJsonRawDataPersistencePort lunaticJsonRa FileUtils fileUtils, DataProcessingContextService dataProcessingContextService, SurveyUnitQualityToolPort surveyUnitQualityToolPort, - Config config + Config config, + DataProcessingContextPersistancePort dataProcessingContextPersistancePort ) { this.controllerUtils = controllerUtils; this.metadataService = metadataService; @@ -76,6 +83,7 @@ public LunaticJsonRawDataService(LunaticJsonRawDataPersistencePort lunaticJsonRa this.surveyUnitQualityService = surveyUnitQualityService; this.fileUtils = fileUtils; this.lunaticJsonRawDataPersistencePort = lunaticJsonRawDataNewPersistencePort; + this.dataProcessingContextPersistancePort = dataProcessingContextPersistancePort; this.surveyUnitQualityToolPort = surveyUnitQualityToolPort; this.dataProcessingContextService = dataProcessingContextService; this.config = config; @@ -391,6 +399,18 @@ public long countResponsesByQuestionnaireId(String campaignId) { return lunaticJsonRawDataPersistencePort.countResponsesByQuestionnaireId(campaignId); } + @Override + public Map> findProcessedIdsgroupedByQuestionnaireSince(LocalDateTime since) { + List idsByQuestionnaire = lunaticJsonRawDataPersistencePort.findProcessedIdsGroupedByQuestionnaireSince(since); + List partitionIds = idsByQuestionnaire.stream().map(GroupedInterrogation::partitionOrCampaignId).toList(); + List contexts = dataProcessingContextPersistancePort.findByPartitionIds(partitionIds); + List partitionIdsWithReview = contexts.stream().filter(DataProcessingContextModel::isWithReview).map(DataProcessingContextModel::getPartitionId).toList(); + return idsByQuestionnaire.stream().filter(groupedInterrogation -> partitionIdsWithReview.contains(groupedInterrogation.partitionOrCampaignId())) + .collect(Collectors.toMap( + GroupedInterrogation::questionnaireId, + GroupedInterrogation::interrogationIds + )); + } @Override public Page findRawDataByCampaignIdAndDate(String campaignId, Instant startDt, Instant endDt, Pageable pageable){ diff --git a/src/main/java/fr/insee/genesis/domain/service/surveyunit/SurveyUnitService.java b/src/main/java/fr/insee/genesis/domain/service/surveyunit/SurveyUnitService.java index 03af71be1..db40d16e3 100644 --- a/src/main/java/fr/insee/genesis/domain/service/surveyunit/SurveyUnitService.java +++ b/src/main/java/fr/insee/genesis/domain/service/surveyunit/SurveyUnitService.java @@ -2,7 +2,7 @@ import fr.insee.bpm.metadata.model.VariablesMap; import fr.insee.genesis.controller.dto.CampaignWithQuestionnaire; -import fr.insee.genesis.controller.dto.InterrogationId; +import fr.insee.genesis.domain.model.surveyunit.InterrogationId; import fr.insee.genesis.controller.dto.QuestionnaireWithCampaign; import fr.insee.genesis.controller.dto.SurveyUnitDto; import fr.insee.genesis.controller.dto.SurveyUnitInputDto; diff --git a/src/main/java/fr/insee/genesis/infrastructure/adapter/DataProcessingContextMongoAdapter.java b/src/main/java/fr/insee/genesis/infrastructure/adapter/DataProcessingContextMongoAdapter.java index 7e79b5411..eeb41c0c6 100644 --- a/src/main/java/fr/insee/genesis/infrastructure/adapter/DataProcessingContextMongoAdapter.java +++ b/src/main/java/fr/insee/genesis/infrastructure/adapter/DataProcessingContextMongoAdapter.java @@ -5,6 +5,7 @@ import fr.insee.genesis.domain.model.context.schedule.KraftwerkExecutionSchedule; import fr.insee.genesis.domain.ports.spi.DataProcessingContextPersistancePort; import fr.insee.genesis.infrastructure.document.context.DataProcessingContextDocument; +import fr.insee.genesis.infrastructure.mappers.DataProcessingContextMapper; import fr.insee.genesis.infrastructure.repository.DataProcessingContextMongoDBRepository; import fr.insee.genesis.infrastructure.utils.context.ContextDedupUtils; import lombok.extern.slf4j.Slf4j; @@ -41,6 +42,13 @@ public DataProcessingContextDocument findByPartitionId(String partitionId) { return ContextDedupUtils.deduplicateContexts(partitionId, existingDocuments); } + @Override + public List findByPartitionIds(List partitionIds){ + List existingDocuments = + dataProcessingContextMongoDBRepository.findByPartitionIdList(partitionIds); + return DataProcessingContextMapper.INSTANCE.listDocumentToListModel(ContextDedupUtils.deduplicateContexts(existingDocuments)); + } + @Override public void save(DataProcessingContextDocument dataProcessingContextDocument) { dataProcessingContextMongoDBRepository.save(dataProcessingContextDocument); diff --git a/src/main/java/fr/insee/genesis/infrastructure/adapter/LunaticJsonRawDataMongoAdapter.java b/src/main/java/fr/insee/genesis/infrastructure/adapter/LunaticJsonRawDataMongoAdapter.java index b32b4f60f..62604428f 100644 --- a/src/main/java/fr/insee/genesis/infrastructure/adapter/LunaticJsonRawDataMongoAdapter.java +++ b/src/main/java/fr/insee/genesis/infrastructure/adapter/LunaticJsonRawDataMongoAdapter.java @@ -1,10 +1,13 @@ package fr.insee.genesis.infrastructure.adapter; import fr.insee.genesis.Constants; +import fr.insee.genesis.domain.model.surveyunit.GroupedInterrogation; +import fr.insee.genesis.domain.model.surveyunit.InterrogationId; import fr.insee.genesis.domain.model.surveyunit.Mode; import fr.insee.genesis.domain.model.surveyunit.rawdata.LunaticJsonRawDataModel; import fr.insee.genesis.domain.ports.spi.LunaticJsonRawDataPersistencePort; import fr.insee.genesis.infrastructure.document.rawdata.LunaticJsonRawDataDocument; +import fr.insee.genesis.infrastructure.mappers.GroupedInterrogationDocumentMapper; import fr.insee.genesis.infrastructure.mappers.LunaticJsonRawDataDocumentMapper; import fr.insee.genesis.infrastructure.repository.LunaticJsonMongoDBRepository; import lombok.extern.slf4j.Slf4j; @@ -83,4 +86,11 @@ public Page findByCampaignIdAndDate(String campaignId, public long countResponsesByQuestionnaireId(String questionnaireId) { return repository.countByQuestionnaireId(questionnaireId); } + + @Override + public List findProcessedIdsGroupedByQuestionnaireSince(LocalDateTime since){ + return GroupedInterrogationDocumentMapper.INSTANCE.listDocumentToListModel(repository.aggregateRawGrouped(since)); + } + + } diff --git a/src/main/java/fr/insee/genesis/infrastructure/document/rawdata/LunaticJsonRawDataDocument.java b/src/main/java/fr/insee/genesis/infrastructure/document/rawdata/LunaticJsonRawDataDocument.java index a0fd12be9..c4221fdd8 100644 --- a/src/main/java/fr/insee/genesis/infrastructure/document/rawdata/LunaticJsonRawDataDocument.java +++ b/src/main/java/fr/insee/genesis/infrastructure/document/rawdata/LunaticJsonRawDataDocument.java @@ -4,6 +4,8 @@ import lombok.Builder; import org.bson.types.ObjectId; import org.springframework.data.annotation.Id; +import org.springframework.data.mongodb.core.index.IndexDirection; +import org.springframework.data.mongodb.core.index.Indexed; import org.springframework.data.mongodb.core.mapping.Document; import java.time.LocalDateTime; @@ -21,5 +23,6 @@ public record LunaticJsonRawDataDocument( Mode mode, Map data, LocalDateTime recordDate, + @Indexed(direction = IndexDirection.DESCENDING) LocalDateTime processDate ){} \ No newline at end of file diff --git a/src/main/java/fr/insee/genesis/infrastructure/document/surveyunit/GroupedInterrogationDocument.java b/src/main/java/fr/insee/genesis/infrastructure/document/surveyunit/GroupedInterrogationDocument.java new file mode 100644 index 000000000..00c520b9c --- /dev/null +++ b/src/main/java/fr/insee/genesis/infrastructure/document/surveyunit/GroupedInterrogationDocument.java @@ -0,0 +1,13 @@ +package fr.insee.genesis.infrastructure.document.surveyunit; + +import lombok.Data; + +import java.util.List; + +@Data +public class GroupedInterrogationDocument { + + private String questionnaireId; + private String partitionOrCampaignId; + private List interrogationIds; +} diff --git a/src/main/java/fr/insee/genesis/infrastructure/mappers/GroupedInterrogationDocumentMapper.java b/src/main/java/fr/insee/genesis/infrastructure/mappers/GroupedInterrogationDocumentMapper.java new file mode 100644 index 000000000..b335084ab --- /dev/null +++ b/src/main/java/fr/insee/genesis/infrastructure/mappers/GroupedInterrogationDocumentMapper.java @@ -0,0 +1,23 @@ +package fr.insee.genesis.infrastructure.mappers; + +import fr.insee.genesis.domain.model.surveyunit.GroupedInterrogation; +import fr.insee.genesis.infrastructure.document.surveyunit.GroupedInterrogationDocument; +import org.mapstruct.Mapper; +import org.mapstruct.factory.Mappers; + +import java.util.List; + +@Mapper +public interface GroupedInterrogationDocumentMapper { + + + GroupedInterrogationDocumentMapper INSTANCE = Mappers.getMapper(GroupedInterrogationDocumentMapper.class); + + GroupedInterrogation documentToModel(GroupedInterrogationDocument interrogationIdDoc); + + GroupedInterrogationDocument modelToDocument(GroupedInterrogation interrogationId); + + List listDocumentToListModel(List interrogationIdDocList); + + List listModelToListDocument(List interrogationIdList); +} diff --git a/src/main/java/fr/insee/genesis/infrastructure/repository/LunaticJsonMongoDBRepository.java b/src/main/java/fr/insee/genesis/infrastructure/repository/LunaticJsonMongoDBRepository.java index 62dc03ff2..acab95ecb 100644 --- a/src/main/java/fr/insee/genesis/infrastructure/repository/LunaticJsonMongoDBRepository.java +++ b/src/main/java/fr/insee/genesis/infrastructure/repository/LunaticJsonMongoDBRepository.java @@ -2,12 +2,15 @@ import fr.insee.genesis.domain.model.surveyunit.Mode; import fr.insee.genesis.infrastructure.document.rawdata.LunaticJsonRawDataDocument; +import fr.insee.genesis.infrastructure.document.surveyunit.GroupedInterrogationDocument; +import org.springframework.data.mongodb.repository.Aggregation; import org.springframework.data.domain.Page; import org.springframework.data.domain.Pageable; import org.springframework.data.mongodb.repository.MongoRepository; import org.springframework.data.mongodb.repository.Query; import org.springframework.stereotype.Repository; +import java.time.LocalDateTime; import java.time.Instant; import java.util.List; @@ -25,6 +28,22 @@ public interface LunaticJsonMongoDBRepository extends MongoRepository findByCampaignIdAndRecordDateBetween(String campagneId, Instant start, Instant end, Pageable pageable); long countByQuestionnaireId(String questionnaireId); - + @Aggregation(pipeline = { + "{ '$match': { 'processDate': { '$gte': ?0 } } }", + "{ '$group': { " + + "'_id': { " + + "'questionnaireId': '$questionnaireId', " + + "'partitionOrCampaignId': { '$ifNull': ['$partitionId', '$campaignId'] } " + + "}, " + + "'interrogationIds': { '$addToSet': '$interrogationId' } " + + "} }", + "{ '$project': { " + + "'questionnaireId': '$_id.questionnaireId', " + + "'partitionOrCampaignId': '$_id.partitionOrCampaignId', " + + "'interrogationIds': 1, " + + "'_id': 0 " + + "} }" + }) + List aggregateRawGrouped(LocalDateTime since); } diff --git a/src/test/java/cucumber/functional_tests/RawDataDefinitions.java b/src/test/java/cucumber/functional_tests/RawDataDefinitions.java index eac22e0f4..f5ed7f61a 100644 --- a/src/test/java/cucumber/functional_tests/RawDataDefinitions.java +++ b/src/test/java/cucumber/functional_tests/RawDataDefinitions.java @@ -63,6 +63,7 @@ public class RawDataDefinitions { LunaticJsonRawDataPersistanceStub lunaticJsonRawDataPersistanceStub = new LunaticJsonRawDataPersistanceStub(); MetadataService metadataService = new MetadataService(); SurveyUnitPersistencePortStub surveyUnitPersistencePortStub = new SurveyUnitPersistencePortStub(); + DataProcessingContextPersistancePortStub contextStub = new DataProcessingContextPersistancePortStub(); Config config = new ConfigStub(); FileUtils fileUtils = new FileUtils(config); SurveyUnitService surveyUnitService = new SurveyUnitService(surveyUnitPersistencePortStub, metadataService, fileUtils); @@ -83,7 +84,9 @@ public class RawDataDefinitions { dataProcessingContextPersistancePortStub, surveyUnitPersistencePortStub), surveyUnitQualityToolPerretAdapterStub, - config); + config, + dataProcessingContextPersistancePortStub); + RawResponseController rawResponseController = new RawResponseController( lunaticJsonRawDataService ); diff --git a/src/test/java/fr/insee/genesis/controller/rest/UtilsControllerTest.java b/src/test/java/fr/insee/genesis/controller/rest/UtilsControllerTest.java index 9936e683b..b2dd078a2 100644 --- a/src/test/java/fr/insee/genesis/controller/rest/UtilsControllerTest.java +++ b/src/test/java/fr/insee/genesis/controller/rest/UtilsControllerTest.java @@ -2,7 +2,7 @@ import cucumber.TestConstants; import fr.insee.genesis.Constants; -import fr.insee.genesis.controller.dto.InterrogationId; +import fr.insee.genesis.domain.model.surveyunit.InterrogationId; import fr.insee.genesis.controller.services.MetadataService; import fr.insee.genesis.controller.utils.ControllerUtils; import fr.insee.genesis.domain.model.surveyunit.DataState; @@ -11,6 +11,7 @@ import fr.insee.genesis.domain.model.surveyunit.VariableModel; import fr.insee.genesis.domain.ports.api.LunaticJsonRawDataApiPort; import fr.insee.genesis.domain.ports.api.SurveyUnitApiPort; +import fr.insee.genesis.domain.ports.spi.DataProcessingContextPersistancePort; import fr.insee.genesis.domain.service.context.DataProcessingContextService; import fr.insee.genesis.domain.service.rawdata.LunaticJsonRawDataService; import fr.insee.genesis.domain.service.surveyunit.SurveyUnitQualityService; @@ -46,6 +47,7 @@ class UtilsControllerTest { static UtilsController utilsControllerStatic; static SurveyUnitPersistencePortStub surveyUnitPersistencePortStub; static LunaticJsonRawDataPersistanceStub lunaticJsonRawDataPersistencePort; + static DataProcessingContextPersistancePort contextStub = new DataProcessingContextPersistancePortStub(); static SurveyUnitQualityToolPerretAdapterStub surveyUnitQualityToolPerretAdapterStub; static List interrogationIdList; static FileUtils fileUtils = new FileUtils(new ConfigStub()); @@ -73,7 +75,9 @@ static void init() { new DataProcessingContextPersistancePortStub(), surveyUnitPersistencePortStub), surveyUnitQualityToolPerretAdapterStub, - new ConfigStub()); + new ConfigStub(), + contextStub); + utilsControllerStatic = new UtilsController( surveyUnitApiPort diff --git a/src/test/java/fr/insee/genesis/controller/rest/responses/InterrogationControllerTest.java b/src/test/java/fr/insee/genesis/controller/rest/responses/InterrogationControllerTest.java index 81f9929e4..e1a45d79c 100644 --- a/src/test/java/fr/insee/genesis/controller/rest/responses/InterrogationControllerTest.java +++ b/src/test/java/fr/insee/genesis/controller/rest/responses/InterrogationControllerTest.java @@ -1,6 +1,6 @@ package fr.insee.genesis.controller.rest.responses; -import fr.insee.genesis.controller.dto.InterrogationId; +import fr.insee.genesis.domain.model.surveyunit.InterrogationId; import fr.insee.genesis.controller.services.MetadataService; import fr.insee.genesis.domain.ports.api.SurveyUnitApiPort; import fr.insee.genesis.domain.service.surveyunit.SurveyUnitService; diff --git a/src/test/java/fr/insee/genesis/controller/rest/responses/RawResponseControllerTest.java b/src/test/java/fr/insee/genesis/controller/rest/responses/RawResponseControllerTest.java index 9624d6ece..434181acc 100644 --- a/src/test/java/fr/insee/genesis/controller/rest/responses/RawResponseControllerTest.java +++ b/src/test/java/fr/insee/genesis/controller/rest/responses/RawResponseControllerTest.java @@ -49,8 +49,10 @@ class RawResponseControllerTest { fileUtils, new DataProcessingContextService(dataProcessingContextPersistancePortStub, surveyUnitPersistencePortStub), surveyUnitQualityToolPerretAdapterStub, - new ConfigStub() + new ConfigStub(), + new DataProcessingContextPersistancePortStub() ); + private final RawResponseController rawResponseController = new RawResponseController(lunaticJsonRawDataApiPort); diff --git a/src/test/java/fr/insee/genesis/controller/rest/responses/ResponseControllerTest.java b/src/test/java/fr/insee/genesis/controller/rest/responses/ResponseControllerTest.java index bf04e3ded..a20086e6e 100644 --- a/src/test/java/fr/insee/genesis/controller/rest/responses/ResponseControllerTest.java +++ b/src/test/java/fr/insee/genesis/controller/rest/responses/ResponseControllerTest.java @@ -3,7 +3,7 @@ import cucumber.TestConstants; import fr.insee.genesis.Constants; import fr.insee.genesis.configuration.Config; -import fr.insee.genesis.controller.dto.InterrogationId; +import fr.insee.genesis.domain.model.surveyunit.InterrogationId; import fr.insee.genesis.controller.dto.SurveyUnitInputDto; import fr.insee.genesis.controller.dto.SurveyUnitQualityToolDto; import fr.insee.genesis.controller.dto.SurveyUnitSimplified; diff --git a/src/test/java/fr/insee/genesis/domain/service/rawdata/LunaticJsonRawDataServiceTest.java b/src/test/java/fr/insee/genesis/domain/service/rawdata/LunaticJsonRawDataServiceTest.java index ce3e24a8a..dc6f1b672 100644 --- a/src/test/java/fr/insee/genesis/domain/service/rawdata/LunaticJsonRawDataServiceTest.java +++ b/src/test/java/fr/insee/genesis/domain/service/rawdata/LunaticJsonRawDataServiceTest.java @@ -40,9 +40,10 @@ class LunaticJsonRawDataServiceTest { MetadataService metadataService = new MetadataService(); SurveyUnitPersistencePortStub surveyUnitPersistencePortStub = new SurveyUnitPersistencePortStub(); + SurveyUnitService surveyUnitService = new SurveyUnitService(surveyUnitPersistencePortStub, metadataService, fileUtils); + SurveyUnitQualityService surveyUnitQualityService = new SurveyUnitQualityService(); + DataProcessingContextPersistancePortStub dataProcessingContextPersistancePortStub = new DataProcessingContextPersistancePortStub(); SurveyUnitQualityToolPerretAdapterStub surveyUnitQualityToolPerretAdapterStub = new SurveyUnitQualityToolPerretAdapterStub(); - DataProcessingContextPersistancePortStub dataProcessingContextPersistancePortStub = - new DataProcessingContextPersistancePortStub(); ConfigStub configStub = new ConfigStub(); LunaticJsonRawDataService lunaticJsonRawDataService = new LunaticJsonRawDataService( @@ -54,7 +55,8 @@ class LunaticJsonRawDataServiceTest { fileUtils, new DataProcessingContextService(dataProcessingContextPersistancePortStub, surveyUnitPersistencePortStub), surveyUnitQualityToolPerretAdapterStub, - configStub + configStub, + dataProcessingContextPersistancePortStub ); @Test diff --git a/src/test/java/fr/insee/genesis/infrastructure/adapter/SurveyUnitModelMongoAdapterTest.java b/src/test/java/fr/insee/genesis/infrastructure/adapter/SurveyUnitModelMongoAdapterTest.java index 718b27255..58c312c37 100644 --- a/src/test/java/fr/insee/genesis/infrastructure/adapter/SurveyUnitModelMongoAdapterTest.java +++ b/src/test/java/fr/insee/genesis/infrastructure/adapter/SurveyUnitModelMongoAdapterTest.java @@ -25,10 +25,8 @@ class SurveyUnitModelMongoAdapterTest { static SurveyUnitMongoDBRepository mongoRepository = Mockito.mock(SurveyUnitMongoDBRepository.class); - @InjectMocks static SurveyUnitMongoAdapter surveyUnitMongoAdapter; - static SurveyUnitDocument suDoc; static SurveyUnitDocument suDoc2; static SurveyUnitDocument suDoc3; @@ -172,4 +170,22 @@ void shouldReturnEmptyList_IfInterrogationIdsNotFoundInDataBase() { Assertions.assertThat(updates).isEmpty(); } + @Test + void shouldReturnCampaignCount() { + when(mongoRepository.countByCampaignId("CAMP42")).thenReturn(7L); + + long count = surveyUnitMongoAdapter.countByCampaignId("CAMP42"); + Assertions.assertThat(count).isEqualTo(7L); + } + + @Test + void shouldReturnZero_whenCountIsEmpty() { + when(mongoRepository.countByCampaignId("CAMP999")).thenReturn(0L); + + long count = surveyUnitMongoAdapter.countByCampaignId("CAMP999"); + Assertions.assertThat(count).isEqualTo(0L); + } + + + } diff --git a/src/test/java/fr/insee/genesis/infrastructure/mapper/GroupedInterrogationDocumentMapperTest.java b/src/test/java/fr/insee/genesis/infrastructure/mapper/GroupedInterrogationDocumentMapperTest.java new file mode 100644 index 000000000..1817d72fe --- /dev/null +++ b/src/test/java/fr/insee/genesis/infrastructure/mapper/GroupedInterrogationDocumentMapperTest.java @@ -0,0 +1,85 @@ +package fr.insee.genesis.infrastructure.mapper; + +import fr.insee.genesis.domain.model.surveyunit.GroupedInterrogation; +import fr.insee.genesis.infrastructure.document.surveyunit.GroupedInterrogationDocument; +import fr.insee.genesis.infrastructure.mappers.GroupedInterrogationDocumentMapper; +import org.assertj.core.api.Assertions; +import org.junit.jupiter.api.Test; + +import java.util.List; + +class GroupedInterrogationDocumentMapperTest { + + private final GroupedInterrogationDocumentMapper mapper = GroupedInterrogationDocumentMapper.INSTANCE; + + @Test + void testDocumentToModel() { + GroupedInterrogationDocument doc = new GroupedInterrogationDocument(); + doc.setQuestionnaireId("Q1"); + doc.setPartitionOrCampaignId("CAMP1"); + doc.setInterrogationIds(List.of("I1", "I2")); + + GroupedInterrogation model = mapper.documentToModel(doc); + + Assertions.assertThat(model).isNotNull(); + Assertions.assertThat(model.questionnaireId()).isEqualTo("Q1"); + Assertions.assertThat(model.partitionOrCampaignId()).isEqualTo("CAMP1"); + Assertions.assertThat(model.interrogationIds()).isEqualTo(List.of("I1", "I2")); + } + + @Test + void testModelToDocument() { + GroupedInterrogation model = GroupedInterrogation.builder() + .questionnaireId("Q2") + .partitionOrCampaignId("CAMP2") + .interrogationIds(List.of("I3", "I4")) + .build(); + + GroupedInterrogationDocument doc = mapper.modelToDocument(model); + + Assertions.assertThat(doc).isNotNull(); + Assertions.assertThat(doc.getQuestionnaireId()).isEqualTo("Q2"); + Assertions.assertThat(doc.getPartitionOrCampaignId()).isEqualTo("CAMP2"); + Assertions.assertThat(doc.getInterrogationIds()).isEqualTo(List.of("I3", "I4")); + } + + @Test + void testListDocumentToListModel() { + GroupedInterrogationDocument doc1 = new GroupedInterrogationDocument(); + doc1.setQuestionnaireId("Q1"); + doc1.setPartitionOrCampaignId("C1"); + doc1.setInterrogationIds(List.of("I1")); + + GroupedInterrogationDocument doc2 = new GroupedInterrogationDocument(); + doc2.setQuestionnaireId("Q2"); + doc2.setPartitionOrCampaignId("C2"); + doc2.setInterrogationIds(List.of("I2")); + + List models = mapper.listDocumentToListModel(List.of(doc1, doc2)); + + Assertions.assertThat(models).hasSize(2); + Assertions.assertThat(models.get(0).questionnaireId()).isEqualTo("Q1"); + Assertions.assertThat(models.get(1).questionnaireId()).isEqualTo("Q2"); + } + + @Test + void testListModelToListDocument() { + GroupedInterrogation model1 = GroupedInterrogation.builder() + .questionnaireId("Q3") + .partitionOrCampaignId("C3") + .interrogationIds(List.of("I3")) + .build(); + + GroupedInterrogation model2 = GroupedInterrogation.builder() + .questionnaireId("Q4") + .partitionOrCampaignId("C4") + .interrogationIds(List.of("I4")) + .build(); + + List docs = mapper.listModelToListDocument(List.of(model1, model2)); + + Assertions.assertThat(docs).hasSize(2); + Assertions.assertThat(docs.get(0).getQuestionnaireId()).isEqualTo("Q3"); + Assertions.assertThat(docs.get(1).getQuestionnaireId()).isEqualTo("Q4"); + } +} diff --git a/src/test/java/fr/insee/genesis/stubs/DataProcessingContextPersistancePortStub.java b/src/test/java/fr/insee/genesis/stubs/DataProcessingContextPersistancePortStub.java index 67fd049d9..c7b1d803b 100644 --- a/src/test/java/fr/insee/genesis/stubs/DataProcessingContextPersistancePortStub.java +++ b/src/test/java/fr/insee/genesis/stubs/DataProcessingContextPersistancePortStub.java @@ -27,6 +27,17 @@ public DataProcessingContextDocument findByPartitionId(String partitionId) { ); } + @Override + public List findByPartitionIds(List partitionIds) { + return DataProcessingContextMapper.INSTANCE.listDocumentToListModel( + ContextDedupUtils.deduplicateContexts( + mongoStub.stream().filter( + dataProcessingContextDocument -> partitionIds.contains(dataProcessingContextDocument.getPartitionId()) + ).toList() + ) + ); + } + @Override public void save(DataProcessingContextDocument dataProcessingContextDocument) { mongoStub.removeIf(existingDoc -> existingDoc.getPartitionId().equals( diff --git a/src/test/java/fr/insee/genesis/stubs/LunaticJsonMongoDBRepositoryStub.java b/src/test/java/fr/insee/genesis/stubs/LunaticJsonMongoDBRepositoryStub.java index 9a48da8aa..bb7aa809e 100644 --- a/src/test/java/fr/insee/genesis/stubs/LunaticJsonMongoDBRepositoryStub.java +++ b/src/test/java/fr/insee/genesis/stubs/LunaticJsonMongoDBRepositoryStub.java @@ -2,6 +2,7 @@ import fr.insee.genesis.domain.model.surveyunit.Mode; import fr.insee.genesis.infrastructure.document.rawdata.LunaticJsonRawDataDocument; +import fr.insee.genesis.infrastructure.document.surveyunit.GroupedInterrogationDocument; import fr.insee.genesis.infrastructure.repository.LunaticJsonMongoDBRepository; import lombok.Getter; import org.springframework.data.domain.Example; @@ -10,13 +11,16 @@ import org.springframework.data.domain.Sort; import org.springframework.data.repository.query.FluentQuery; +import java.time.LocalDateTime; import java.time.Instant; import java.util.ArrayList; import java.util.HashMap; +import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Objects; import java.util.Optional; +import java.util.Set; import java.util.function.Function; @Getter @@ -61,6 +65,42 @@ public long countByQuestionnaireId(String questionnaireId) { return countByQuestionnaireIdMap.getOrDefault(questionnaireId, 0L); } + @Override + public List aggregateRawGrouped(LocalDateTime since) { + List recentDocs = documents.stream().filter( + rawData -> rawData.processDate() != null && rawData.processDate().isAfter(since) + ).toList(); + + // Aggregation map: key = (questionnaireId, partitionOrCampaignId), value = Set of interrogationId + Map>> groupedMap = new HashMap<>(); + + for (LunaticJsonRawDataDocument doc : recentDocs) { + String questionnaireId = doc.questionnaireId(); + String partitionOrCampaignId = doc.campaignId(); + String interrogationId = doc.interrogationId(); + groupedMap + .computeIfAbsent(questionnaireId, q -> new HashMap<>()) + .computeIfAbsent(partitionOrCampaignId, p -> new HashSet<>()) + .add(interrogationId); + } + + // Conversion to a list of GroupedInterrogationDocument + List result = new ArrayList<>(); + for (Map.Entry>> entry1 : groupedMap.entrySet()) { + String questionnaireId = entry1.getKey(); + Map> innerMap = entry1.getValue(); + + for (Map.Entry> entry2 : innerMap.entrySet()) { + GroupedInterrogationDocument grouped = new GroupedInterrogationDocument(); + grouped.setQuestionnaireId(questionnaireId); + grouped.setPartitionOrCampaignId(entry2.getKey()); + grouped.setInterrogationIds(new ArrayList<>(entry2.getValue())); + result.add(grouped); + } + } + return result; + } + // Implémentations vides requises par MongoRepository @Override public S save(S entity) { return null; } @Override public Optional findById(String s) { return Optional.empty(); } diff --git a/src/test/java/fr/insee/genesis/stubs/LunaticJsonRawDataPersistanceStub.java b/src/test/java/fr/insee/genesis/stubs/LunaticJsonRawDataPersistanceStub.java index 0e01fd97c..d49e153fd 100644 --- a/src/test/java/fr/insee/genesis/stubs/LunaticJsonRawDataPersistanceStub.java +++ b/src/test/java/fr/insee/genesis/stubs/LunaticJsonRawDataPersistanceStub.java @@ -1,9 +1,13 @@ package fr.insee.genesis.stubs; +import fr.insee.genesis.domain.model.surveyunit.GroupedInterrogation; +import fr.insee.genesis.domain.model.surveyunit.InterrogationId; import fr.insee.genesis.domain.model.surveyunit.Mode; import fr.insee.genesis.domain.model.surveyunit.rawdata.LunaticJsonRawDataModel; import fr.insee.genesis.domain.ports.spi.LunaticJsonRawDataPersistencePort; import fr.insee.genesis.infrastructure.document.rawdata.LunaticJsonRawDataDocument; +import fr.insee.genesis.infrastructure.document.surveyunit.GroupedInterrogationDocument; +import fr.insee.genesis.infrastructure.mappers.GroupedInterrogationDocumentMapper; import fr.insee.genesis.infrastructure.mappers.LunaticJsonRawDataDocumentMapper; import lombok.Getter; import org.springframework.data.domain.Page; @@ -14,8 +18,10 @@ import java.time.LocalDateTime; import java.time.ZoneOffset; import java.util.ArrayList; +import java.util.HashMap; import java.util.HashSet; import java.util.List; +import java.util.Map; import java.util.Set; @Getter @@ -99,4 +105,41 @@ public Page findByCampaignIdAndDate(String campaignId, public long countResponsesByQuestionnaireId(String questionnaireId) { return mongoStub.size(); } + + @Override + public List findProcessedIdsGroupedByQuestionnaireSince(LocalDateTime since) { + List recentDocs = mongoStub.stream().filter( + rawData -> rawData.processDate() != null && rawData.processDate().isAfter(since) + ).toList(); + + // Aggregation map: key = (questionnaireId, partitionOrCampaignId), value = Set of interrogationId + Map>> groupedMap = new HashMap<>(); + + for (LunaticJsonRawDataDocument doc : recentDocs) { + String questionnaireId = doc.questionnaireId(); + String partitionOrCampaignId = doc.campaignId(); + String interrogationId = doc.interrogationId(); + groupedMap + .computeIfAbsent(questionnaireId, q -> new HashMap<>()) + .computeIfAbsent(partitionOrCampaignId, p -> new HashSet<>()) + .add(interrogationId); + } + + // Conversion to a list of GroupedInterrogationDocument + List result = new ArrayList<>(); + for (Map.Entry>> entry1 : groupedMap.entrySet()) { + String questionnaireId = entry1.getKey(); + Map> innerMap = entry1.getValue(); + + for (Map.Entry> entry2 : innerMap.entrySet()) { + GroupedInterrogationDocument grouped = new GroupedInterrogationDocument(); + grouped.setQuestionnaireId(questionnaireId); + grouped.setPartitionOrCampaignId(entry2.getKey()); + grouped.setInterrogationIds(new ArrayList<>(entry2.getValue())); + result.add(grouped); + } + } + return GroupedInterrogationDocumentMapper.INSTANCE.listDocumentToListModel(result); + } + }