From 00d1362d1b0ea62e52a15b6ec375fe98c21771bb Mon Sep 17 00:00:00 2001 From: QDIBYS Date: Wed, 18 Jun 2025 11:30:28 +0200 Subject: [PATCH 01/12] feat: send processed interrogationIds to Perret --- src/main/java/fr/insee/genesis/Constants.java | 8 ++-- .../insee/genesis/configuration/Config.java | 3 ++ .../ports/spi/SurveyUnitQualityToolPort.java | 10 +++++ .../rawdata/LunaticJsonRawDataService.java | 42 +++++++++++++++++-- .../insee/genesis/domain/utils/HttpUtils.java | 22 ++++++++++ .../SurveyQualityToolPerretAdapter.java | 29 +++++++++++++ src/main/resources/application-dev.properties | 7 +++- .../resources/application-preprod.properties | 7 +++- .../resources/application-prod.properties | 7 +++- .../application-test-cucumber.properties | 2 + .../resources/application-test.properties | 2 + .../functional_tests/RawDataDefinitions.java | 5 ++- .../controller/rest/UtilsControllerTest.java | 4 +- .../responses/RawResponseControllerTest.java | 14 ++++++- .../LunaticJsonRawDataServiceTest.java | 15 +++++-- ...urveyUnitQualityToolPerretAdapterStub.java | 23 ++++++++++ 16 files changed, 183 insertions(+), 17 deletions(-) create mode 100644 src/main/java/fr/insee/genesis/domain/ports/spi/SurveyUnitQualityToolPort.java create mode 100644 src/main/java/fr/insee/genesis/domain/utils/HttpUtils.java create mode 100644 src/main/java/fr/insee/genesis/infrastructure/adapter/SurveyQualityToolPerretAdapter.java create mode 100644 src/test/java/fr/insee/genesis/stubs/SurveyUnitQualityToolPerretAdapterStub.java diff --git a/src/main/java/fr/insee/genesis/Constants.java b/src/main/java/fr/insee/genesis/Constants.java index 9fb7edc67..ce9c5aaa5 100644 --- a/src/main/java/fr/insee/genesis/Constants.java +++ b/src/main/java/fr/insee/genesis/Constants.java @@ -1,5 +1,8 @@ package fr.insee.genesis; +import lombok.experimental.UtilityClass; + +@UtilityClass public class Constants { public static final String XSLT_STRUCTURED_VARIABLES = "xslt/structured-variables.xsl"; @@ -58,10 +61,7 @@ public class Constants { // Kraftwerk service path parameters public static final String KRAFTWERK_MAIN_ENDPOINT = ""; - private Constants() { - throw new IllegalStateException("Constants class"); - } - + public static final int RAW_DATA_PROCESSING_BATCH_SIZE = 1000; public static String[] getEnoVariables() { return ENO_VARIABLES; diff --git a/src/main/java/fr/insee/genesis/configuration/Config.java b/src/main/java/fr/insee/genesis/configuration/Config.java index 15d511749..b90e95fda 100644 --- a/src/main/java/fr/insee/genesis/configuration/Config.java +++ b/src/main/java/fr/insee/genesis/configuration/Config.java @@ -40,6 +40,9 @@ public class Config { @Value("#{'${fr.insee.genesis.security.whitelist-matchers}'.split(',')}") private String[] whiteList; + @Value("${fr.insee.genesis.survey-quality-tool.url}") + private String surveyQualityToolUrl; + private final String logFolder; //Extract log folder from log filename property diff --git a/src/main/java/fr/insee/genesis/domain/ports/spi/SurveyUnitQualityToolPort.java b/src/main/java/fr/insee/genesis/domain/ports/spi/SurveyUnitQualityToolPort.java new file mode 100644 index 000000000..2c658811f --- /dev/null +++ b/src/main/java/fr/insee/genesis/domain/ports/spi/SurveyUnitQualityToolPort.java @@ -0,0 +1,10 @@ +package fr.insee.genesis.domain.ports.spi; + +import org.springframework.http.ResponseEntity; + +import java.util.Map; +import java.util.Set; + +public interface SurveyUnitQualityToolPort { + ResponseEntity sendProcessedIds(Map> processedIdsMap); +} diff --git a/src/main/java/fr/insee/genesis/domain/service/rawdata/LunaticJsonRawDataService.java b/src/main/java/fr/insee/genesis/domain/service/rawdata/LunaticJsonRawDataService.java index 42db85031..0131cdb6c 100644 --- a/src/main/java/fr/insee/genesis/domain/service/rawdata/LunaticJsonRawDataService.java +++ b/src/main/java/fr/insee/genesis/domain/service/rawdata/LunaticJsonRawDataService.java @@ -14,6 +14,7 @@ import fr.insee.genesis.domain.model.surveyunit.rawdata.RawDataModelType; import fr.insee.genesis.domain.ports.api.LunaticJsonRawDataApiPort; import fr.insee.genesis.domain.ports.spi.LunaticJsonRawDataPersistencePort; +import fr.insee.genesis.domain.ports.spi.SurveyUnitQualityToolPort; import fr.insee.genesis.domain.service.surveyunit.SurveyUnitQualityService; import fr.insee.genesis.domain.service.surveyunit.SurveyUnitService; import fr.insee.genesis.domain.utils.GroupUtils; @@ -24,10 +25,12 @@ import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Qualifier; +import org.springframework.http.ResponseEntity; import org.springframework.stereotype.Service; import java.time.LocalDateTime; import java.util.ArrayList; +import java.util.HashMap; import java.util.HashSet; import java.util.List; import java.util.Map; @@ -41,20 +44,28 @@ public class LunaticJsonRawDataService implements LunaticJsonRawDataApiPort { private final MetadataService metadataService; private final SurveyUnitService surveyUnitService; private final SurveyUnitQualityService surveyUnitQualityService; + private final SurveyUnitQualityToolPort surveyUnitQualityToolPort; private final FileUtils fileUtils; - private static final int BATCH_SIZE = 1000; @Qualifier("lunaticJsonMongoAdapterNew") private final LunaticJsonRawDataPersistencePort lunaticJsonRawDataPersistencePort; @Autowired - public LunaticJsonRawDataService(LunaticJsonRawDataPersistencePort lunaticJsonRawDataNewPersistencePort, ControllerUtils controllerUtils, MetadataService metadataService, SurveyUnitService surveyUnitService, SurveyUnitQualityService surveyUnitQualityService, FileUtils fileUtils) { + public LunaticJsonRawDataService(LunaticJsonRawDataPersistencePort lunaticJsonRawDataNewPersistencePort, + ControllerUtils controllerUtils, + MetadataService metadataService, + SurveyUnitService surveyUnitService, + SurveyUnitQualityService surveyUnitQualityService, + FileUtils fileUtils, + SurveyUnitQualityToolPort surveyUnitQualityToolPort + ) { this.controllerUtils = controllerUtils; this.metadataService = metadataService; this.surveyUnitService = surveyUnitService; this.surveyUnitQualityService = surveyUnitQualityService; this.fileUtils = fileUtils; this.lunaticJsonRawDataPersistencePort = lunaticJsonRawDataNewPersistencePort; + this.surveyUnitQualityToolPort = surveyUnitQualityToolPort; } @Override @@ -82,12 +93,12 @@ public DataProcessResult processRawData(String campaignName, List interr .formatted(mode, errors.getLast().getMessage()) ); } - int totalBatchs = Math.ceilDiv(interrogationIdList.size() , BATCH_SIZE); + int totalBatchs = Math.ceilDiv(interrogationIdList.size() , Constants.RAW_DATA_PROCESSING_BATCH_SIZE); int batchNumber = 1; List interrogationIdListForMode = new ArrayList<>(interrogationIdList); while(!interrogationIdListForMode.isEmpty()){ log.info("Processing raw data batch {}/{}", batchNumber, totalBatchs); - int maxIndex = Math.min(interrogationIdListForMode.size(), BATCH_SIZE); + int maxIndex = Math.min(interrogationIdListForMode.size(), Constants.RAW_DATA_PROCESSING_BATCH_SIZE); List interrogationIdToProcess = interrogationIdListForMode.subList(0, maxIndex); List rawData = getRawData(campaignName,mode, interrogationIdToProcess); @@ -110,6 +121,14 @@ public DataProcessResult processRawData(String campaignName, List interr surveyUnitModel -> surveyUnitModel.getState().equals(DataState.FORMATTED) ).toList().size(); + //Send processed ids grouped by questionnaire + ResponseEntity response = + surveyUnitQualityToolPort.sendProcessedIds(getProcessedIdsMap(surveyUnitModels)); + if(!response.getStatusCode().is2xxSuccessful()){ + log.warn("Survey unit quality tool responded non-2xx code {} and body {}", + response.getStatusCode(), response.getBody()); + } + //Remove processed ids from list interrogationIdListForMode = interrogationIdListForMode.subList(maxIndex, interrogationIdListForMode.size()); @@ -119,6 +138,21 @@ public DataProcessResult processRawData(String campaignName, List interr return new DataProcessResult(dataCount, formattedDataCount); } + private Map> getProcessedIdsMap(List surveyUnitModels) { + Map> processedInterrogationIdsPerQuestionnaire = new HashMap<>(); + for(SurveyUnitModel surveyUnitModel : surveyUnitModels){ + String questionnaireId = surveyUnitModel.getQuestionnaireId(); + String interrogationId = surveyUnitModel.getInterrogationId(); + + if(!processedInterrogationIdsPerQuestionnaire.containsKey(questionnaireId)){ + processedInterrogationIdsPerQuestionnaire.put(questionnaireId, new HashSet<>()); + } + + processedInterrogationIdsPerQuestionnaire.get(questionnaireId).add(interrogationId); + } + return processedInterrogationIdsPerQuestionnaire; + } + @Override public List convertRawData(List rawDataList, VariablesMap variablesMap) { //Convert to genesis model diff --git a/src/main/java/fr/insee/genesis/domain/utils/HttpUtils.java b/src/main/java/fr/insee/genesis/domain/utils/HttpUtils.java new file mode 100644 index 000000000..19cf14189 --- /dev/null +++ b/src/main/java/fr/insee/genesis/domain/utils/HttpUtils.java @@ -0,0 +1,22 @@ +package fr.insee.genesis.domain.utils; + +import lombok.experimental.UtilityClass; +import org.springframework.boot.web.client.RestTemplateBuilder; +import org.springframework.http.HttpEntity; +import org.springframework.http.HttpHeaders; +import org.springframework.http.HttpMethod; +import org.springframework.http.MediaType; +import org.springframework.http.ResponseEntity; +import org.springframework.web.client.RestTemplate; + +@UtilityClass +public class HttpUtils { + public static ResponseEntity makeApiCall(String url, HttpMethod method, T requestBody, Class responseType){ + RestTemplate restTemplate = new RestTemplateBuilder().build(); + HttpHeaders headers = new HttpHeaders(); + headers.setContentType(MediaType.APPLICATION_JSON); + HttpEntity requestEntity = new HttpEntity<>(requestBody, headers); + + return restTemplate.exchange(url, method, requestEntity, responseType); + } +} diff --git a/src/main/java/fr/insee/genesis/infrastructure/adapter/SurveyQualityToolPerretAdapter.java b/src/main/java/fr/insee/genesis/infrastructure/adapter/SurveyQualityToolPerretAdapter.java new file mode 100644 index 000000000..4cfcfd140 --- /dev/null +++ b/src/main/java/fr/insee/genesis/infrastructure/adapter/SurveyQualityToolPerretAdapter.java @@ -0,0 +1,29 @@ +package fr.insee.genesis.infrastructure.adapter; + +import fr.insee.genesis.configuration.Config; +import fr.insee.genesis.domain.ports.spi.SurveyUnitQualityToolPort; +import fr.insee.genesis.domain.utils.HttpUtils; +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Qualifier; +import org.springframework.http.HttpMethod; +import org.springframework.http.ResponseEntity; +import org.springframework.stereotype.Service; + +import java.util.Map; +import java.util.Set; + +@Service +@Qualifier("surveyQualityToolPerretAdapter") +@Slf4j +public class SurveyQualityToolPerretAdapter implements SurveyUnitQualityToolPort { + + @Autowired + Config config; + + + @Override + public ResponseEntity sendProcessedIds(Map> processedIdsMap) { + return HttpUtils.makeApiCall(config.getSurveyQualityToolUrl(), HttpMethod.POST, processedIdsMap, Object.class); + } +} diff --git a/src/main/resources/application-dev.properties b/src/main/resources/application-dev.properties index 9e16ba510..ddad30609 100644 --- a/src/main/resources/application-dev.properties +++ b/src/main/resources/application-dev.properties @@ -22,4 +22,9 @@ app.role.user-kraftwerk.claims=*** app.role.user-platine.claims=*** app.role.reader.claims=*** app.role.collect-platform.claims=*** -app.role.scheduler.claims=*** \ No newline at end of file +app.role.scheduler.claims=*** + +#-------------------------------------------------------------------------- +# Survey quality tool +#-------------------------------------------------------------------------- +fr.insee.genesis.survey-quality-tool.url=*** \ No newline at end of file diff --git a/src/main/resources/application-preprod.properties b/src/main/resources/application-preprod.properties index 19e1c751f..3faf95de3 100644 --- a/src/main/resources/application-preprod.properties +++ b/src/main/resources/application-preprod.properties @@ -22,4 +22,9 @@ app.role.user-kraftwerk.claims=*** app.role.user-platine.claims=*** app.role.reader.claims=*** app.role.collect-platform.claims=*** -app.role.scheduler.claims=*** \ No newline at end of file +app.role.scheduler.claims=*** + +#-------------------------------------------------------------------------- +# Survey quality tool +#-------------------------------------------------------------------------- +fr.insee.genesis.survey-quality-tool.url=*** \ No newline at end of file diff --git a/src/main/resources/application-prod.properties b/src/main/resources/application-prod.properties index a98155889..92cc4a134 100644 --- a/src/main/resources/application-prod.properties +++ b/src/main/resources/application-prod.properties @@ -22,4 +22,9 @@ app.role.user-kraftwerk.claims=*** app.role.user-platine.claims=*** app.role.reader.claims=*** app.role.collect-platform.claims=*** -app.role.scheduler.claims=*** \ No newline at end of file +app.role.scheduler.claims=*** + +#-------------------------------------------------------------------------- +# Survey quality tool +#-------------------------------------------------------------------------- +fr.insee.genesis.survey-quality-tool.url=*** \ No newline at end of file diff --git a/src/main/resources/application-test-cucumber.properties b/src/main/resources/application-test-cucumber.properties index 39bde09ad..0bd1334ac 100644 --- a/src/main/resources/application-test-cucumber.properties +++ b/src/main/resources/application-test-cucumber.properties @@ -16,3 +16,5 @@ fr.insee.genesis.oidc.realm=test-realm springdoc.swagger-ui.oauth.client-id=client-id-test fr.insee.genesis.authentication = NONE + +fr.insee.genesis.survey-quality-tool.url=*** \ No newline at end of file diff --git a/src/main/resources/application-test.properties b/src/main/resources/application-test.properties index 932289c53..ab67d3a65 100644 --- a/src/main/resources/application-test.properties +++ b/src/main/resources/application-test.properties @@ -17,3 +17,5 @@ springdoc.swagger-ui.oauth.client-id=client-id-test fr.insee.genesis.authentication = OIDC + +fr.insee.genesis.survey-quality-tool.url=*** \ No newline at end of file diff --git a/src/test/java/cucumber/functional_tests/RawDataDefinitions.java b/src/test/java/cucumber/functional_tests/RawDataDefinitions.java index 951a99e1d..792da6499 100644 --- a/src/test/java/cucumber/functional_tests/RawDataDefinitions.java +++ b/src/test/java/cucumber/functional_tests/RawDataDefinitions.java @@ -17,6 +17,7 @@ import fr.insee.genesis.infrastructure.utils.FileUtils; import fr.insee.genesis.stubs.ConfigStub; import fr.insee.genesis.stubs.LunaticJsonRawDataPersistanceStub; +import fr.insee.genesis.stubs.SurveyUnitQualityToolPerretAdapterStub; import fr.insee.genesis.stubs.SurveyUnitPersistencePortStub; import io.cucumber.java.Before; import io.cucumber.java.en.Given; @@ -65,7 +66,9 @@ public class RawDataDefinitions { SurveyUnitService surveyUnitService = new SurveyUnitService(surveyUnitPersistencePortStub, metadataService, fileUtils); ControllerUtils controllerUtils = new ControllerUtils(fileUtils); SurveyUnitQualityService surveyUnitQualityService = new SurveyUnitQualityService(); - LunaticJsonRawDataService lunaticJsonRawDataService = new LunaticJsonRawDataService(lunaticJsonRawDataPersistanceStub,controllerUtils,metadataService,surveyUnitService,surveyUnitQualityService,fileUtils); + + SurveyUnitQualityToolPerretAdapterStub surveyUnitQualityToolPerretAdapterStub = new SurveyUnitQualityToolPerretAdapterStub(); + LunaticJsonRawDataService lunaticJsonRawDataService = new LunaticJsonRawDataService(lunaticJsonRawDataPersistanceStub,controllerUtils,metadataService,surveyUnitService,surveyUnitQualityService,fileUtils, surveyUnitQualityToolPerretAdapterStub); RawResponseController rawResponseController = new RawResponseController( lunaticJsonRawDataService ); diff --git a/src/test/java/fr/insee/genesis/controller/rest/UtilsControllerTest.java b/src/test/java/fr/insee/genesis/controller/rest/UtilsControllerTest.java index 2885083a1..a3fe67515 100644 --- a/src/test/java/fr/insee/genesis/controller/rest/UtilsControllerTest.java +++ b/src/test/java/fr/insee/genesis/controller/rest/UtilsControllerTest.java @@ -18,6 +18,7 @@ import fr.insee.genesis.infrastructure.utils.FileUtils; import fr.insee.genesis.stubs.ConfigStub; import fr.insee.genesis.stubs.LunaticJsonRawDataPersistanceStub; +import fr.insee.genesis.stubs.SurveyUnitQualityToolPerretAdapterStub; import fr.insee.genesis.stubs.SurveyUnitPersistencePortStub; import org.assertj.core.api.Assertions; import org.junit.jupiter.api.AfterAll; @@ -43,6 +44,7 @@ class UtilsControllerTest { static UtilsController utilsControllerStatic; static SurveyUnitPersistencePortStub surveyUnitPersistencePortStub; static LunaticJsonRawDataPersistanceStub lunaticJsonRawDataPersistencePort; + static SurveyUnitQualityToolPerretAdapterStub surveyUnitQualityToolPerretAdapterStub; static List interrogationIdList; static FileUtils fileUtils = new FileUtils(new ConfigStub()); static ControllerUtils controllerUtils = new ControllerUtils(fileUtils); @@ -58,7 +60,7 @@ static void init() { surveyUnitPersistencePortStub = new SurveyUnitPersistencePortStub(); lunaticJsonRawDataPersistencePort = new LunaticJsonRawDataPersistanceStub(); SurveyUnitApiPort surveyUnitApiPort = new SurveyUnitService(surveyUnitPersistencePortStub, metadataService, fileUtils); - LunaticJsonRawDataApiPort lunaticJsonRawDataApiPort = new LunaticJsonRawDataService(lunaticJsonRawDataPersistencePort,controllerUtils,metadataService,surveyUnitService,surveyUnitQualityService,fileUtils); + LunaticJsonRawDataApiPort lunaticJsonRawDataApiPort = new LunaticJsonRawDataService(lunaticJsonRawDataPersistencePort,controllerUtils,metadataService,surveyUnitService,surveyUnitQualityService,fileUtils, surveyUnitQualityToolPerretAdapterStub); utilsControllerStatic = new UtilsController( surveyUnitApiPort diff --git a/src/test/java/fr/insee/genesis/controller/rest/responses/RawResponseControllerTest.java b/src/test/java/fr/insee/genesis/controller/rest/responses/RawResponseControllerTest.java index b068c785e..1770073e0 100644 --- a/src/test/java/fr/insee/genesis/controller/rest/responses/RawResponseControllerTest.java +++ b/src/test/java/fr/insee/genesis/controller/rest/responses/RawResponseControllerTest.java @@ -1,6 +1,7 @@ package fr.insee.genesis.controller.rest.responses; +import fr.insee.genesis.Constants; import fr.insee.genesis.controller.dto.rawdata.LunaticJsonRawDataUnprocessedDto; import fr.insee.genesis.controller.services.MetadataService; import fr.insee.genesis.controller.utils.ControllerUtils; @@ -15,6 +16,7 @@ import fr.insee.genesis.infrastructure.utils.FileUtils; import fr.insee.genesis.stubs.ConfigStub; import fr.insee.genesis.stubs.LunaticJsonRawDataPersistanceStub; +import fr.insee.genesis.stubs.SurveyUnitQualityToolPerretAdapterStub; import fr.insee.genesis.stubs.SurveyUnitPersistencePortStub; import org.assertj.core.api.Assertions; import org.junit.jupiter.api.Test; @@ -29,12 +31,15 @@ class RawResponseControllerTest { private final FileUtils fileUtils = new FileUtils(new ConfigStub()); private final LunaticJsonRawDataPersistanceStub lunaticJsonRawDataPersistanceStub = new LunaticJsonRawDataPersistanceStub(); private final SurveyUnitPersistencePortStub surveyUnitPersistencePortStub = new SurveyUnitPersistencePortStub(); + private final SurveyUnitQualityToolPerretAdapterStub surveyUnitQualityToolPerretAdapterStub = new SurveyUnitQualityToolPerretAdapterStub(); private final LunaticJsonRawDataApiPort lunaticJsonRawDataApiPort = new LunaticJsonRawDataService(lunaticJsonRawDataPersistanceStub, new ControllerUtils(fileUtils), new MetadataService(), new SurveyUnitService(surveyUnitPersistencePortStub, new MetadataService(), fileUtils), new SurveyUnitQualityService(), - fileUtils); + fileUtils, + surveyUnitQualityToolPerretAdapterStub + ); private final RawResponseController rawResponseController = new RawResponseController(lunaticJsonRawDataApiPort); @@ -118,6 +123,7 @@ void processJsonRawDataTest(){ //GIVEN lunaticJsonRawDataPersistanceStub.getMongoStub().clear(); surveyUnitPersistencePortStub.getMongoStub().clear(); + surveyUnitQualityToolPerretAdapterStub.getReceivedMaps().clear(); String campaignId = "SAMPLETEST-PARADATA-v2"; String questionnaireId = campaignId + "_quest"; String interrogationId = "testinterrogationId1"; @@ -150,6 +156,12 @@ void processJsonRawDataTest(){ //Process date check Assertions.assertThat(lunaticJsonRawDataPersistanceStub.getMongoStub().getFirst().processDate()).isNotNull(); + //Perret call check + Assertions.assertThat(surveyUnitQualityToolPerretAdapterStub.getReceivedMaps()) + .hasSize(1); + Assertions.assertThat(surveyUnitQualityToolPerretAdapterStub.getReceivedMaps().getFirst()).containsKey(questionnaireId); + Assertions.assertThat(surveyUnitQualityToolPerretAdapterStub.getReceivedMaps().getFirst().get(questionnaireId)) + .contains(interrogationId); } diff --git a/src/test/java/fr/insee/genesis/domain/service/rawdata/LunaticJsonRawDataServiceTest.java b/src/test/java/fr/insee/genesis/domain/service/rawdata/LunaticJsonRawDataServiceTest.java index 5886bea7a..c77d40881 100644 --- a/src/test/java/fr/insee/genesis/domain/service/rawdata/LunaticJsonRawDataServiceTest.java +++ b/src/test/java/fr/insee/genesis/domain/service/rawdata/LunaticJsonRawDataServiceTest.java @@ -1,6 +1,7 @@ package fr.insee.genesis.domain.service.rawdata; import fr.insee.bpm.metadata.model.VariablesMap; +import fr.insee.genesis.Constants; import fr.insee.genesis.controller.services.MetadataService; import fr.insee.genesis.controller.utils.ControllerUtils; import fr.insee.genesis.domain.model.surveyunit.DataState; @@ -16,7 +17,9 @@ import fr.insee.genesis.stubs.ConfigStub; import fr.insee.genesis.stubs.LunaticJsonRawDataPersistanceStub; import fr.insee.genesis.stubs.SurveyUnitPersistencePortStub; +import fr.insee.genesis.stubs.SurveyUnitQualityToolPerretAdapterStub; import org.assertj.core.api.Assertions; +import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.ValueSource; @@ -34,10 +37,10 @@ class LunaticJsonRawDataServiceTest { MetadataService metadataService = new MetadataService(); SurveyUnitPersistencePortStub surveyUnitPersistencePortStub = new SurveyUnitPersistencePortStub(); + SurveyUnitQualityToolPerretAdapterStub surveyUnitQualityToolPerretAdapterStub = new SurveyUnitQualityToolPerretAdapterStub(); SurveyUnitService surveyUnitService = new SurveyUnitService(surveyUnitPersistencePortStub, metadataService, fileUtils); SurveyUnitQualityService surveyUnitQualityService = new SurveyUnitQualityService(); - - LunaticJsonRawDataService lunaticJsonRawDataService = new LunaticJsonRawDataService(lunaticJsonRawDataPersistanceStub,controllerUtils,metadataService,surveyUnitService,surveyUnitQualityService,fileUtils); + LunaticJsonRawDataService lunaticJsonRawDataService = new LunaticJsonRawDataService(lunaticJsonRawDataPersistanceStub,controllerUtils,metadataService,surveyUnitService,surveyUnitQualityService,fileUtils,surveyUnitQualityToolPerretAdapterStub); @Test void saveDataTest_valid_only_collected_array() throws Exception { @@ -403,12 +406,13 @@ void convertRawData_if_collected_not_present_test() throws Exception { void convertRawData_multipleBatchs(int rawDataSize) throws Exception { //CLEAN surveyUnitPersistencePortStub.getMongoStub().clear(); + surveyUnitQualityToolPerretAdapterStub.getReceivedMaps().clear(); //GIVEN List interrogationIdList = new ArrayList<>(); String campaignId = "SAMPLETEST-PARADATA-v1"; + String questionnaireId = "TESTIDQUEST"; for (int i = 0; i < rawDataSize; i++) { - String questionnaireId = "TESTIDQUEST"; String interrogationId = "TESTinterrogationId" + (i + 1); String json = "{\"EXTERNAL\": {\"RPPRENOM\": \"TEST_EXT%d\"}, ".formatted(i) + "\"COLLECTED\": {\"PRENOMREP\": {\"COLLECTED\": [\"test%d\"], \"EDITED\": [\"test_ed%d\"]}}}" @@ -434,5 +438,10 @@ void convertRawData_multipleBatchs(int rawDataSize) throws Exception { //THEN Assertions.assertThat(dataProcessResult.dataCount()).isEqualTo(rawDataSize * 2/*EDITED*/); Assertions.assertThat(surveyUnitPersistencePortStub.getMongoStub()).hasSize(rawDataSize * 2); + Assertions.assertThat(surveyUnitQualityToolPerretAdapterStub.getReceivedMaps()) + .hasSize(Math.ceilDiv(rawDataSize,Constants.RAW_DATA_PROCESSING_BATCH_SIZE)); + Assertions.assertThat(surveyUnitQualityToolPerretAdapterStub.getReceivedMaps().getFirst()).containsKey(questionnaireId); + Assertions.assertThat(surveyUnitQualityToolPerretAdapterStub.getReceivedMaps().getFirst().get(questionnaireId)) + .contains("TESTinterrogationId1"); } } \ No newline at end of file diff --git a/src/test/java/fr/insee/genesis/stubs/SurveyUnitQualityToolPerretAdapterStub.java b/src/test/java/fr/insee/genesis/stubs/SurveyUnitQualityToolPerretAdapterStub.java new file mode 100644 index 000000000..683c72b5d --- /dev/null +++ b/src/test/java/fr/insee/genesis/stubs/SurveyUnitQualityToolPerretAdapterStub.java @@ -0,0 +1,23 @@ +package fr.insee.genesis.stubs; + +import fr.insee.genesis.domain.ports.spi.SurveyUnitQualityToolPort; +import lombok.Getter; +import lombok.Setter; +import org.springframework.http.ResponseEntity; + +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.Set; + +@Getter +@Setter +public class SurveyUnitQualityToolPerretAdapterStub implements SurveyUnitQualityToolPort { + List>> receivedMaps = new ArrayList<>(); + + @Override + public ResponseEntity sendProcessedIds(Map> processedIdsMap) { + receivedMaps.add(processedIdsMap); + return ResponseEntity.ok().build(); + } +} From 9920ac2bad6a8e29ee490586777947da2733fa4f Mon Sep 17 00:00:00 2001 From: QDIBYS Date: Wed, 18 Jun 2025 11:36:23 +0200 Subject: [PATCH 02/12] chore: add path resolving placeholder --- .../infrastructure/adapter/SurveyQualityToolPerretAdapter.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/main/java/fr/insee/genesis/infrastructure/adapter/SurveyQualityToolPerretAdapter.java b/src/main/java/fr/insee/genesis/infrastructure/adapter/SurveyQualityToolPerretAdapter.java index 4cfcfd140..342e313be 100644 --- a/src/main/java/fr/insee/genesis/infrastructure/adapter/SurveyQualityToolPerretAdapter.java +++ b/src/main/java/fr/insee/genesis/infrastructure/adapter/SurveyQualityToolPerretAdapter.java @@ -24,6 +24,8 @@ public class SurveyQualityToolPerretAdapter implements SurveyUnitQualityToolPort @Override public ResponseEntity sendProcessedIds(Map> processedIdsMap) { + String url = String.format("%s/%s",config.getSurveyQualityToolUrl(), ""); //TODO Add the real path once service is set up + return HttpUtils.makeApiCall(config.getSurveyQualityToolUrl(), HttpMethod.POST, processedIdsMap, Object.class); } } From a1561e89eec0c8e54ef898f331c57989c13fd4e3 Mon Sep 17 00:00:00 2001 From: QDIBYS Date: Wed, 18 Jun 2025 13:06:06 +0200 Subject: [PATCH 03/12] feat: raw data processing batch size property --- src/main/java/fr/insee/genesis/Constants.java | 2 -- .../java/fr/insee/genesis/configuration/Config.java | 3 +++ .../service/rawdata/LunaticJsonRawDataService.java | 10 +++++++--- .../cucumber/functional_tests/RawDataDefinitions.java | 4 +++- .../genesis/controller/rest/UtilsControllerTest.java | 5 ++++- .../rest/responses/RawResponseControllerTest.java | 4 +++- .../service/rawdata/LunaticJsonRawDataServiceTest.java | 8 ++++---- src/test/java/fr/insee/genesis/stubs/ConfigStub.java | 5 +++++ 8 files changed, 29 insertions(+), 12 deletions(-) diff --git a/src/main/java/fr/insee/genesis/Constants.java b/src/main/java/fr/insee/genesis/Constants.java index ce9c5aaa5..7d2122a3f 100644 --- a/src/main/java/fr/insee/genesis/Constants.java +++ b/src/main/java/fr/insee/genesis/Constants.java @@ -61,8 +61,6 @@ public class Constants { // Kraftwerk service path parameters public static final String KRAFTWERK_MAIN_ENDPOINT = ""; - public static final int RAW_DATA_PROCESSING_BATCH_SIZE = 1000; - public static String[] getEnoVariables() { return ENO_VARIABLES; } diff --git a/src/main/java/fr/insee/genesis/configuration/Config.java b/src/main/java/fr/insee/genesis/configuration/Config.java index b90e95fda..068b28bd6 100644 --- a/src/main/java/fr/insee/genesis/configuration/Config.java +++ b/src/main/java/fr/insee/genesis/configuration/Config.java @@ -40,6 +40,9 @@ public class Config { @Value("#{'${fr.insee.genesis.security.whitelist-matchers}'.split(',')}") private String[] whiteList; + @Value("${fr.insee.genesis.rawdata.process.batch-size}") + private int rawDataProcessingBatchSize; + @Value("${fr.insee.genesis.survey-quality-tool.url}") private String surveyQualityToolUrl; diff --git a/src/main/java/fr/insee/genesis/domain/service/rawdata/LunaticJsonRawDataService.java b/src/main/java/fr/insee/genesis/domain/service/rawdata/LunaticJsonRawDataService.java index 0131cdb6c..91cb79ae1 100644 --- a/src/main/java/fr/insee/genesis/domain/service/rawdata/LunaticJsonRawDataService.java +++ b/src/main/java/fr/insee/genesis/domain/service/rawdata/LunaticJsonRawDataService.java @@ -2,6 +2,7 @@ import fr.insee.bpm.metadata.model.VariablesMap; import fr.insee.genesis.Constants; +import fr.insee.genesis.configuration.Config; import fr.insee.genesis.controller.dto.rawdata.LunaticJsonRawDataUnprocessedDto; import fr.insee.genesis.controller.services.MetadataService; import fr.insee.genesis.controller.utils.ControllerUtils; @@ -46,6 +47,7 @@ public class LunaticJsonRawDataService implements LunaticJsonRawDataApiPort { private final SurveyUnitQualityService surveyUnitQualityService; private final SurveyUnitQualityToolPort surveyUnitQualityToolPort; private final FileUtils fileUtils; + private final Config config; @Qualifier("lunaticJsonMongoAdapterNew") private final LunaticJsonRawDataPersistencePort lunaticJsonRawDataPersistencePort; @@ -57,7 +59,8 @@ public LunaticJsonRawDataService(LunaticJsonRawDataPersistencePort lunaticJsonRa SurveyUnitService surveyUnitService, SurveyUnitQualityService surveyUnitQualityService, FileUtils fileUtils, - SurveyUnitQualityToolPort surveyUnitQualityToolPort + SurveyUnitQualityToolPort surveyUnitQualityToolPort, + Config config ) { this.controllerUtils = controllerUtils; this.metadataService = metadataService; @@ -66,6 +69,7 @@ public LunaticJsonRawDataService(LunaticJsonRawDataPersistencePort lunaticJsonRa this.fileUtils = fileUtils; this.lunaticJsonRawDataPersistencePort = lunaticJsonRawDataNewPersistencePort; this.surveyUnitQualityToolPort = surveyUnitQualityToolPort; + this.config = config; } @Override @@ -93,12 +97,12 @@ public DataProcessResult processRawData(String campaignName, List interr .formatted(mode, errors.getLast().getMessage()) ); } - int totalBatchs = Math.ceilDiv(interrogationIdList.size() , Constants.RAW_DATA_PROCESSING_BATCH_SIZE); + int totalBatchs = Math.ceilDiv(interrogationIdList.size() , config.getRawDataProcessingBatchSize()); int batchNumber = 1; List interrogationIdListForMode = new ArrayList<>(interrogationIdList); while(!interrogationIdListForMode.isEmpty()){ log.info("Processing raw data batch {}/{}", batchNumber, totalBatchs); - int maxIndex = Math.min(interrogationIdListForMode.size(), Constants.RAW_DATA_PROCESSING_BATCH_SIZE); + int maxIndex = Math.min(interrogationIdListForMode.size(), config.getRawDataProcessingBatchSize()); List interrogationIdToProcess = interrogationIdListForMode.subList(0, maxIndex); List rawData = getRawData(campaignName,mode, interrogationIdToProcess); diff --git a/src/test/java/cucumber/functional_tests/RawDataDefinitions.java b/src/test/java/cucumber/functional_tests/RawDataDefinitions.java index 792da6499..8f8e42a9d 100644 --- a/src/test/java/cucumber/functional_tests/RawDataDefinitions.java +++ b/src/test/java/cucumber/functional_tests/RawDataDefinitions.java @@ -68,7 +68,9 @@ public class RawDataDefinitions { SurveyUnitQualityService surveyUnitQualityService = new SurveyUnitQualityService(); SurveyUnitQualityToolPerretAdapterStub surveyUnitQualityToolPerretAdapterStub = new SurveyUnitQualityToolPerretAdapterStub(); - LunaticJsonRawDataService lunaticJsonRawDataService = new LunaticJsonRawDataService(lunaticJsonRawDataPersistanceStub,controllerUtils,metadataService,surveyUnitService,surveyUnitQualityService,fileUtils, surveyUnitQualityToolPerretAdapterStub); + LunaticJsonRawDataService lunaticJsonRawDataService = + new LunaticJsonRawDataService(lunaticJsonRawDataPersistanceStub,controllerUtils,metadataService, + surveyUnitService,surveyUnitQualityService,fileUtils, surveyUnitQualityToolPerretAdapterStub,config); RawResponseController rawResponseController = new RawResponseController( lunaticJsonRawDataService ); diff --git a/src/test/java/fr/insee/genesis/controller/rest/UtilsControllerTest.java b/src/test/java/fr/insee/genesis/controller/rest/UtilsControllerTest.java index a3fe67515..8605e60a2 100644 --- a/src/test/java/fr/insee/genesis/controller/rest/UtilsControllerTest.java +++ b/src/test/java/fr/insee/genesis/controller/rest/UtilsControllerTest.java @@ -60,7 +60,10 @@ static void init() { surveyUnitPersistencePortStub = new SurveyUnitPersistencePortStub(); lunaticJsonRawDataPersistencePort = new LunaticJsonRawDataPersistanceStub(); SurveyUnitApiPort surveyUnitApiPort = new SurveyUnitService(surveyUnitPersistencePortStub, metadataService, fileUtils); - LunaticJsonRawDataApiPort lunaticJsonRawDataApiPort = new LunaticJsonRawDataService(lunaticJsonRawDataPersistencePort,controllerUtils,metadataService,surveyUnitService,surveyUnitQualityService,fileUtils, surveyUnitQualityToolPerretAdapterStub); + LunaticJsonRawDataApiPort lunaticJsonRawDataApiPort = + new LunaticJsonRawDataService(lunaticJsonRawDataPersistencePort,controllerUtils,metadataService, + surveyUnitService,surveyUnitQualityService,fileUtils, surveyUnitQualityToolPerretAdapterStub, + new ConfigStub()); utilsControllerStatic = new UtilsController( surveyUnitApiPort diff --git a/src/test/java/fr/insee/genesis/controller/rest/responses/RawResponseControllerTest.java b/src/test/java/fr/insee/genesis/controller/rest/responses/RawResponseControllerTest.java index 1770073e0..e1778bf57 100644 --- a/src/test/java/fr/insee/genesis/controller/rest/responses/RawResponseControllerTest.java +++ b/src/test/java/fr/insee/genesis/controller/rest/responses/RawResponseControllerTest.java @@ -2,6 +2,7 @@ import fr.insee.genesis.Constants; +import fr.insee.genesis.configuration.Config; import fr.insee.genesis.controller.dto.rawdata.LunaticJsonRawDataUnprocessedDto; import fr.insee.genesis.controller.services.MetadataService; import fr.insee.genesis.controller.utils.ControllerUtils; @@ -38,7 +39,8 @@ class RawResponseControllerTest { new SurveyUnitService(surveyUnitPersistencePortStub, new MetadataService(), fileUtils), new SurveyUnitQualityService(), fileUtils, - surveyUnitQualityToolPerretAdapterStub + surveyUnitQualityToolPerretAdapterStub, + new ConfigStub() ); private final RawResponseController rawResponseController = new RawResponseController(lunaticJsonRawDataApiPort); diff --git a/src/test/java/fr/insee/genesis/domain/service/rawdata/LunaticJsonRawDataServiceTest.java b/src/test/java/fr/insee/genesis/domain/service/rawdata/LunaticJsonRawDataServiceTest.java index c77d40881..14e5f3bfa 100644 --- a/src/test/java/fr/insee/genesis/domain/service/rawdata/LunaticJsonRawDataServiceTest.java +++ b/src/test/java/fr/insee/genesis/domain/service/rawdata/LunaticJsonRawDataServiceTest.java @@ -1,7 +1,6 @@ package fr.insee.genesis.domain.service.rawdata; import fr.insee.bpm.metadata.model.VariablesMap; -import fr.insee.genesis.Constants; import fr.insee.genesis.controller.services.MetadataService; import fr.insee.genesis.controller.utils.ControllerUtils; import fr.insee.genesis.domain.model.surveyunit.DataState; @@ -19,7 +18,6 @@ import fr.insee.genesis.stubs.SurveyUnitPersistencePortStub; import fr.insee.genesis.stubs.SurveyUnitQualityToolPerretAdapterStub; import org.assertj.core.api.Assertions; -import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.ValueSource; @@ -40,7 +38,9 @@ class LunaticJsonRawDataServiceTest { SurveyUnitQualityToolPerretAdapterStub surveyUnitQualityToolPerretAdapterStub = new SurveyUnitQualityToolPerretAdapterStub(); SurveyUnitService surveyUnitService = new SurveyUnitService(surveyUnitPersistencePortStub, metadataService, fileUtils); SurveyUnitQualityService surveyUnitQualityService = new SurveyUnitQualityService(); - LunaticJsonRawDataService lunaticJsonRawDataService = new LunaticJsonRawDataService(lunaticJsonRawDataPersistanceStub,controllerUtils,metadataService,surveyUnitService,surveyUnitQualityService,fileUtils,surveyUnitQualityToolPerretAdapterStub); + ConfigStub configStub = new ConfigStub(); + LunaticJsonRawDataService lunaticJsonRawDataService = + new LunaticJsonRawDataService(lunaticJsonRawDataPersistanceStub,controllerUtils,metadataService,surveyUnitService,surveyUnitQualityService,fileUtils,surveyUnitQualityToolPerretAdapterStub,configStub); @Test void saveDataTest_valid_only_collected_array() throws Exception { @@ -439,7 +439,7 @@ void convertRawData_multipleBatchs(int rawDataSize) throws Exception { Assertions.assertThat(dataProcessResult.dataCount()).isEqualTo(rawDataSize * 2/*EDITED*/); Assertions.assertThat(surveyUnitPersistencePortStub.getMongoStub()).hasSize(rawDataSize * 2); Assertions.assertThat(surveyUnitQualityToolPerretAdapterStub.getReceivedMaps()) - .hasSize(Math.ceilDiv(rawDataSize,Constants.RAW_DATA_PROCESSING_BATCH_SIZE)); + .hasSize(Math.ceilDiv(rawDataSize, configStub.getRawDataProcessingBatchSize())); Assertions.assertThat(surveyUnitQualityToolPerretAdapterStub.getReceivedMaps().getFirst()).containsKey(questionnaireId); Assertions.assertThat(surveyUnitQualityToolPerretAdapterStub.getReceivedMaps().getFirst().get(questionnaireId)) .contains("TESTinterrogationId1"); diff --git a/src/test/java/fr/insee/genesis/stubs/ConfigStub.java b/src/test/java/fr/insee/genesis/stubs/ConfigStub.java index 6d4f1c0c5..6ee444722 100644 --- a/src/test/java/fr/insee/genesis/stubs/ConfigStub.java +++ b/src/test/java/fr/insee/genesis/stubs/ConfigStub.java @@ -32,4 +32,9 @@ public String getSpecFolderSource() { public String getAuthType() { return "NONE"; } + + @Override + public int getRawDataProcessingBatchSize(){ + return 1000; + } } From 20df140a3c3dfe7e9cde733023006f4e2543190e Mon Sep 17 00:00:00 2001 From: QDIBYS Date: Wed, 18 Jun 2025 13:09:36 +0200 Subject: [PATCH 04/12] refactor: for each model --- .../rawdata/LunaticJsonRawDataService.java | 15 +++++---------- 1 file changed, 5 insertions(+), 10 deletions(-) diff --git a/src/main/java/fr/insee/genesis/domain/service/rawdata/LunaticJsonRawDataService.java b/src/main/java/fr/insee/genesis/domain/service/rawdata/LunaticJsonRawDataService.java index 91cb79ae1..3aeaefd17 100644 --- a/src/main/java/fr/insee/genesis/domain/service/rawdata/LunaticJsonRawDataService.java +++ b/src/main/java/fr/insee/genesis/domain/service/rawdata/LunaticJsonRawDataService.java @@ -144,16 +144,11 @@ public DataProcessResult processRawData(String campaignName, List interr private Map> getProcessedIdsMap(List surveyUnitModels) { Map> processedInterrogationIdsPerQuestionnaire = new HashMap<>(); - for(SurveyUnitModel surveyUnitModel : surveyUnitModels){ - String questionnaireId = surveyUnitModel.getQuestionnaireId(); - String interrogationId = surveyUnitModel.getInterrogationId(); - - if(!processedInterrogationIdsPerQuestionnaire.containsKey(questionnaireId)){ - processedInterrogationIdsPerQuestionnaire.put(questionnaireId, new HashSet<>()); - } - - processedInterrogationIdsPerQuestionnaire.get(questionnaireId).add(interrogationId); - } + surveyUnitModels.forEach(model -> + processedInterrogationIdsPerQuestionnaire + .computeIfAbsent(model.getQuestionnaireId(), k -> new HashSet<>()) + .add(model.getInterrogationId()) + ); return processedInterrogationIdsPerQuestionnaire; } From 218d386e5ee227186a7e5ece9b7761a56881909c Mon Sep 17 00:00:00 2001 From: QDIBYS Date: Wed, 18 Jun 2025 13:33:11 +0200 Subject: [PATCH 05/12] refactor: use WebClient --- pom.xml | 4 +++ .../insee/genesis/domain/utils/HttpUtils.java | 25 +++++++++++++------ .../SurveyQualityToolPerretAdapter.java | 6 ++--- 3 files changed, 25 insertions(+), 10 deletions(-) diff --git a/pom.xml b/pom.xml index 664e3b2d4..862b56f6c 100644 --- a/pom.xml +++ b/pom.xml @@ -51,6 +51,10 @@ org.springframework.boot spring-boot-starter-oauth2-resource-server + + org.springframework.boot + spring-boot-starter-webflux + org.springframework.boot diff --git a/src/main/java/fr/insee/genesis/domain/utils/HttpUtils.java b/src/main/java/fr/insee/genesis/domain/utils/HttpUtils.java index 19cf14189..2aacdacad 100644 --- a/src/main/java/fr/insee/genesis/domain/utils/HttpUtils.java +++ b/src/main/java/fr/insee/genesis/domain/utils/HttpUtils.java @@ -8,15 +8,26 @@ import org.springframework.http.MediaType; import org.springframework.http.ResponseEntity; import org.springframework.web.client.RestTemplate; +import org.springframework.web.reactive.function.client.WebClient; +import reactor.core.publisher.Mono; + +import java.util.Map; @UtilityClass public class HttpUtils { - public static ResponseEntity makeApiCall(String url, HttpMethod method, T requestBody, Class responseType){ - RestTemplate restTemplate = new RestTemplateBuilder().build(); - HttpHeaders headers = new HttpHeaders(); - headers.setContentType(MediaType.APPLICATION_JSON); - HttpEntity requestEntity = new HttpEntity<>(requestBody, headers); - - return restTemplate.exchange(url, method, requestEntity, responseType); + public static ResponseEntity makeApiCall(String baseUrl, + String path, + HttpMethod method, + T requestBody, + Class responseType){ + WebClient webClient = WebClient.create(baseUrl); + WebClient.ResponseSpec responseSpec = webClient + .method(method) + .uri(path) + .bodyValue(requestBody) + .header(HttpHeaders.CONTENT_TYPE, MediaType.APPLICATION_JSON_VALUE) + .retrieve(); + Mono> response = responseSpec.toEntity(responseType); + return response.block(); } } diff --git a/src/main/java/fr/insee/genesis/infrastructure/adapter/SurveyQualityToolPerretAdapter.java b/src/main/java/fr/insee/genesis/infrastructure/adapter/SurveyQualityToolPerretAdapter.java index 342e313be..6440a2930 100644 --- a/src/main/java/fr/insee/genesis/infrastructure/adapter/SurveyQualityToolPerretAdapter.java +++ b/src/main/java/fr/insee/genesis/infrastructure/adapter/SurveyQualityToolPerretAdapter.java @@ -24,8 +24,8 @@ public class SurveyQualityToolPerretAdapter implements SurveyUnitQualityToolPort @Override public ResponseEntity sendProcessedIds(Map> processedIdsMap) { - String url = String.format("%s/%s",config.getSurveyQualityToolUrl(), ""); //TODO Add the real path once service is set up - - return HttpUtils.makeApiCall(config.getSurveyQualityToolUrl(), HttpMethod.POST, processedIdsMap, Object.class); + //TODO Add the real path once service is set up + return HttpUtils.makeApiCall(config.getSurveyQualityToolUrl(), "TODO",HttpMethod.POST, processedIdsMap, + Object.class); } } From 1f22e7c0c4f763c7cd3ec387b0e88ff389be4195 Mon Sep 17 00:00:00 2001 From: QDIBYS Date: Wed, 18 Jun 2025 13:49:54 +0200 Subject: [PATCH 06/12] fix: missing property in test --- src/main/resources/application-test-cucumber.properties | 3 ++- src/main/resources/application-test.properties | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/src/main/resources/application-test-cucumber.properties b/src/main/resources/application-test-cucumber.properties index 0bd1334ac..6e372598e 100644 --- a/src/main/resources/application-test-cucumber.properties +++ b/src/main/resources/application-test-cucumber.properties @@ -17,4 +17,5 @@ springdoc.swagger-ui.oauth.client-id=client-id-test fr.insee.genesis.authentication = NONE -fr.insee.genesis.survey-quality-tool.url=*** \ No newline at end of file +fr.insee.genesis.survey-quality-tool.url=*** +fr.insee.genesis.rawdata.process.batch-size=1000 \ No newline at end of file diff --git a/src/main/resources/application-test.properties b/src/main/resources/application-test.properties index ab67d3a65..378436c77 100644 --- a/src/main/resources/application-test.properties +++ b/src/main/resources/application-test.properties @@ -18,4 +18,5 @@ springdoc.swagger-ui.oauth.client-id=client-id-test fr.insee.genesis.authentication = OIDC -fr.insee.genesis.survey-quality-tool.url=*** \ No newline at end of file +fr.insee.genesis.survey-quality-tool.url=*** +fr.insee.genesis.rawdata.process.batch-size=1000 \ No newline at end of file From 8b581fb0ce48d20a0960b793ec3aa1fed1a7714f Mon Sep 17 00:00:00 2001 From: QDIBYS Date: Fri, 20 Jun 2025 10:25:36 +0200 Subject: [PATCH 07/12] feat: don't send ids if review false --- .../rawdata/LunaticJsonRawDataService.java | 21 ++-- .../SurveyQualityToolPerretAdapter.java | 2 +- .../functional_tests/RawDataDefinitions.java | 19 +++- .../controller/rest/UtilsControllerTest.java | 16 +++- .../responses/RawResponseControllerTest.java | 22 ++++- .../LunaticJsonRawDataServiceTest.java | 96 +++++++++++++++---- 6 files changed, 142 insertions(+), 34 deletions(-) diff --git a/src/main/java/fr/insee/genesis/domain/service/rawdata/LunaticJsonRawDataService.java b/src/main/java/fr/insee/genesis/domain/service/rawdata/LunaticJsonRawDataService.java index 3aeaefd17..044bb454d 100644 --- a/src/main/java/fr/insee/genesis/domain/service/rawdata/LunaticJsonRawDataService.java +++ b/src/main/java/fr/insee/genesis/domain/service/rawdata/LunaticJsonRawDataService.java @@ -6,6 +6,7 @@ import fr.insee.genesis.controller.dto.rawdata.LunaticJsonRawDataUnprocessedDto; import fr.insee.genesis.controller.services.MetadataService; import fr.insee.genesis.controller.utils.ControllerUtils; +import fr.insee.genesis.domain.model.context.DataProcessingContextModel; import fr.insee.genesis.domain.model.surveyunit.DataState; import fr.insee.genesis.domain.model.surveyunit.Mode; import fr.insee.genesis.domain.model.surveyunit.SurveyUnitModel; @@ -16,6 +17,7 @@ import fr.insee.genesis.domain.ports.api.LunaticJsonRawDataApiPort; import fr.insee.genesis.domain.ports.spi.LunaticJsonRawDataPersistencePort; import fr.insee.genesis.domain.ports.spi.SurveyUnitQualityToolPort; +import fr.insee.genesis.domain.service.context.DataProcessingContextService; import fr.insee.genesis.domain.service.surveyunit.SurveyUnitQualityService; import fr.insee.genesis.domain.service.surveyunit.SurveyUnitService; import fr.insee.genesis.domain.utils.GroupUtils; @@ -46,6 +48,7 @@ public class LunaticJsonRawDataService implements LunaticJsonRawDataApiPort { private final SurveyUnitService surveyUnitService; private final SurveyUnitQualityService surveyUnitQualityService; private final SurveyUnitQualityToolPort surveyUnitQualityToolPort; + private final DataProcessingContextService dataProcessingContextService; private final FileUtils fileUtils; private final Config config; @@ -59,6 +62,7 @@ public LunaticJsonRawDataService(LunaticJsonRawDataPersistencePort lunaticJsonRa SurveyUnitService surveyUnitService, SurveyUnitQualityService surveyUnitQualityService, FileUtils fileUtils, + DataProcessingContextService dataProcessingContextService, SurveyUnitQualityToolPort surveyUnitQualityToolPort, Config config ) { @@ -69,6 +73,7 @@ public LunaticJsonRawDataService(LunaticJsonRawDataPersistencePort lunaticJsonRa this.fileUtils = fileUtils; this.lunaticJsonRawDataPersistencePort = lunaticJsonRawDataNewPersistencePort; this.surveyUnitQualityToolPort = surveyUnitQualityToolPort; + this.dataProcessingContextService = dataProcessingContextService; this.config = config; } @@ -86,6 +91,8 @@ public List getRawData(String campaignName, Mode mode, public DataProcessResult processRawData(String campaignName, List interrogationIdList, List errors) throws GenesisException { int dataCount=0; int formattedDataCount=0; + DataProcessingContextModel dataProcessingContext = + dataProcessingContextService.getContextByPartitionId(campaignName); List modesList = controllerUtils.getModesList(campaignName, null); for (Mode mode : modesList) { //Load and save metadata into database, throw exception if none @@ -125,12 +132,14 @@ public DataProcessResult processRawData(String campaignName, List interr surveyUnitModel -> surveyUnitModel.getState().equals(DataState.FORMATTED) ).toList().size(); - //Send processed ids grouped by questionnaire - ResponseEntity response = - surveyUnitQualityToolPort.sendProcessedIds(getProcessedIdsMap(surveyUnitModels)); - if(!response.getStatusCode().is2xxSuccessful()){ - log.warn("Survey unit quality tool responded non-2xx code {} and body {}", - response.getStatusCode(), response.getBody()); + //Send processed ids grouped by questionnaire (if review activated) + if(dataProcessingContext != null && dataProcessingContext.isWithReview()) { + ResponseEntity response = + surveyUnitQualityToolPort.sendProcessedIds(getProcessedIdsMap(surveyUnitModels)); + if (!response.getStatusCode().is2xxSuccessful()) { + log.warn("Survey unit quality tool responded non-2xx code {} and body {}", + response.getStatusCode(), response.getBody()); + } } //Remove processed ids from list diff --git a/src/main/java/fr/insee/genesis/infrastructure/adapter/SurveyQualityToolPerretAdapter.java b/src/main/java/fr/insee/genesis/infrastructure/adapter/SurveyQualityToolPerretAdapter.java index 6440a2930..debcf3a3a 100644 --- a/src/main/java/fr/insee/genesis/infrastructure/adapter/SurveyQualityToolPerretAdapter.java +++ b/src/main/java/fr/insee/genesis/infrastructure/adapter/SurveyQualityToolPerretAdapter.java @@ -25,7 +25,7 @@ public class SurveyQualityToolPerretAdapter implements SurveyUnitQualityToolPort @Override public ResponseEntity sendProcessedIds(Map> processedIdsMap) { //TODO Add the real path once service is set up - return HttpUtils.makeApiCall(config.getSurveyQualityToolUrl(), "TODO",HttpMethod.POST, processedIdsMap, + return HttpUtils.makeApiCall(config.getSurveyQualityToolUrl(), "/interrogations/",HttpMethod.POST, processedIdsMap, Object.class); } } diff --git a/src/test/java/cucumber/functional_tests/RawDataDefinitions.java b/src/test/java/cucumber/functional_tests/RawDataDefinitions.java index 8f8e42a9d..eac22e0f4 100644 --- a/src/test/java/cucumber/functional_tests/RawDataDefinitions.java +++ b/src/test/java/cucumber/functional_tests/RawDataDefinitions.java @@ -10,12 +10,14 @@ import fr.insee.genesis.domain.model.surveyunit.Mode; import fr.insee.genesis.domain.model.surveyunit.SurveyUnitModel; import fr.insee.genesis.domain.model.surveyunit.VariableModel; +import fr.insee.genesis.domain.service.context.DataProcessingContextService; import fr.insee.genesis.domain.service.rawdata.LunaticJsonRawDataService; import fr.insee.genesis.domain.service.surveyunit.SurveyUnitQualityService; import fr.insee.genesis.domain.service.surveyunit.SurveyUnitService; import fr.insee.genesis.domain.utils.JsonUtils; import fr.insee.genesis.infrastructure.utils.FileUtils; import fr.insee.genesis.stubs.ConfigStub; +import fr.insee.genesis.stubs.DataProcessingContextPersistancePortStub; import fr.insee.genesis.stubs.LunaticJsonRawDataPersistanceStub; import fr.insee.genesis.stubs.SurveyUnitQualityToolPerretAdapterStub; import fr.insee.genesis.stubs.SurveyUnitPersistencePortStub; @@ -66,11 +68,22 @@ public class RawDataDefinitions { SurveyUnitService surveyUnitService = new SurveyUnitService(surveyUnitPersistencePortStub, metadataService, fileUtils); ControllerUtils controllerUtils = new ControllerUtils(fileUtils); SurveyUnitQualityService surveyUnitQualityService = new SurveyUnitQualityService(); - + DataProcessingContextPersistancePortStub dataProcessingContextPersistancePortStub = + new DataProcessingContextPersistancePortStub(); SurveyUnitQualityToolPerretAdapterStub surveyUnitQualityToolPerretAdapterStub = new SurveyUnitQualityToolPerretAdapterStub(); LunaticJsonRawDataService lunaticJsonRawDataService = - new LunaticJsonRawDataService(lunaticJsonRawDataPersistanceStub,controllerUtils,metadataService, - surveyUnitService,surveyUnitQualityService,fileUtils, surveyUnitQualityToolPerretAdapterStub,config); + new LunaticJsonRawDataService( + lunaticJsonRawDataPersistanceStub, + controllerUtils, + metadataService, + surveyUnitService, + surveyUnitQualityService, + fileUtils, + new DataProcessingContextService( + dataProcessingContextPersistancePortStub, + surveyUnitPersistencePortStub), + surveyUnitQualityToolPerretAdapterStub, + config); RawResponseController rawResponseController = new RawResponseController( lunaticJsonRawDataService ); diff --git a/src/test/java/fr/insee/genesis/controller/rest/UtilsControllerTest.java b/src/test/java/fr/insee/genesis/controller/rest/UtilsControllerTest.java index 8605e60a2..9936e683b 100644 --- a/src/test/java/fr/insee/genesis/controller/rest/UtilsControllerTest.java +++ b/src/test/java/fr/insee/genesis/controller/rest/UtilsControllerTest.java @@ -11,12 +11,14 @@ import fr.insee.genesis.domain.model.surveyunit.VariableModel; import fr.insee.genesis.domain.ports.api.LunaticJsonRawDataApiPort; import fr.insee.genesis.domain.ports.api.SurveyUnitApiPort; +import fr.insee.genesis.domain.service.context.DataProcessingContextService; import fr.insee.genesis.domain.service.rawdata.LunaticJsonRawDataService; import fr.insee.genesis.domain.service.surveyunit.SurveyUnitQualityService; import fr.insee.genesis.domain.service.surveyunit.SurveyUnitService; import fr.insee.genesis.domain.service.volumetry.VolumetryLogService; import fr.insee.genesis.infrastructure.utils.FileUtils; import fr.insee.genesis.stubs.ConfigStub; +import fr.insee.genesis.stubs.DataProcessingContextPersistancePortStub; import fr.insee.genesis.stubs.LunaticJsonRawDataPersistanceStub; import fr.insee.genesis.stubs.SurveyUnitQualityToolPerretAdapterStub; import fr.insee.genesis.stubs.SurveyUnitPersistencePortStub; @@ -60,9 +62,17 @@ static void init() { surveyUnitPersistencePortStub = new SurveyUnitPersistencePortStub(); lunaticJsonRawDataPersistencePort = new LunaticJsonRawDataPersistanceStub(); SurveyUnitApiPort surveyUnitApiPort = new SurveyUnitService(surveyUnitPersistencePortStub, metadataService, fileUtils); - LunaticJsonRawDataApiPort lunaticJsonRawDataApiPort = - new LunaticJsonRawDataService(lunaticJsonRawDataPersistencePort,controllerUtils,metadataService, - surveyUnitService,surveyUnitQualityService,fileUtils, surveyUnitQualityToolPerretAdapterStub, + LunaticJsonRawDataApiPort lunaticJsonRawDataApiPort = new LunaticJsonRawDataService( + lunaticJsonRawDataPersistencePort, + controllerUtils, + metadataService, + surveyUnitService, + surveyUnitQualityService, + fileUtils, + new DataProcessingContextService( + new DataProcessingContextPersistancePortStub(), + surveyUnitPersistencePortStub), + surveyUnitQualityToolPerretAdapterStub, new ConfigStub()); utilsControllerStatic = new UtilsController( diff --git a/src/test/java/fr/insee/genesis/controller/rest/responses/RawResponseControllerTest.java b/src/test/java/fr/insee/genesis/controller/rest/responses/RawResponseControllerTest.java index e1778bf57..3b4f48160 100644 --- a/src/test/java/fr/insee/genesis/controller/rest/responses/RawResponseControllerTest.java +++ b/src/test/java/fr/insee/genesis/controller/rest/responses/RawResponseControllerTest.java @@ -1,24 +1,26 @@ package fr.insee.genesis.controller.rest.responses; -import fr.insee.genesis.Constants; -import fr.insee.genesis.configuration.Config; import fr.insee.genesis.controller.dto.rawdata.LunaticJsonRawDataUnprocessedDto; import fr.insee.genesis.controller.services.MetadataService; import fr.insee.genesis.controller.utils.ControllerUtils; +import fr.insee.genesis.domain.model.context.DataProcessingContextModel; import fr.insee.genesis.domain.model.surveyunit.DataState; import fr.insee.genesis.domain.model.surveyunit.Mode; import fr.insee.genesis.domain.ports.api.LunaticJsonRawDataApiPort; +import fr.insee.genesis.domain.service.context.DataProcessingContextService; import fr.insee.genesis.domain.service.rawdata.LunaticJsonRawDataService; import fr.insee.genesis.domain.service.surveyunit.SurveyUnitQualityService; import fr.insee.genesis.domain.service.surveyunit.SurveyUnitService; import fr.insee.genesis.domain.utils.JsonUtils; import fr.insee.genesis.infrastructure.document.rawdata.LunaticJsonRawDataDocument; +import fr.insee.genesis.infrastructure.mappers.DataProcessingContextMapper; import fr.insee.genesis.infrastructure.utils.FileUtils; import fr.insee.genesis.stubs.ConfigStub; +import fr.insee.genesis.stubs.DataProcessingContextPersistancePortStub; import fr.insee.genesis.stubs.LunaticJsonRawDataPersistanceStub; -import fr.insee.genesis.stubs.SurveyUnitQualityToolPerretAdapterStub; import fr.insee.genesis.stubs.SurveyUnitPersistencePortStub; +import fr.insee.genesis.stubs.SurveyUnitQualityToolPerretAdapterStub; import org.assertj.core.api.Assertions; import org.junit.jupiter.api.Test; import org.springframework.http.ResponseEntity; @@ -33,12 +35,15 @@ class RawResponseControllerTest { private final LunaticJsonRawDataPersistanceStub lunaticJsonRawDataPersistanceStub = new LunaticJsonRawDataPersistanceStub(); private final SurveyUnitPersistencePortStub surveyUnitPersistencePortStub = new SurveyUnitPersistencePortStub(); private final SurveyUnitQualityToolPerretAdapterStub surveyUnitQualityToolPerretAdapterStub = new SurveyUnitQualityToolPerretAdapterStub(); + private final DataProcessingContextPersistancePortStub dataProcessingContextPersistancePortStub = + new DataProcessingContextPersistancePortStub(); private final LunaticJsonRawDataApiPort lunaticJsonRawDataApiPort = new LunaticJsonRawDataService(lunaticJsonRawDataPersistanceStub, new ControllerUtils(fileUtils), new MetadataService(), new SurveyUnitService(surveyUnitPersistencePortStub, new MetadataService(), fileUtils), new SurveyUnitQualityService(), fileUtils, + new DataProcessingContextService(dataProcessingContextPersistancePortStub, surveyUnitPersistencePortStub), surveyUnitQualityToolPerretAdapterStub, new ConfigStub() ); @@ -133,6 +138,17 @@ void processJsonRawDataTest(){ String varValue = "TEST"; addJsonRawDataDocumentToStub(campaignId, questionnaireId, interrogationId, null, varName, varValue); + dataProcessingContextPersistancePortStub.getMongoStub().add( + DataProcessingContextMapper.INSTANCE.modelToDocument( + DataProcessingContextModel.builder() + .partitionId(campaignId) + .kraftwerkExecutionScheduleList(new ArrayList<>()) + .withReview(true) + .build() + ) + ); + + List interrogationIdList = new ArrayList<>(); interrogationIdList.add(interrogationId); diff --git a/src/test/java/fr/insee/genesis/domain/service/rawdata/LunaticJsonRawDataServiceTest.java b/src/test/java/fr/insee/genesis/domain/service/rawdata/LunaticJsonRawDataServiceTest.java index 14e5f3bfa..ce3e24a8a 100644 --- a/src/test/java/fr/insee/genesis/domain/service/rawdata/LunaticJsonRawDataServiceTest.java +++ b/src/test/java/fr/insee/genesis/domain/service/rawdata/LunaticJsonRawDataServiceTest.java @@ -1,19 +1,24 @@ package fr.insee.genesis.domain.service.rawdata; +import com.fasterxml.jackson.core.JsonProcessingException; import fr.insee.bpm.metadata.model.VariablesMap; import fr.insee.genesis.controller.services.MetadataService; import fr.insee.genesis.controller.utils.ControllerUtils; +import fr.insee.genesis.domain.model.context.DataProcessingContextModel; import fr.insee.genesis.domain.model.surveyunit.DataState; import fr.insee.genesis.domain.model.surveyunit.Mode; import fr.insee.genesis.domain.model.surveyunit.SurveyUnitModel; import fr.insee.genesis.domain.model.surveyunit.rawdata.DataProcessResult; import fr.insee.genesis.domain.model.surveyunit.rawdata.LunaticJsonRawDataModel; +import fr.insee.genesis.domain.service.context.DataProcessingContextService; import fr.insee.genesis.domain.service.surveyunit.SurveyUnitQualityService; import fr.insee.genesis.domain.service.surveyunit.SurveyUnitService; import fr.insee.genesis.domain.utils.JsonUtils; +import fr.insee.genesis.infrastructure.mappers.DataProcessingContextMapper; import fr.insee.genesis.infrastructure.mappers.LunaticJsonRawDataDocumentMapper; import fr.insee.genesis.infrastructure.utils.FileUtils; import fr.insee.genesis.stubs.ConfigStub; +import fr.insee.genesis.stubs.DataProcessingContextPersistancePortStub; import fr.insee.genesis.stubs.LunaticJsonRawDataPersistanceStub; import fr.insee.genesis.stubs.SurveyUnitPersistencePortStub; import fr.insee.genesis.stubs.SurveyUnitQualityToolPerretAdapterStub; @@ -36,11 +41,21 @@ class LunaticJsonRawDataServiceTest { SurveyUnitPersistencePortStub surveyUnitPersistencePortStub = new SurveyUnitPersistencePortStub(); SurveyUnitQualityToolPerretAdapterStub surveyUnitQualityToolPerretAdapterStub = new SurveyUnitQualityToolPerretAdapterStub(); - SurveyUnitService surveyUnitService = new SurveyUnitService(surveyUnitPersistencePortStub, metadataService, fileUtils); - SurveyUnitQualityService surveyUnitQualityService = new SurveyUnitQualityService(); + DataProcessingContextPersistancePortStub dataProcessingContextPersistancePortStub = + new DataProcessingContextPersistancePortStub(); ConfigStub configStub = new ConfigStub(); LunaticJsonRawDataService lunaticJsonRawDataService = - new LunaticJsonRawDataService(lunaticJsonRawDataPersistanceStub,controllerUtils,metadataService,surveyUnitService,surveyUnitQualityService,fileUtils,surveyUnitQualityToolPerretAdapterStub,configStub); + new LunaticJsonRawDataService( + lunaticJsonRawDataPersistanceStub, + controllerUtils, + metadataService, + new SurveyUnitService(surveyUnitPersistencePortStub, metadataService, fileUtils), + new SurveyUnitQualityService(), + fileUtils, + new DataProcessingContextService(dataProcessingContextPersistancePortStub, surveyUnitPersistencePortStub), + surveyUnitQualityToolPerretAdapterStub, + configStub + ); @Test void saveDataTest_valid_only_collected_array() throws Exception { @@ -404,14 +419,71 @@ void convertRawData_if_collected_not_present_test() throws Exception { @ParameterizedTest @ValueSource(ints = {5,500,5000,10000}) void convertRawData_multipleBatchs(int rawDataSize) throws Exception { + //GIVEN + String campaignId = "SAMPLETEST-PARADATA-v1"; + String questionnaireId = "TESTIDQUEST"; + List interrogationIdList = prepareConvertTest(rawDataSize, campaignId, questionnaireId); + //Activate review + dataProcessingContextPersistancePortStub.getMongoStub().add( + DataProcessingContextMapper.INSTANCE.modelToDocument( + DataProcessingContextModel.builder() + .partitionId(campaignId) + .withReview(true) + .kraftwerkExecutionScheduleList(new ArrayList<>()) + .build() + ) + ); + + //WHEN + DataProcessResult dataProcessResult = lunaticJsonRawDataService.processRawData(campaignId, interrogationIdList, + new ArrayList<>()); + + //THEN + Assertions.assertThat(dataProcessResult.dataCount()).isEqualTo(rawDataSize * 2/*EDITED*/); + Assertions.assertThat(surveyUnitPersistencePortStub.getMongoStub()).hasSize(rawDataSize * 2); + Assertions.assertThat(surveyUnitQualityToolPerretAdapterStub.getReceivedMaps()) + .hasSize(Math.ceilDiv(rawDataSize, configStub.getRawDataProcessingBatchSize())); + Assertions.assertThat(surveyUnitQualityToolPerretAdapterStub.getReceivedMaps().getFirst()).containsKey(questionnaireId); + Assertions.assertThat(surveyUnitQualityToolPerretAdapterStub.getReceivedMaps().getFirst().get(questionnaireId)) + .contains("TESTinterrogationId1"); + } + + @Test + void convertRawData_review_desactivated() throws Exception { + //GIVEN + String campaignId = "SAMPLETEST-PARADATA-v1"; + String questionnaireId = "TESTIDQUEST"; + List interrogationIdList = prepareConvertTest(1, campaignId, questionnaireId); + + //Desactivate review + dataProcessingContextPersistancePortStub.getMongoStub().add( + DataProcessingContextMapper.INSTANCE.modelToDocument( + DataProcessingContextModel.builder() + .partitionId(campaignId) + .withReview(false) + .kraftwerkExecutionScheduleList(new ArrayList<>()) + .build() + ) + ); + + //WHEN + DataProcessResult dataProcessResult = lunaticJsonRawDataService.processRawData(campaignId, interrogationIdList, + new ArrayList<>()); + + //THEN + Assertions.assertThat(dataProcessResult.dataCount()).isEqualTo(2); + Assertions.assertThat(surveyUnitPersistencePortStub.getMongoStub()).hasSize(2); + Assertions.assertThat(surveyUnitQualityToolPerretAdapterStub.getReceivedMaps()).isEmpty(); + } + + private List prepareConvertTest(int rawDataSize, String campaignId, String questionnaireId) throws JsonProcessingException { //CLEAN surveyUnitPersistencePortStub.getMongoStub().clear(); surveyUnitQualityToolPerretAdapterStub.getReceivedMaps().clear(); + dataProcessingContextPersistancePortStub.getMongoStub().clear(); //GIVEN List interrogationIdList = new ArrayList<>(); - String campaignId = "SAMPLETEST-PARADATA-v1"; - String questionnaireId = "TESTIDQUEST"; for (int i = 0; i < rawDataSize; i++) { String interrogationId = "TESTinterrogationId" + (i + 1); String json = "{\"EXTERNAL\": {\"RPPRENOM\": \"TEST_EXT%d\"}, ".formatted(i) + @@ -430,18 +502,6 @@ void convertRawData_multipleBatchs(int rawDataSize) throws Exception { lunaticJsonRawDataPersistanceStub.getMongoStub() .add(LunaticJsonRawDataDocumentMapper.INSTANCE.modelToDocument(rawDataModel)); } - - //WHEN - DataProcessResult dataProcessResult = lunaticJsonRawDataService.processRawData(campaignId, interrogationIdList, - new ArrayList<>()); - - //THEN - Assertions.assertThat(dataProcessResult.dataCount()).isEqualTo(rawDataSize * 2/*EDITED*/); - Assertions.assertThat(surveyUnitPersistencePortStub.getMongoStub()).hasSize(rawDataSize * 2); - Assertions.assertThat(surveyUnitQualityToolPerretAdapterStub.getReceivedMaps()) - .hasSize(Math.ceilDiv(rawDataSize, configStub.getRawDataProcessingBatchSize())); - Assertions.assertThat(surveyUnitQualityToolPerretAdapterStub.getReceivedMaps().getFirst()).containsKey(questionnaireId); - Assertions.assertThat(surveyUnitQualityToolPerretAdapterStub.getReceivedMaps().getFirst().get(questionnaireId)) - .contains("TESTinterrogationId1"); + return interrogationIdList; } } \ No newline at end of file From 7a76730451eda4ec75aadf1eb93e7adf4df4f7dd Mon Sep 17 00:00:00 2001 From: amarchal-insee Date: Wed, 25 Jun 2025 12:10:34 +0200 Subject: [PATCH 08/12] Update SurveyUnitDocument.java add some indexes in MongoDB responses collection for perf. issues from Perret --- .../document/surveyunit/SurveyUnitDocument.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/main/java/fr/insee/genesis/infrastructure/document/surveyunit/SurveyUnitDocument.java b/src/main/java/fr/insee/genesis/infrastructure/document/surveyunit/SurveyUnitDocument.java index cbd05258e..d67173323 100644 --- a/src/main/java/fr/insee/genesis/infrastructure/document/surveyunit/SurveyUnitDocument.java +++ b/src/main/java/fr/insee/genesis/infrastructure/document/surveyunit/SurveyUnitDocument.java @@ -3,6 +3,7 @@ import fr.insee.genesis.Constants; import lombok.Data; import org.springframework.data.mongodb.core.index.CompoundIndex; +import org.springframework.data.mongodb.core.index.Indexed; import org.springframework.data.mongodb.core.mapping.Document; import java.time.LocalDateTime; @@ -13,13 +14,12 @@ @CompoundIndex(name = "campaignId_1_questionnaireId_1", def = "{'campaignId': 1, 'questionnaireId': 1}") //1 = ascending, -1 = descending @CompoundIndex(name = "questionnaireId_1_campaignId_1", def = "{'questionnaireId': 1, 'campaignId': 1}") @CompoundIndex(name = "questionnaireId_1_interrogationId_1", def = "{'questionnaireId': 1, 'interrogationId': 1}") +@CompoundIndex(name = "interrogationId_1_questionnaireId_1", def = "{'interrogationId': 1, 'questionnaireId': 1}") public class SurveyUnitDocument { private String campaignId; + @Indexed private String interrogationId; private String questionnaireId; - private String contextualId; - private Boolean isCapturedIndirectly; - private LocalDateTime validationDate; private String state; private String mode; private LocalDateTime recordDate; From d8ca43108276ce824f9db0a6f6a56df9e299dcbe Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lo=C3=AFc=20Henninger?= <45652566+loichenninger@users.noreply.github.com> Date: Wed, 25 Jun 2025 13:32:34 +0200 Subject: [PATCH 09/12] fix: remove logger used for debugging --- .../genesis/domain/service/surveyunit/SurveyUnitService.java | 1 - 1 file changed, 1 deletion(-) diff --git a/src/main/java/fr/insee/genesis/domain/service/surveyunit/SurveyUnitService.java b/src/main/java/fr/insee/genesis/domain/service/surveyunit/SurveyUnitService.java index 09d6fd27b..e0ee35fa9 100644 --- a/src/main/java/fr/insee/genesis/domain/service/surveyunit/SurveyUnitService.java +++ b/src/main/java/fr/insee/genesis/domain/service/surveyunit/SurveyUnitService.java @@ -505,7 +505,6 @@ private void extractVariables(SurveyUnitModel surveyUnitModel, surveyUnitModel.setCollectedVariables(new ArrayList<>()); } for (VariableModel collectedVariable : surveyUnitModel.getCollectedVariables()) { - log.info("Variable name : {}", collectedVariable.varId()); VarIdScopeTuple loopIdTuple = new VarIdScopeTuple(collectedVariable.varId(), collectedVariable.scope(), collectedVariable.iteration()); VariableDto variableDto = collectedVariableMap.get(loopIdTuple); From 88756ecf23bf2745a43f933d26321e7c2a3eed3e Mon Sep 17 00:00:00 2001 From: QDIBYS Date: Thu, 26 Jun 2025 11:43:03 +0200 Subject: [PATCH 10/12] Fix: send to Perret with authentification --- .../insee/genesis/configuration/Config.java | 5 ++ .../ports/spi/SurveyUnitQualityToolPort.java | 3 +- .../rawdata/LunaticJsonRawDataService.java | 14 +++- .../SurveyQualityToolPerretAdapter.java | 12 ++- .../utils/http}/HttpUtils.java | 17 +++-- .../utils/http/OidcService.java | 76 +++++++++++++++++++ .../utils/http/PerretAuthWebClientFilter.java | 41 ++++++++++ .../application-test-cucumber.properties | 3 + .../resources/application-test.properties | 3 + 9 files changed, 159 insertions(+), 15 deletions(-) rename src/main/java/fr/insee/genesis/{domain/utils => infrastructure/utils/http}/HttpUtils.java (66%) create mode 100644 src/main/java/fr/insee/genesis/infrastructure/utils/http/OidcService.java create mode 100644 src/main/java/fr/insee/genesis/infrastructure/utils/http/PerretAuthWebClientFilter.java diff --git a/src/main/java/fr/insee/genesis/configuration/Config.java b/src/main/java/fr/insee/genesis/configuration/Config.java index 068b28bd6..74e346dbb 100644 --- a/src/main/java/fr/insee/genesis/configuration/Config.java +++ b/src/main/java/fr/insee/genesis/configuration/Config.java @@ -31,6 +31,11 @@ public class Config { @Value("${fr.insee.genesis.oidc.realm}") private String realm; + @Value("${fr.insee.genesis.oidc.service.client-id}") + private String serviceClientId; + @Value("${fr.insee.genesus.oidc.service.client-secret}") + private String serviceClientSecret; + @Value("${fr.insee.genesis.security.token.oidc-claim-role}") private String oidcClaimRole; diff --git a/src/main/java/fr/insee/genesis/domain/ports/spi/SurveyUnitQualityToolPort.java b/src/main/java/fr/insee/genesis/domain/ports/spi/SurveyUnitQualityToolPort.java index 2c658811f..549e89464 100644 --- a/src/main/java/fr/insee/genesis/domain/ports/spi/SurveyUnitQualityToolPort.java +++ b/src/main/java/fr/insee/genesis/domain/ports/spi/SurveyUnitQualityToolPort.java @@ -2,9 +2,10 @@ import org.springframework.http.ResponseEntity; +import java.io.IOException; import java.util.Map; import java.util.Set; public interface SurveyUnitQualityToolPort { - ResponseEntity sendProcessedIds(Map> processedIdsMap); + ResponseEntity sendProcessedIds(Map> processedIdsMap) throws IOException; } diff --git a/src/main/java/fr/insee/genesis/domain/service/rawdata/LunaticJsonRawDataService.java b/src/main/java/fr/insee/genesis/domain/service/rawdata/LunaticJsonRawDataService.java index 044bb454d..00ba32e3b 100644 --- a/src/main/java/fr/insee/genesis/domain/service/rawdata/LunaticJsonRawDataService.java +++ b/src/main/java/fr/insee/genesis/domain/service/rawdata/LunaticJsonRawDataService.java @@ -31,6 +31,7 @@ import org.springframework.http.ResponseEntity; import org.springframework.stereotype.Service; +import java.io.IOException; import java.time.LocalDateTime; import java.util.ArrayList; import java.util.HashMap; @@ -134,12 +135,19 @@ public DataProcessResult processRawData(String campaignName, List interr //Send processed ids grouped by questionnaire (if review activated) if(dataProcessingContext != null && dataProcessingContext.isWithReview()) { - ResponseEntity response = - surveyUnitQualityToolPort.sendProcessedIds(getProcessedIdsMap(surveyUnitModels)); - if (!response.getStatusCode().is2xxSuccessful()) { + try { + ResponseEntity response = + surveyUnitQualityToolPort.sendProcessedIds(getProcessedIdsMap(surveyUnitModels)); + + if (response.getStatusCode().is2xxSuccessful()) { + log.info("Successfully sent {} ids to quality tool", getProcessedIdsMap(surveyUnitModels).keySet().size()); + }else{ log.warn("Survey unit quality tool responded non-2xx code {} and body {}", response.getStatusCode(), response.getBody()); } + }catch (IOException e){ + log.error("Error during Perret call request building : {}", e.toString()); + } } //Remove processed ids from list diff --git a/src/main/java/fr/insee/genesis/infrastructure/adapter/SurveyQualityToolPerretAdapter.java b/src/main/java/fr/insee/genesis/infrastructure/adapter/SurveyQualityToolPerretAdapter.java index debcf3a3a..38ea3d58a 100644 --- a/src/main/java/fr/insee/genesis/infrastructure/adapter/SurveyQualityToolPerretAdapter.java +++ b/src/main/java/fr/insee/genesis/infrastructure/adapter/SurveyQualityToolPerretAdapter.java @@ -2,7 +2,8 @@ import fr.insee.genesis.configuration.Config; import fr.insee.genesis.domain.ports.spi.SurveyUnitQualityToolPort; -import fr.insee.genesis.domain.utils.HttpUtils; +import fr.insee.genesis.infrastructure.utils.http.OidcService; +import fr.insee.genesis.infrastructure.utils.http.HttpUtils; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Qualifier; @@ -10,6 +11,7 @@ import org.springframework.http.ResponseEntity; import org.springframework.stereotype.Service; +import java.io.IOException; import java.util.Map; import java.util.Set; @@ -21,11 +23,13 @@ public class SurveyQualityToolPerretAdapter implements SurveyUnitQualityToolPort @Autowired Config config; + @Autowired + OidcService oidcService; + @Override - public ResponseEntity sendProcessedIds(Map> processedIdsMap) { - //TODO Add the real path once service is set up + public ResponseEntity sendProcessedIds(Map> processedIdsMap) throws IOException { return HttpUtils.makeApiCall(config.getSurveyQualityToolUrl(), "/interrogations/",HttpMethod.POST, processedIdsMap, - Object.class); + Object.class, oidcService); } } diff --git a/src/main/java/fr/insee/genesis/domain/utils/HttpUtils.java b/src/main/java/fr/insee/genesis/infrastructure/utils/http/HttpUtils.java similarity index 66% rename from src/main/java/fr/insee/genesis/domain/utils/HttpUtils.java rename to src/main/java/fr/insee/genesis/infrastructure/utils/http/HttpUtils.java index 2aacdacad..ebb3382aa 100644 --- a/src/main/java/fr/insee/genesis/domain/utils/HttpUtils.java +++ b/src/main/java/fr/insee/genesis/infrastructure/utils/http/HttpUtils.java @@ -1,17 +1,14 @@ -package fr.insee.genesis.domain.utils; +package fr.insee.genesis.infrastructure.utils.http; import lombok.experimental.UtilityClass; -import org.springframework.boot.web.client.RestTemplateBuilder; -import org.springframework.http.HttpEntity; import org.springframework.http.HttpHeaders; import org.springframework.http.HttpMethod; import org.springframework.http.MediaType; import org.springframework.http.ResponseEntity; -import org.springframework.web.client.RestTemplate; import org.springframework.web.reactive.function.client.WebClient; import reactor.core.publisher.Mono; -import java.util.Map; +import java.io.IOException; @UtilityClass public class HttpUtils { @@ -19,13 +16,19 @@ public static ResponseEntity makeApiCall(String baseUrl, String path, HttpMethod method, T requestBody, - Class responseType){ - WebClient webClient = WebClient.create(baseUrl); + Class responseType, + OidcService oidcService + ) throws IOException { + WebClient webClient = WebClient.builder() + .baseUrl(baseUrl) + .filter(PerretAuthWebClientFilter.perretAuthFilter(oidcService)) + .build(); WebClient.ResponseSpec responseSpec = webClient .method(method) .uri(path) .bodyValue(requestBody) .header(HttpHeaders.CONTENT_TYPE, MediaType.APPLICATION_JSON_VALUE) + .header(HttpHeaders.AUTHORIZATION, "Bearer " + oidcService.getServiceAccountToken()) .retrieve(); Mono> response = responseSpec.toEntity(responseType); return response.block(); diff --git a/src/main/java/fr/insee/genesis/infrastructure/utils/http/OidcService.java b/src/main/java/fr/insee/genesis/infrastructure/utils/http/OidcService.java new file mode 100644 index 000000000..3c1729607 --- /dev/null +++ b/src/main/java/fr/insee/genesis/infrastructure/utils/http/OidcService.java @@ -0,0 +1,76 @@ +package fr.insee.genesis.infrastructure.utils.http; + +import fr.insee.genesis.configuration.Config; +import lombok.Getter; +import lombok.extern.slf4j.Slf4j; +import org.springframework.http.HttpEntity; +import org.springframework.http.HttpHeaders; +import org.springframework.http.MediaType; +import org.springframework.http.ResponseEntity; +import org.springframework.stereotype.Service; +import org.springframework.web.client.HttpStatusCodeException; +import org.springframework.web.client.RestTemplate; + +import java.io.IOException; +import java.util.Map; + +@Service +@Slf4j +public class OidcService { + + public static final String ACCESS_TOKEN = "access_token"; + @Getter + private final Config config; + private String serviceAccountToken; + private long tokenExpirationTime = 0; + private final RestTemplate restTemplate; + + public OidcService(Config config) { + this.restTemplate = new RestTemplate(); + this.config = config; + } + + protected void retrieveServiceAccountToken() throws IOException { + String tokenUrl = String.format("%s/realms/%s/protocol/openid-connect/token", + config.getAuthServerUrl(), + config.getRealm()); + + HttpHeaders headers = new HttpHeaders(); + headers.setContentType(MediaType.APPLICATION_FORM_URLENCODED); + + String body = String.format("grant_type=client_credentials&client_id=%s&client_secret=%s&scope=openid profile roles", + config.getServiceClientId(), + config.getServiceClientSecret()); + + HttpEntity request = new HttpEntity<>(body, headers); + try { + ResponseEntity response = restTemplate.postForEntity(tokenUrl, request, Map.class); + if (response.getStatusCode().is2xxSuccessful() && response.hasBody() && response.getBody() != null) { + Map responseBody = response.getBody(); + + if (responseBody == null || !responseBody.containsKey(ACCESS_TOKEN) || !(responseBody.get(ACCESS_TOKEN) instanceof String)) { + throw new IOException("Invalid response: Missing or incorrect 'access_token'"); + } + + serviceAccountToken = (String) responseBody.get(ACCESS_TOKEN); + Integer expiresIn = (Integer) responseBody.get("expires_in"); + tokenExpirationTime = System.currentTimeMillis() + (expiresIn.longValue() * 1000L); + } else { + throw new IOException("Failed to retrieve service account token, status: " + response.getStatusCode()); + } + } catch (HttpStatusCodeException e) { + throw new IOException("HTTP error: " + e.getStatusCode() + " - " + e.getResponseBodyAsString(), e); + } catch (Exception e) { + throw new IOException("Unexpected error while fetching token: " + e.getMessage(), e); + } + } + + public synchronized String getServiceAccountToken() throws IOException { + //We had a margin of 5 seconds for the expiration time + if (serviceAccountToken == null || System.currentTimeMillis() >= tokenExpirationTime - 5000L) { + retrieveServiceAccountToken(); + } + return serviceAccountToken; + } + +} diff --git a/src/main/java/fr/insee/genesis/infrastructure/utils/http/PerretAuthWebClientFilter.java b/src/main/java/fr/insee/genesis/infrastructure/utils/http/PerretAuthWebClientFilter.java new file mode 100644 index 000000000..c9842068c --- /dev/null +++ b/src/main/java/fr/insee/genesis/infrastructure/utils/http/PerretAuthWebClientFilter.java @@ -0,0 +1,41 @@ +package fr.insee.genesis.infrastructure.utils.http; + +import lombok.extern.slf4j.Slf4j; +import org.springframework.web.reactive.function.client.ClientRequest; +import org.springframework.web.reactive.function.client.ExchangeFilterFunction; +import reactor.core.publisher.Mono; + +import java.io.IOException; + +@Slf4j +public class PerretAuthWebClientFilter { + public static ExchangeFilterFunction perretAuthFilter(OidcService oidcService) { + return ExchangeFilterFunction.ofRequestProcessor(request -> { + try { + String token = oidcService.getServiceAccountToken(); + ClientRequest newRequest = ClientRequest.from(request) + .headers(headers -> headers.setBearerAuth(token)) + .build(); + return Mono.just(newRequest); + } catch (IOException e) { + return Mono.error(new RuntimeException("Failed to get token", e)); + } + }).andThen((request, next) -> next.exchange(request) + .flatMap(response -> { + if (response.statusCode().value() == 401) { + log.warn("Received 401, refreshing token and retrying once"); + try { + oidcService.retrieveServiceAccountToken(); // Force refresh + String refreshedToken = oidcService.getServiceAccountToken(); + ClientRequest retryRequest = ClientRequest.from(request) + .headers(headers -> headers.setBearerAuth(refreshedToken)) + .build(); + return next.exchange(retryRequest); + } catch (IOException e) { + return Mono.error(new RuntimeException("Failed to refresh token", e)); + } + } + return Mono.just(response); + })); + } +} diff --git a/src/main/resources/application-test-cucumber.properties b/src/main/resources/application-test-cucumber.properties index 6e372598e..590d886fb 100644 --- a/src/main/resources/application-test-cucumber.properties +++ b/src/main/resources/application-test-cucumber.properties @@ -18,4 +18,7 @@ springdoc.swagger-ui.oauth.client-id=client-id-test fr.insee.genesis.authentication = NONE fr.insee.genesis.survey-quality-tool.url=*** +fr.insee.genesis.oidc.service.client-id=*** +fr.insee.genesus.oidc.service.client-secret=*** + fr.insee.genesis.rawdata.process.batch-size=1000 \ No newline at end of file diff --git a/src/main/resources/application-test.properties b/src/main/resources/application-test.properties index 378436c77..f4e3d5daf 100644 --- a/src/main/resources/application-test.properties +++ b/src/main/resources/application-test.properties @@ -19,4 +19,7 @@ springdoc.swagger-ui.oauth.client-id=client-id-test fr.insee.genesis.authentication = OIDC fr.insee.genesis.survey-quality-tool.url=*** +fr.insee.genesis.oidc.service.client-id=*** +fr.insee.genesus.oidc.service.client-secret=*** +private String serviceClientSecret; fr.insee.genesis.rawdata.process.batch-size=1000 \ No newline at end of file From 3194bb8a90a1b4e8822087173a19c17b4cc35dd9 Mon Sep 17 00:00:00 2001 From: QDIBYS Date: Thu, 26 Jun 2025 15:21:42 +0200 Subject: [PATCH 11/12] fix: not sus --- src/main/java/fr/insee/genesis/configuration/Config.java | 2 +- src/main/resources/application-test-cucumber.properties | 2 +- src/main/resources/application-test.properties | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/main/java/fr/insee/genesis/configuration/Config.java b/src/main/java/fr/insee/genesis/configuration/Config.java index 74e346dbb..9b084caef 100644 --- a/src/main/java/fr/insee/genesis/configuration/Config.java +++ b/src/main/java/fr/insee/genesis/configuration/Config.java @@ -33,7 +33,7 @@ public class Config { @Value("${fr.insee.genesis.oidc.service.client-id}") private String serviceClientId; - @Value("${fr.insee.genesus.oidc.service.client-secret}") + @Value("${fr.insee.genesis.oidc.service.client-secret}") private String serviceClientSecret; @Value("${fr.insee.genesis.security.token.oidc-claim-role}") diff --git a/src/main/resources/application-test-cucumber.properties b/src/main/resources/application-test-cucumber.properties index 590d886fb..9903a417d 100644 --- a/src/main/resources/application-test-cucumber.properties +++ b/src/main/resources/application-test-cucumber.properties @@ -19,6 +19,6 @@ fr.insee.genesis.authentication = NONE fr.insee.genesis.survey-quality-tool.url=*** fr.insee.genesis.oidc.service.client-id=*** -fr.insee.genesus.oidc.service.client-secret=*** +fr.insee.genesis.oidc.service.client-secret=*** fr.insee.genesis.rawdata.process.batch-size=1000 \ No newline at end of file diff --git a/src/main/resources/application-test.properties b/src/main/resources/application-test.properties index f4e3d5daf..7cfd35300 100644 --- a/src/main/resources/application-test.properties +++ b/src/main/resources/application-test.properties @@ -20,6 +20,6 @@ fr.insee.genesis.authentication = OIDC fr.insee.genesis.survey-quality-tool.url=*** fr.insee.genesis.oidc.service.client-id=*** -fr.insee.genesus.oidc.service.client-secret=*** +fr.insee.genesis.oidc.service.client-secret=*** private String serviceClientSecret; fr.insee.genesis.rawdata.process.batch-size=1000 \ No newline at end of file From 4cb4a079ad595ce75e6c9dfe45a5101be44b333d Mon Sep 17 00:00:00 2001 From: QDIBYS Date: Thu, 26 Jun 2025 15:22:36 +0200 Subject: [PATCH 12/12] fix: properties --- src/main/resources/application-test.properties | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/resources/application-test.properties b/src/main/resources/application-test.properties index 7cfd35300..db7f901bf 100644 --- a/src/main/resources/application-test.properties +++ b/src/main/resources/application-test.properties @@ -21,5 +21,5 @@ fr.insee.genesis.authentication = OIDC fr.insee.genesis.survey-quality-tool.url=*** fr.insee.genesis.oidc.service.client-id=*** fr.insee.genesis.oidc.service.client-secret=*** -private String serviceClientSecret; + fr.insee.genesis.rawdata.process.batch-size=1000 \ No newline at end of file