diff --git a/pom.xml b/pom.xml index 664e3b2d4..862b56f6c 100644 --- a/pom.xml +++ b/pom.xml @@ -51,6 +51,10 @@ org.springframework.boot spring-boot-starter-oauth2-resource-server + + org.springframework.boot + spring-boot-starter-webflux + org.springframework.boot diff --git a/src/main/java/fr/insee/genesis/Constants.java b/src/main/java/fr/insee/genesis/Constants.java index 9fb7edc67..7d2122a3f 100644 --- a/src/main/java/fr/insee/genesis/Constants.java +++ b/src/main/java/fr/insee/genesis/Constants.java @@ -1,5 +1,8 @@ package fr.insee.genesis; +import lombok.experimental.UtilityClass; + +@UtilityClass public class Constants { public static final String XSLT_STRUCTURED_VARIABLES = "xslt/structured-variables.xsl"; @@ -58,11 +61,6 @@ public class Constants { // Kraftwerk service path parameters public static final String KRAFTWERK_MAIN_ENDPOINT = ""; - private Constants() { - throw new IllegalStateException("Constants class"); - } - - public static String[] getEnoVariables() { return ENO_VARIABLES; } diff --git a/src/main/java/fr/insee/genesis/configuration/Config.java b/src/main/java/fr/insee/genesis/configuration/Config.java index 15d511749..9b084caef 100644 --- a/src/main/java/fr/insee/genesis/configuration/Config.java +++ b/src/main/java/fr/insee/genesis/configuration/Config.java @@ -31,6 +31,11 @@ public class Config { @Value("${fr.insee.genesis.oidc.realm}") private String realm; + @Value("${fr.insee.genesis.oidc.service.client-id}") + private String serviceClientId; + @Value("${fr.insee.genesis.oidc.service.client-secret}") + private String serviceClientSecret; + @Value("${fr.insee.genesis.security.token.oidc-claim-role}") private String oidcClaimRole; @@ -40,6 +45,12 @@ public class Config { @Value("#{'${fr.insee.genesis.security.whitelist-matchers}'.split(',')}") private String[] whiteList; + @Value("${fr.insee.genesis.rawdata.process.batch-size}") + private int rawDataProcessingBatchSize; + + @Value("${fr.insee.genesis.survey-quality-tool.url}") + private String surveyQualityToolUrl; + private final String logFolder; //Extract log folder from log filename property diff --git a/src/main/java/fr/insee/genesis/domain/ports/spi/SurveyUnitQualityToolPort.java b/src/main/java/fr/insee/genesis/domain/ports/spi/SurveyUnitQualityToolPort.java new file mode 100644 index 000000000..549e89464 --- /dev/null +++ b/src/main/java/fr/insee/genesis/domain/ports/spi/SurveyUnitQualityToolPort.java @@ -0,0 +1,11 @@ +package fr.insee.genesis.domain.ports.spi; + +import org.springframework.http.ResponseEntity; + +import java.io.IOException; +import java.util.Map; +import java.util.Set; + +public interface SurveyUnitQualityToolPort { + ResponseEntity sendProcessedIds(Map> processedIdsMap) throws IOException; +} diff --git a/src/main/java/fr/insee/genesis/domain/service/rawdata/LunaticJsonRawDataService.java b/src/main/java/fr/insee/genesis/domain/service/rawdata/LunaticJsonRawDataService.java index 42db85031..00ba32e3b 100644 --- a/src/main/java/fr/insee/genesis/domain/service/rawdata/LunaticJsonRawDataService.java +++ b/src/main/java/fr/insee/genesis/domain/service/rawdata/LunaticJsonRawDataService.java @@ -2,9 +2,11 @@ import fr.insee.bpm.metadata.model.VariablesMap; import fr.insee.genesis.Constants; +import fr.insee.genesis.configuration.Config; import fr.insee.genesis.controller.dto.rawdata.LunaticJsonRawDataUnprocessedDto; import fr.insee.genesis.controller.services.MetadataService; import fr.insee.genesis.controller.utils.ControllerUtils; +import fr.insee.genesis.domain.model.context.DataProcessingContextModel; import fr.insee.genesis.domain.model.surveyunit.DataState; import fr.insee.genesis.domain.model.surveyunit.Mode; import fr.insee.genesis.domain.model.surveyunit.SurveyUnitModel; @@ -14,6 +16,8 @@ import fr.insee.genesis.domain.model.surveyunit.rawdata.RawDataModelType; import fr.insee.genesis.domain.ports.api.LunaticJsonRawDataApiPort; import fr.insee.genesis.domain.ports.spi.LunaticJsonRawDataPersistencePort; +import fr.insee.genesis.domain.ports.spi.SurveyUnitQualityToolPort; +import fr.insee.genesis.domain.service.context.DataProcessingContextService; import fr.insee.genesis.domain.service.surveyunit.SurveyUnitQualityService; import fr.insee.genesis.domain.service.surveyunit.SurveyUnitService; import fr.insee.genesis.domain.utils.GroupUtils; @@ -24,10 +28,13 @@ import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Qualifier; +import org.springframework.http.ResponseEntity; import org.springframework.stereotype.Service; +import java.io.IOException; import java.time.LocalDateTime; import java.util.ArrayList; +import java.util.HashMap; import java.util.HashSet; import java.util.List; import java.util.Map; @@ -41,20 +48,34 @@ public class LunaticJsonRawDataService implements LunaticJsonRawDataApiPort { private final MetadataService metadataService; private final SurveyUnitService surveyUnitService; private final SurveyUnitQualityService surveyUnitQualityService; + private final SurveyUnitQualityToolPort surveyUnitQualityToolPort; + private final DataProcessingContextService dataProcessingContextService; private final FileUtils fileUtils; - private static final int BATCH_SIZE = 1000; + private final Config config; @Qualifier("lunaticJsonMongoAdapterNew") private final LunaticJsonRawDataPersistencePort lunaticJsonRawDataPersistencePort; @Autowired - public LunaticJsonRawDataService(LunaticJsonRawDataPersistencePort lunaticJsonRawDataNewPersistencePort, ControllerUtils controllerUtils, MetadataService metadataService, SurveyUnitService surveyUnitService, SurveyUnitQualityService surveyUnitQualityService, FileUtils fileUtils) { + public LunaticJsonRawDataService(LunaticJsonRawDataPersistencePort lunaticJsonRawDataNewPersistencePort, + ControllerUtils controllerUtils, + MetadataService metadataService, + SurveyUnitService surveyUnitService, + SurveyUnitQualityService surveyUnitQualityService, + FileUtils fileUtils, + DataProcessingContextService dataProcessingContextService, + SurveyUnitQualityToolPort surveyUnitQualityToolPort, + Config config + ) { this.controllerUtils = controllerUtils; this.metadataService = metadataService; this.surveyUnitService = surveyUnitService; this.surveyUnitQualityService = surveyUnitQualityService; this.fileUtils = fileUtils; this.lunaticJsonRawDataPersistencePort = lunaticJsonRawDataNewPersistencePort; + this.surveyUnitQualityToolPort = surveyUnitQualityToolPort; + this.dataProcessingContextService = dataProcessingContextService; + this.config = config; } @Override @@ -71,6 +92,8 @@ public List getRawData(String campaignName, Mode mode, public DataProcessResult processRawData(String campaignName, List interrogationIdList, List errors) throws GenesisException { int dataCount=0; int formattedDataCount=0; + DataProcessingContextModel dataProcessingContext = + dataProcessingContextService.getContextByPartitionId(campaignName); List modesList = controllerUtils.getModesList(campaignName, null); for (Mode mode : modesList) { //Load and save metadata into database, throw exception if none @@ -82,12 +105,12 @@ public DataProcessResult processRawData(String campaignName, List interr .formatted(mode, errors.getLast().getMessage()) ); } - int totalBatchs = Math.ceilDiv(interrogationIdList.size() , BATCH_SIZE); + int totalBatchs = Math.ceilDiv(interrogationIdList.size() , config.getRawDataProcessingBatchSize()); int batchNumber = 1; List interrogationIdListForMode = new ArrayList<>(interrogationIdList); while(!interrogationIdListForMode.isEmpty()){ log.info("Processing raw data batch {}/{}", batchNumber, totalBatchs); - int maxIndex = Math.min(interrogationIdListForMode.size(), BATCH_SIZE); + int maxIndex = Math.min(interrogationIdListForMode.size(), config.getRawDataProcessingBatchSize()); List interrogationIdToProcess = interrogationIdListForMode.subList(0, maxIndex); List rawData = getRawData(campaignName,mode, interrogationIdToProcess); @@ -110,6 +133,23 @@ public DataProcessResult processRawData(String campaignName, List interr surveyUnitModel -> surveyUnitModel.getState().equals(DataState.FORMATTED) ).toList().size(); + //Send processed ids grouped by questionnaire (if review activated) + if(dataProcessingContext != null && dataProcessingContext.isWithReview()) { + try { + ResponseEntity response = + surveyUnitQualityToolPort.sendProcessedIds(getProcessedIdsMap(surveyUnitModels)); + + if (response.getStatusCode().is2xxSuccessful()) { + log.info("Successfully sent {} ids to quality tool", getProcessedIdsMap(surveyUnitModels).keySet().size()); + }else{ + log.warn("Survey unit quality tool responded non-2xx code {} and body {}", + response.getStatusCode(), response.getBody()); + } + }catch (IOException e){ + log.error("Error during Perret call request building : {}", e.toString()); + } + } + //Remove processed ids from list interrogationIdListForMode = interrogationIdListForMode.subList(maxIndex, interrogationIdListForMode.size()); @@ -119,6 +159,16 @@ public DataProcessResult processRawData(String campaignName, List interr return new DataProcessResult(dataCount, formattedDataCount); } + private Map> getProcessedIdsMap(List surveyUnitModels) { + Map> processedInterrogationIdsPerQuestionnaire = new HashMap<>(); + surveyUnitModels.forEach(model -> + processedInterrogationIdsPerQuestionnaire + .computeIfAbsent(model.getQuestionnaireId(), k -> new HashSet<>()) + .add(model.getInterrogationId()) + ); + return processedInterrogationIdsPerQuestionnaire; + } + @Override public List convertRawData(List rawDataList, VariablesMap variablesMap) { //Convert to genesis model diff --git a/src/main/java/fr/insee/genesis/domain/service/surveyunit/SurveyUnitService.java b/src/main/java/fr/insee/genesis/domain/service/surveyunit/SurveyUnitService.java index 09d6fd27b..e0ee35fa9 100644 --- a/src/main/java/fr/insee/genesis/domain/service/surveyunit/SurveyUnitService.java +++ b/src/main/java/fr/insee/genesis/domain/service/surveyunit/SurveyUnitService.java @@ -505,7 +505,6 @@ private void extractVariables(SurveyUnitModel surveyUnitModel, surveyUnitModel.setCollectedVariables(new ArrayList<>()); } for (VariableModel collectedVariable : surveyUnitModel.getCollectedVariables()) { - log.info("Variable name : {}", collectedVariable.varId()); VarIdScopeTuple loopIdTuple = new VarIdScopeTuple(collectedVariable.varId(), collectedVariable.scope(), collectedVariable.iteration()); VariableDto variableDto = collectedVariableMap.get(loopIdTuple); diff --git a/src/main/java/fr/insee/genesis/infrastructure/adapter/SurveyQualityToolPerretAdapter.java b/src/main/java/fr/insee/genesis/infrastructure/adapter/SurveyQualityToolPerretAdapter.java new file mode 100644 index 000000000..38ea3d58a --- /dev/null +++ b/src/main/java/fr/insee/genesis/infrastructure/adapter/SurveyQualityToolPerretAdapter.java @@ -0,0 +1,35 @@ +package fr.insee.genesis.infrastructure.adapter; + +import fr.insee.genesis.configuration.Config; +import fr.insee.genesis.domain.ports.spi.SurveyUnitQualityToolPort; +import fr.insee.genesis.infrastructure.utils.http.OidcService; +import fr.insee.genesis.infrastructure.utils.http.HttpUtils; +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Qualifier; +import org.springframework.http.HttpMethod; +import org.springframework.http.ResponseEntity; +import org.springframework.stereotype.Service; + +import java.io.IOException; +import java.util.Map; +import java.util.Set; + +@Service +@Qualifier("surveyQualityToolPerretAdapter") +@Slf4j +public class SurveyQualityToolPerretAdapter implements SurveyUnitQualityToolPort { + + @Autowired + Config config; + + @Autowired + OidcService oidcService; + + + @Override + public ResponseEntity sendProcessedIds(Map> processedIdsMap) throws IOException { + return HttpUtils.makeApiCall(config.getSurveyQualityToolUrl(), "/interrogations/",HttpMethod.POST, processedIdsMap, + Object.class, oidcService); + } +} diff --git a/src/main/java/fr/insee/genesis/infrastructure/document/surveyunit/SurveyUnitDocument.java b/src/main/java/fr/insee/genesis/infrastructure/document/surveyunit/SurveyUnitDocument.java index cbd05258e..d67173323 100644 --- a/src/main/java/fr/insee/genesis/infrastructure/document/surveyunit/SurveyUnitDocument.java +++ b/src/main/java/fr/insee/genesis/infrastructure/document/surveyunit/SurveyUnitDocument.java @@ -3,6 +3,7 @@ import fr.insee.genesis.Constants; import lombok.Data; import org.springframework.data.mongodb.core.index.CompoundIndex; +import org.springframework.data.mongodb.core.index.Indexed; import org.springframework.data.mongodb.core.mapping.Document; import java.time.LocalDateTime; @@ -13,13 +14,12 @@ @CompoundIndex(name = "campaignId_1_questionnaireId_1", def = "{'campaignId': 1, 'questionnaireId': 1}") //1 = ascending, -1 = descending @CompoundIndex(name = "questionnaireId_1_campaignId_1", def = "{'questionnaireId': 1, 'campaignId': 1}") @CompoundIndex(name = "questionnaireId_1_interrogationId_1", def = "{'questionnaireId': 1, 'interrogationId': 1}") +@CompoundIndex(name = "interrogationId_1_questionnaireId_1", def = "{'interrogationId': 1, 'questionnaireId': 1}") public class SurveyUnitDocument { private String campaignId; + @Indexed private String interrogationId; private String questionnaireId; - private String contextualId; - private Boolean isCapturedIndirectly; - private LocalDateTime validationDate; private String state; private String mode; private LocalDateTime recordDate; diff --git a/src/main/java/fr/insee/genesis/infrastructure/utils/http/HttpUtils.java b/src/main/java/fr/insee/genesis/infrastructure/utils/http/HttpUtils.java new file mode 100644 index 000000000..ebb3382aa --- /dev/null +++ b/src/main/java/fr/insee/genesis/infrastructure/utils/http/HttpUtils.java @@ -0,0 +1,36 @@ +package fr.insee.genesis.infrastructure.utils.http; + +import lombok.experimental.UtilityClass; +import org.springframework.http.HttpHeaders; +import org.springframework.http.HttpMethod; +import org.springframework.http.MediaType; +import org.springframework.http.ResponseEntity; +import org.springframework.web.reactive.function.client.WebClient; +import reactor.core.publisher.Mono; + +import java.io.IOException; + +@UtilityClass +public class HttpUtils { + public static ResponseEntity makeApiCall(String baseUrl, + String path, + HttpMethod method, + T requestBody, + Class responseType, + OidcService oidcService + ) throws IOException { + WebClient webClient = WebClient.builder() + .baseUrl(baseUrl) + .filter(PerretAuthWebClientFilter.perretAuthFilter(oidcService)) + .build(); + WebClient.ResponseSpec responseSpec = webClient + .method(method) + .uri(path) + .bodyValue(requestBody) + .header(HttpHeaders.CONTENT_TYPE, MediaType.APPLICATION_JSON_VALUE) + .header(HttpHeaders.AUTHORIZATION, "Bearer " + oidcService.getServiceAccountToken()) + .retrieve(); + Mono> response = responseSpec.toEntity(responseType); + return response.block(); + } +} diff --git a/src/main/java/fr/insee/genesis/infrastructure/utils/http/OidcService.java b/src/main/java/fr/insee/genesis/infrastructure/utils/http/OidcService.java new file mode 100644 index 000000000..3c1729607 --- /dev/null +++ b/src/main/java/fr/insee/genesis/infrastructure/utils/http/OidcService.java @@ -0,0 +1,76 @@ +package fr.insee.genesis.infrastructure.utils.http; + +import fr.insee.genesis.configuration.Config; +import lombok.Getter; +import lombok.extern.slf4j.Slf4j; +import org.springframework.http.HttpEntity; +import org.springframework.http.HttpHeaders; +import org.springframework.http.MediaType; +import org.springframework.http.ResponseEntity; +import org.springframework.stereotype.Service; +import org.springframework.web.client.HttpStatusCodeException; +import org.springframework.web.client.RestTemplate; + +import java.io.IOException; +import java.util.Map; + +@Service +@Slf4j +public class OidcService { + + public static final String ACCESS_TOKEN = "access_token"; + @Getter + private final Config config; + private String serviceAccountToken; + private long tokenExpirationTime = 0; + private final RestTemplate restTemplate; + + public OidcService(Config config) { + this.restTemplate = new RestTemplate(); + this.config = config; + } + + protected void retrieveServiceAccountToken() throws IOException { + String tokenUrl = String.format("%s/realms/%s/protocol/openid-connect/token", + config.getAuthServerUrl(), + config.getRealm()); + + HttpHeaders headers = new HttpHeaders(); + headers.setContentType(MediaType.APPLICATION_FORM_URLENCODED); + + String body = String.format("grant_type=client_credentials&client_id=%s&client_secret=%s&scope=openid profile roles", + config.getServiceClientId(), + config.getServiceClientSecret()); + + HttpEntity request = new HttpEntity<>(body, headers); + try { + ResponseEntity response = restTemplate.postForEntity(tokenUrl, request, Map.class); + if (response.getStatusCode().is2xxSuccessful() && response.hasBody() && response.getBody() != null) { + Map responseBody = response.getBody(); + + if (responseBody == null || !responseBody.containsKey(ACCESS_TOKEN) || !(responseBody.get(ACCESS_TOKEN) instanceof String)) { + throw new IOException("Invalid response: Missing or incorrect 'access_token'"); + } + + serviceAccountToken = (String) responseBody.get(ACCESS_TOKEN); + Integer expiresIn = (Integer) responseBody.get("expires_in"); + tokenExpirationTime = System.currentTimeMillis() + (expiresIn.longValue() * 1000L); + } else { + throw new IOException("Failed to retrieve service account token, status: " + response.getStatusCode()); + } + } catch (HttpStatusCodeException e) { + throw new IOException("HTTP error: " + e.getStatusCode() + " - " + e.getResponseBodyAsString(), e); + } catch (Exception e) { + throw new IOException("Unexpected error while fetching token: " + e.getMessage(), e); + } + } + + public synchronized String getServiceAccountToken() throws IOException { + //We had a margin of 5 seconds for the expiration time + if (serviceAccountToken == null || System.currentTimeMillis() >= tokenExpirationTime - 5000L) { + retrieveServiceAccountToken(); + } + return serviceAccountToken; + } + +} diff --git a/src/main/java/fr/insee/genesis/infrastructure/utils/http/PerretAuthWebClientFilter.java b/src/main/java/fr/insee/genesis/infrastructure/utils/http/PerretAuthWebClientFilter.java new file mode 100644 index 000000000..c9842068c --- /dev/null +++ b/src/main/java/fr/insee/genesis/infrastructure/utils/http/PerretAuthWebClientFilter.java @@ -0,0 +1,41 @@ +package fr.insee.genesis.infrastructure.utils.http; + +import lombok.extern.slf4j.Slf4j; +import org.springframework.web.reactive.function.client.ClientRequest; +import org.springframework.web.reactive.function.client.ExchangeFilterFunction; +import reactor.core.publisher.Mono; + +import java.io.IOException; + +@Slf4j +public class PerretAuthWebClientFilter { + public static ExchangeFilterFunction perretAuthFilter(OidcService oidcService) { + return ExchangeFilterFunction.ofRequestProcessor(request -> { + try { + String token = oidcService.getServiceAccountToken(); + ClientRequest newRequest = ClientRequest.from(request) + .headers(headers -> headers.setBearerAuth(token)) + .build(); + return Mono.just(newRequest); + } catch (IOException e) { + return Mono.error(new RuntimeException("Failed to get token", e)); + } + }).andThen((request, next) -> next.exchange(request) + .flatMap(response -> { + if (response.statusCode().value() == 401) { + log.warn("Received 401, refreshing token and retrying once"); + try { + oidcService.retrieveServiceAccountToken(); // Force refresh + String refreshedToken = oidcService.getServiceAccountToken(); + ClientRequest retryRequest = ClientRequest.from(request) + .headers(headers -> headers.setBearerAuth(refreshedToken)) + .build(); + return next.exchange(retryRequest); + } catch (IOException e) { + return Mono.error(new RuntimeException("Failed to refresh token", e)); + } + } + return Mono.just(response); + })); + } +} diff --git a/src/main/resources/application-dev.properties b/src/main/resources/application-dev.properties index 9e16ba510..ddad30609 100644 --- a/src/main/resources/application-dev.properties +++ b/src/main/resources/application-dev.properties @@ -22,4 +22,9 @@ app.role.user-kraftwerk.claims=*** app.role.user-platine.claims=*** app.role.reader.claims=*** app.role.collect-platform.claims=*** -app.role.scheduler.claims=*** \ No newline at end of file +app.role.scheduler.claims=*** + +#-------------------------------------------------------------------------- +# Survey quality tool +#-------------------------------------------------------------------------- +fr.insee.genesis.survey-quality-tool.url=*** \ No newline at end of file diff --git a/src/main/resources/application-preprod.properties b/src/main/resources/application-preprod.properties index 19e1c751f..3faf95de3 100644 --- a/src/main/resources/application-preprod.properties +++ b/src/main/resources/application-preprod.properties @@ -22,4 +22,9 @@ app.role.user-kraftwerk.claims=*** app.role.user-platine.claims=*** app.role.reader.claims=*** app.role.collect-platform.claims=*** -app.role.scheduler.claims=*** \ No newline at end of file +app.role.scheduler.claims=*** + +#-------------------------------------------------------------------------- +# Survey quality tool +#-------------------------------------------------------------------------- +fr.insee.genesis.survey-quality-tool.url=*** \ No newline at end of file diff --git a/src/main/resources/application-prod.properties b/src/main/resources/application-prod.properties index a98155889..92cc4a134 100644 --- a/src/main/resources/application-prod.properties +++ b/src/main/resources/application-prod.properties @@ -22,4 +22,9 @@ app.role.user-kraftwerk.claims=*** app.role.user-platine.claims=*** app.role.reader.claims=*** app.role.collect-platform.claims=*** -app.role.scheduler.claims=*** \ No newline at end of file +app.role.scheduler.claims=*** + +#-------------------------------------------------------------------------- +# Survey quality tool +#-------------------------------------------------------------------------- +fr.insee.genesis.survey-quality-tool.url=*** \ No newline at end of file diff --git a/src/main/resources/application-test-cucumber.properties b/src/main/resources/application-test-cucumber.properties index 39bde09ad..9903a417d 100644 --- a/src/main/resources/application-test-cucumber.properties +++ b/src/main/resources/application-test-cucumber.properties @@ -16,3 +16,9 @@ fr.insee.genesis.oidc.realm=test-realm springdoc.swagger-ui.oauth.client-id=client-id-test fr.insee.genesis.authentication = NONE + +fr.insee.genesis.survey-quality-tool.url=*** +fr.insee.genesis.oidc.service.client-id=*** +fr.insee.genesis.oidc.service.client-secret=*** + +fr.insee.genesis.rawdata.process.batch-size=1000 \ No newline at end of file diff --git a/src/main/resources/application-test.properties b/src/main/resources/application-test.properties index 932289c53..db7f901bf 100644 --- a/src/main/resources/application-test.properties +++ b/src/main/resources/application-test.properties @@ -17,3 +17,9 @@ springdoc.swagger-ui.oauth.client-id=client-id-test fr.insee.genesis.authentication = OIDC + +fr.insee.genesis.survey-quality-tool.url=*** +fr.insee.genesis.oidc.service.client-id=*** +fr.insee.genesis.oidc.service.client-secret=*** + +fr.insee.genesis.rawdata.process.batch-size=1000 \ No newline at end of file diff --git a/src/test/java/cucumber/functional_tests/RawDataDefinitions.java b/src/test/java/cucumber/functional_tests/RawDataDefinitions.java index 951a99e1d..eac22e0f4 100644 --- a/src/test/java/cucumber/functional_tests/RawDataDefinitions.java +++ b/src/test/java/cucumber/functional_tests/RawDataDefinitions.java @@ -10,13 +10,16 @@ import fr.insee.genesis.domain.model.surveyunit.Mode; import fr.insee.genesis.domain.model.surveyunit.SurveyUnitModel; import fr.insee.genesis.domain.model.surveyunit.VariableModel; +import fr.insee.genesis.domain.service.context.DataProcessingContextService; import fr.insee.genesis.domain.service.rawdata.LunaticJsonRawDataService; import fr.insee.genesis.domain.service.surveyunit.SurveyUnitQualityService; import fr.insee.genesis.domain.service.surveyunit.SurveyUnitService; import fr.insee.genesis.domain.utils.JsonUtils; import fr.insee.genesis.infrastructure.utils.FileUtils; import fr.insee.genesis.stubs.ConfigStub; +import fr.insee.genesis.stubs.DataProcessingContextPersistancePortStub; import fr.insee.genesis.stubs.LunaticJsonRawDataPersistanceStub; +import fr.insee.genesis.stubs.SurveyUnitQualityToolPerretAdapterStub; import fr.insee.genesis.stubs.SurveyUnitPersistencePortStub; import io.cucumber.java.Before; import io.cucumber.java.en.Given; @@ -65,7 +68,22 @@ public class RawDataDefinitions { SurveyUnitService surveyUnitService = new SurveyUnitService(surveyUnitPersistencePortStub, metadataService, fileUtils); ControllerUtils controllerUtils = new ControllerUtils(fileUtils); SurveyUnitQualityService surveyUnitQualityService = new SurveyUnitQualityService(); - LunaticJsonRawDataService lunaticJsonRawDataService = new LunaticJsonRawDataService(lunaticJsonRawDataPersistanceStub,controllerUtils,metadataService,surveyUnitService,surveyUnitQualityService,fileUtils); + DataProcessingContextPersistancePortStub dataProcessingContextPersistancePortStub = + new DataProcessingContextPersistancePortStub(); + SurveyUnitQualityToolPerretAdapterStub surveyUnitQualityToolPerretAdapterStub = new SurveyUnitQualityToolPerretAdapterStub(); + LunaticJsonRawDataService lunaticJsonRawDataService = + new LunaticJsonRawDataService( + lunaticJsonRawDataPersistanceStub, + controllerUtils, + metadataService, + surveyUnitService, + surveyUnitQualityService, + fileUtils, + new DataProcessingContextService( + dataProcessingContextPersistancePortStub, + surveyUnitPersistencePortStub), + surveyUnitQualityToolPerretAdapterStub, + config); RawResponseController rawResponseController = new RawResponseController( lunaticJsonRawDataService ); diff --git a/src/test/java/fr/insee/genesis/controller/rest/UtilsControllerTest.java b/src/test/java/fr/insee/genesis/controller/rest/UtilsControllerTest.java index 2885083a1..9936e683b 100644 --- a/src/test/java/fr/insee/genesis/controller/rest/UtilsControllerTest.java +++ b/src/test/java/fr/insee/genesis/controller/rest/UtilsControllerTest.java @@ -11,13 +11,16 @@ import fr.insee.genesis.domain.model.surveyunit.VariableModel; import fr.insee.genesis.domain.ports.api.LunaticJsonRawDataApiPort; import fr.insee.genesis.domain.ports.api.SurveyUnitApiPort; +import fr.insee.genesis.domain.service.context.DataProcessingContextService; import fr.insee.genesis.domain.service.rawdata.LunaticJsonRawDataService; import fr.insee.genesis.domain.service.surveyunit.SurveyUnitQualityService; import fr.insee.genesis.domain.service.surveyunit.SurveyUnitService; import fr.insee.genesis.domain.service.volumetry.VolumetryLogService; import fr.insee.genesis.infrastructure.utils.FileUtils; import fr.insee.genesis.stubs.ConfigStub; +import fr.insee.genesis.stubs.DataProcessingContextPersistancePortStub; import fr.insee.genesis.stubs.LunaticJsonRawDataPersistanceStub; +import fr.insee.genesis.stubs.SurveyUnitQualityToolPerretAdapterStub; import fr.insee.genesis.stubs.SurveyUnitPersistencePortStub; import org.assertj.core.api.Assertions; import org.junit.jupiter.api.AfterAll; @@ -43,6 +46,7 @@ class UtilsControllerTest { static UtilsController utilsControllerStatic; static SurveyUnitPersistencePortStub surveyUnitPersistencePortStub; static LunaticJsonRawDataPersistanceStub lunaticJsonRawDataPersistencePort; + static SurveyUnitQualityToolPerretAdapterStub surveyUnitQualityToolPerretAdapterStub; static List interrogationIdList; static FileUtils fileUtils = new FileUtils(new ConfigStub()); static ControllerUtils controllerUtils = new ControllerUtils(fileUtils); @@ -58,7 +62,18 @@ static void init() { surveyUnitPersistencePortStub = new SurveyUnitPersistencePortStub(); lunaticJsonRawDataPersistencePort = new LunaticJsonRawDataPersistanceStub(); SurveyUnitApiPort surveyUnitApiPort = new SurveyUnitService(surveyUnitPersistencePortStub, metadataService, fileUtils); - LunaticJsonRawDataApiPort lunaticJsonRawDataApiPort = new LunaticJsonRawDataService(lunaticJsonRawDataPersistencePort,controllerUtils,metadataService,surveyUnitService,surveyUnitQualityService,fileUtils); + LunaticJsonRawDataApiPort lunaticJsonRawDataApiPort = new LunaticJsonRawDataService( + lunaticJsonRawDataPersistencePort, + controllerUtils, + metadataService, + surveyUnitService, + surveyUnitQualityService, + fileUtils, + new DataProcessingContextService( + new DataProcessingContextPersistancePortStub(), + surveyUnitPersistencePortStub), + surveyUnitQualityToolPerretAdapterStub, + new ConfigStub()); utilsControllerStatic = new UtilsController( surveyUnitApiPort diff --git a/src/test/java/fr/insee/genesis/controller/rest/responses/RawResponseControllerTest.java b/src/test/java/fr/insee/genesis/controller/rest/responses/RawResponseControllerTest.java index b068c785e..3b4f48160 100644 --- a/src/test/java/fr/insee/genesis/controller/rest/responses/RawResponseControllerTest.java +++ b/src/test/java/fr/insee/genesis/controller/rest/responses/RawResponseControllerTest.java @@ -4,18 +4,23 @@ import fr.insee.genesis.controller.dto.rawdata.LunaticJsonRawDataUnprocessedDto; import fr.insee.genesis.controller.services.MetadataService; import fr.insee.genesis.controller.utils.ControllerUtils; +import fr.insee.genesis.domain.model.context.DataProcessingContextModel; import fr.insee.genesis.domain.model.surveyunit.DataState; import fr.insee.genesis.domain.model.surveyunit.Mode; import fr.insee.genesis.domain.ports.api.LunaticJsonRawDataApiPort; +import fr.insee.genesis.domain.service.context.DataProcessingContextService; import fr.insee.genesis.domain.service.rawdata.LunaticJsonRawDataService; import fr.insee.genesis.domain.service.surveyunit.SurveyUnitQualityService; import fr.insee.genesis.domain.service.surveyunit.SurveyUnitService; import fr.insee.genesis.domain.utils.JsonUtils; import fr.insee.genesis.infrastructure.document.rawdata.LunaticJsonRawDataDocument; +import fr.insee.genesis.infrastructure.mappers.DataProcessingContextMapper; import fr.insee.genesis.infrastructure.utils.FileUtils; import fr.insee.genesis.stubs.ConfigStub; +import fr.insee.genesis.stubs.DataProcessingContextPersistancePortStub; import fr.insee.genesis.stubs.LunaticJsonRawDataPersistanceStub; import fr.insee.genesis.stubs.SurveyUnitPersistencePortStub; +import fr.insee.genesis.stubs.SurveyUnitQualityToolPerretAdapterStub; import org.assertj.core.api.Assertions; import org.junit.jupiter.api.Test; import org.springframework.http.ResponseEntity; @@ -29,12 +34,19 @@ class RawResponseControllerTest { private final FileUtils fileUtils = new FileUtils(new ConfigStub()); private final LunaticJsonRawDataPersistanceStub lunaticJsonRawDataPersistanceStub = new LunaticJsonRawDataPersistanceStub(); private final SurveyUnitPersistencePortStub surveyUnitPersistencePortStub = new SurveyUnitPersistencePortStub(); + private final SurveyUnitQualityToolPerretAdapterStub surveyUnitQualityToolPerretAdapterStub = new SurveyUnitQualityToolPerretAdapterStub(); + private final DataProcessingContextPersistancePortStub dataProcessingContextPersistancePortStub = + new DataProcessingContextPersistancePortStub(); private final LunaticJsonRawDataApiPort lunaticJsonRawDataApiPort = new LunaticJsonRawDataService(lunaticJsonRawDataPersistanceStub, new ControllerUtils(fileUtils), new MetadataService(), new SurveyUnitService(surveyUnitPersistencePortStub, new MetadataService(), fileUtils), new SurveyUnitQualityService(), - fileUtils); + fileUtils, + new DataProcessingContextService(dataProcessingContextPersistancePortStub, surveyUnitPersistencePortStub), + surveyUnitQualityToolPerretAdapterStub, + new ConfigStub() + ); private final RawResponseController rawResponseController = new RawResponseController(lunaticJsonRawDataApiPort); @@ -118,6 +130,7 @@ void processJsonRawDataTest(){ //GIVEN lunaticJsonRawDataPersistanceStub.getMongoStub().clear(); surveyUnitPersistencePortStub.getMongoStub().clear(); + surveyUnitQualityToolPerretAdapterStub.getReceivedMaps().clear(); String campaignId = "SAMPLETEST-PARADATA-v2"; String questionnaireId = campaignId + "_quest"; String interrogationId = "testinterrogationId1"; @@ -125,6 +138,17 @@ void processJsonRawDataTest(){ String varValue = "TEST"; addJsonRawDataDocumentToStub(campaignId, questionnaireId, interrogationId, null, varName, varValue); + dataProcessingContextPersistancePortStub.getMongoStub().add( + DataProcessingContextMapper.INSTANCE.modelToDocument( + DataProcessingContextModel.builder() + .partitionId(campaignId) + .kraftwerkExecutionScheduleList(new ArrayList<>()) + .withReview(true) + .build() + ) + ); + + List interrogationIdList = new ArrayList<>(); interrogationIdList.add(interrogationId); @@ -150,6 +174,12 @@ void processJsonRawDataTest(){ //Process date check Assertions.assertThat(lunaticJsonRawDataPersistanceStub.getMongoStub().getFirst().processDate()).isNotNull(); + //Perret call check + Assertions.assertThat(surveyUnitQualityToolPerretAdapterStub.getReceivedMaps()) + .hasSize(1); + Assertions.assertThat(surveyUnitQualityToolPerretAdapterStub.getReceivedMaps().getFirst()).containsKey(questionnaireId); + Assertions.assertThat(surveyUnitQualityToolPerretAdapterStub.getReceivedMaps().getFirst().get(questionnaireId)) + .contains(interrogationId); } diff --git a/src/test/java/fr/insee/genesis/domain/service/rawdata/LunaticJsonRawDataServiceTest.java b/src/test/java/fr/insee/genesis/domain/service/rawdata/LunaticJsonRawDataServiceTest.java index 5886bea7a..ce3e24a8a 100644 --- a/src/test/java/fr/insee/genesis/domain/service/rawdata/LunaticJsonRawDataServiceTest.java +++ b/src/test/java/fr/insee/genesis/domain/service/rawdata/LunaticJsonRawDataServiceTest.java @@ -1,21 +1,27 @@ package fr.insee.genesis.domain.service.rawdata; +import com.fasterxml.jackson.core.JsonProcessingException; import fr.insee.bpm.metadata.model.VariablesMap; import fr.insee.genesis.controller.services.MetadataService; import fr.insee.genesis.controller.utils.ControllerUtils; +import fr.insee.genesis.domain.model.context.DataProcessingContextModel; import fr.insee.genesis.domain.model.surveyunit.DataState; import fr.insee.genesis.domain.model.surveyunit.Mode; import fr.insee.genesis.domain.model.surveyunit.SurveyUnitModel; import fr.insee.genesis.domain.model.surveyunit.rawdata.DataProcessResult; import fr.insee.genesis.domain.model.surveyunit.rawdata.LunaticJsonRawDataModel; +import fr.insee.genesis.domain.service.context.DataProcessingContextService; import fr.insee.genesis.domain.service.surveyunit.SurveyUnitQualityService; import fr.insee.genesis.domain.service.surveyunit.SurveyUnitService; import fr.insee.genesis.domain.utils.JsonUtils; +import fr.insee.genesis.infrastructure.mappers.DataProcessingContextMapper; import fr.insee.genesis.infrastructure.mappers.LunaticJsonRawDataDocumentMapper; import fr.insee.genesis.infrastructure.utils.FileUtils; import fr.insee.genesis.stubs.ConfigStub; +import fr.insee.genesis.stubs.DataProcessingContextPersistancePortStub; import fr.insee.genesis.stubs.LunaticJsonRawDataPersistanceStub; import fr.insee.genesis.stubs.SurveyUnitPersistencePortStub; +import fr.insee.genesis.stubs.SurveyUnitQualityToolPerretAdapterStub; import org.assertj.core.api.Assertions; import org.junit.jupiter.api.Test; import org.junit.jupiter.params.ParameterizedTest; @@ -34,10 +40,22 @@ class LunaticJsonRawDataServiceTest { MetadataService metadataService = new MetadataService(); SurveyUnitPersistencePortStub surveyUnitPersistencePortStub = new SurveyUnitPersistencePortStub(); - SurveyUnitService surveyUnitService = new SurveyUnitService(surveyUnitPersistencePortStub, metadataService, fileUtils); - SurveyUnitQualityService surveyUnitQualityService = new SurveyUnitQualityService(); - - LunaticJsonRawDataService lunaticJsonRawDataService = new LunaticJsonRawDataService(lunaticJsonRawDataPersistanceStub,controllerUtils,metadataService,surveyUnitService,surveyUnitQualityService,fileUtils); + SurveyUnitQualityToolPerretAdapterStub surveyUnitQualityToolPerretAdapterStub = new SurveyUnitQualityToolPerretAdapterStub(); + DataProcessingContextPersistancePortStub dataProcessingContextPersistancePortStub = + new DataProcessingContextPersistancePortStub(); + ConfigStub configStub = new ConfigStub(); + LunaticJsonRawDataService lunaticJsonRawDataService = + new LunaticJsonRawDataService( + lunaticJsonRawDataPersistanceStub, + controllerUtils, + metadataService, + new SurveyUnitService(surveyUnitPersistencePortStub, metadataService, fileUtils), + new SurveyUnitQualityService(), + fileUtils, + new DataProcessingContextService(dataProcessingContextPersistancePortStub, surveyUnitPersistencePortStub), + surveyUnitQualityToolPerretAdapterStub, + configStub + ); @Test void saveDataTest_valid_only_collected_array() throws Exception { @@ -401,14 +419,72 @@ void convertRawData_if_collected_not_present_test() throws Exception { @ParameterizedTest @ValueSource(ints = {5,500,5000,10000}) void convertRawData_multipleBatchs(int rawDataSize) throws Exception { + //GIVEN + String campaignId = "SAMPLETEST-PARADATA-v1"; + String questionnaireId = "TESTIDQUEST"; + List interrogationIdList = prepareConvertTest(rawDataSize, campaignId, questionnaireId); + //Activate review + dataProcessingContextPersistancePortStub.getMongoStub().add( + DataProcessingContextMapper.INSTANCE.modelToDocument( + DataProcessingContextModel.builder() + .partitionId(campaignId) + .withReview(true) + .kraftwerkExecutionScheduleList(new ArrayList<>()) + .build() + ) + ); + + //WHEN + DataProcessResult dataProcessResult = lunaticJsonRawDataService.processRawData(campaignId, interrogationIdList, + new ArrayList<>()); + + //THEN + Assertions.assertThat(dataProcessResult.dataCount()).isEqualTo(rawDataSize * 2/*EDITED*/); + Assertions.assertThat(surveyUnitPersistencePortStub.getMongoStub()).hasSize(rawDataSize * 2); + Assertions.assertThat(surveyUnitQualityToolPerretAdapterStub.getReceivedMaps()) + .hasSize(Math.ceilDiv(rawDataSize, configStub.getRawDataProcessingBatchSize())); + Assertions.assertThat(surveyUnitQualityToolPerretAdapterStub.getReceivedMaps().getFirst()).containsKey(questionnaireId); + Assertions.assertThat(surveyUnitQualityToolPerretAdapterStub.getReceivedMaps().getFirst().get(questionnaireId)) + .contains("TESTinterrogationId1"); + } + + @Test + void convertRawData_review_desactivated() throws Exception { + //GIVEN + String campaignId = "SAMPLETEST-PARADATA-v1"; + String questionnaireId = "TESTIDQUEST"; + List interrogationIdList = prepareConvertTest(1, campaignId, questionnaireId); + + //Desactivate review + dataProcessingContextPersistancePortStub.getMongoStub().add( + DataProcessingContextMapper.INSTANCE.modelToDocument( + DataProcessingContextModel.builder() + .partitionId(campaignId) + .withReview(false) + .kraftwerkExecutionScheduleList(new ArrayList<>()) + .build() + ) + ); + + //WHEN + DataProcessResult dataProcessResult = lunaticJsonRawDataService.processRawData(campaignId, interrogationIdList, + new ArrayList<>()); + + //THEN + Assertions.assertThat(dataProcessResult.dataCount()).isEqualTo(2); + Assertions.assertThat(surveyUnitPersistencePortStub.getMongoStub()).hasSize(2); + Assertions.assertThat(surveyUnitQualityToolPerretAdapterStub.getReceivedMaps()).isEmpty(); + } + + private List prepareConvertTest(int rawDataSize, String campaignId, String questionnaireId) throws JsonProcessingException { //CLEAN surveyUnitPersistencePortStub.getMongoStub().clear(); + surveyUnitQualityToolPerretAdapterStub.getReceivedMaps().clear(); + dataProcessingContextPersistancePortStub.getMongoStub().clear(); //GIVEN List interrogationIdList = new ArrayList<>(); - String campaignId = "SAMPLETEST-PARADATA-v1"; for (int i = 0; i < rawDataSize; i++) { - String questionnaireId = "TESTIDQUEST"; String interrogationId = "TESTinterrogationId" + (i + 1); String json = "{\"EXTERNAL\": {\"RPPRENOM\": \"TEST_EXT%d\"}, ".formatted(i) + "\"COLLECTED\": {\"PRENOMREP\": {\"COLLECTED\": [\"test%d\"], \"EDITED\": [\"test_ed%d\"]}}}" @@ -426,13 +502,6 @@ void convertRawData_multipleBatchs(int rawDataSize) throws Exception { lunaticJsonRawDataPersistanceStub.getMongoStub() .add(LunaticJsonRawDataDocumentMapper.INSTANCE.modelToDocument(rawDataModel)); } - - //WHEN - DataProcessResult dataProcessResult = lunaticJsonRawDataService.processRawData(campaignId, interrogationIdList, - new ArrayList<>()); - - //THEN - Assertions.assertThat(dataProcessResult.dataCount()).isEqualTo(rawDataSize * 2/*EDITED*/); - Assertions.assertThat(surveyUnitPersistencePortStub.getMongoStub()).hasSize(rawDataSize * 2); + return interrogationIdList; } } \ No newline at end of file diff --git a/src/test/java/fr/insee/genesis/stubs/ConfigStub.java b/src/test/java/fr/insee/genesis/stubs/ConfigStub.java index 6d4f1c0c5..6ee444722 100644 --- a/src/test/java/fr/insee/genesis/stubs/ConfigStub.java +++ b/src/test/java/fr/insee/genesis/stubs/ConfigStub.java @@ -32,4 +32,9 @@ public String getSpecFolderSource() { public String getAuthType() { return "NONE"; } + + @Override + public int getRawDataProcessingBatchSize(){ + return 1000; + } } diff --git a/src/test/java/fr/insee/genesis/stubs/SurveyUnitQualityToolPerretAdapterStub.java b/src/test/java/fr/insee/genesis/stubs/SurveyUnitQualityToolPerretAdapterStub.java new file mode 100644 index 000000000..683c72b5d --- /dev/null +++ b/src/test/java/fr/insee/genesis/stubs/SurveyUnitQualityToolPerretAdapterStub.java @@ -0,0 +1,23 @@ +package fr.insee.genesis.stubs; + +import fr.insee.genesis.domain.ports.spi.SurveyUnitQualityToolPort; +import lombok.Getter; +import lombok.Setter; +import org.springframework.http.ResponseEntity; + +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.Set; + +@Getter +@Setter +public class SurveyUnitQualityToolPerretAdapterStub implements SurveyUnitQualityToolPort { + List>> receivedMaps = new ArrayList<>(); + + @Override + public ResponseEntity sendProcessedIds(Map> processedIdsMap) { + receivedMaps.add(processedIdsMap); + return ResponseEntity.ok().build(); + } +}