Envoi des interrogationIds convertis à Perret

pom.xml

@@ -51,6 +51,10 @@
             <groupId>org.springframework.boot</groupId>
             <artifactId>spring-boot-starter-oauth2-resource-server</artifactId>
         </dependency>
+        <dependency>
+            <groupId>org.springframework.boot</groupId>
+            <artifactId>spring-boot-starter-webflux</artifactId>
+        </dependency>
 
         <dependency>
             <groupId>org.springframework.boot</groupId>

src/main/java/fr/insee/genesis/Constants.java

@@ -1,5 +1,8 @@
 package fr.insee.genesis;
 
+import lombok.experimental.UtilityClass;
+
+@UtilityClass
 public class Constants {
 
     public static final String XSLT_STRUCTURED_VARIABLES = "xslt/structured-variables.xsl";
@@ -58,11 +61,6 @@ public class Constants {
     // Kraftwerk service path parameters
     public static final String KRAFTWERK_MAIN_ENDPOINT = "";
 
-    private Constants() {
-        throw new IllegalStateException("Constants class");
-    }
-
-
     public static String[] getEnoVariables() {
         return ENO_VARIABLES;
     }

src/main/java/fr/insee/genesis/configuration/Config.java

@@ -31,6 +31,11 @@ public class Config {
 	@Value("${fr.insee.genesis.oidc.realm}")
 	private String realm;
 
+	@Value("${fr.insee.genesis.oidc.service.client-id}")
+	private String serviceClientId;
+	@Value("${fr.insee.genesis.oidc.service.client-secret}")
+	private String serviceClientSecret;
+
 	@Value("${fr.insee.genesis.security.token.oidc-claim-role}")
 	private String oidcClaimRole;
 
@@ -40,6 +45,12 @@ public class Config {
 	@Value("#{'${fr.insee.genesis.security.whitelist-matchers}'.split(',')}")
 	private String[] whiteList;
 
+	@Value("${fr.insee.genesis.rawdata.process.batch-size}")
+	private int rawDataProcessingBatchSize;
+
+	@Value("${fr.insee.genesis.survey-quality-tool.url}")
+	private String surveyQualityToolUrl;
+
 	private final String logFolder;
 
 	//Extract log folder from log filename property

src/main/java/fr/insee/genesis/domain/ports/spi/SurveyUnitQualityToolPort.java

@@ -0,0 +1,11 @@
+package fr.insee.genesis.domain.ports.spi;
+
+import org.springframework.http.ResponseEntity;
+
+import java.io.IOException;
+import java.util.Map;
+import java.util.Set;
+
+public interface SurveyUnitQualityToolPort {
+    ResponseEntity<Object> sendProcessedIds(Map<String, Set<String>> processedIdsMap) throws IOException;
+}

src/main/java/fr/insee/genesis/domain/service/rawdata/LunaticJsonRawDataService.java

@@ -2,9 +2,11 @@
 
 import fr.insee.bpm.metadata.model.VariablesMap;
 import fr.insee.genesis.Constants;
+import fr.insee.genesis.configuration.Config;
 import fr.insee.genesis.controller.dto.rawdata.LunaticJsonRawDataUnprocessedDto;
 import fr.insee.genesis.controller.services.MetadataService;
 import fr.insee.genesis.controller.utils.ControllerUtils;
+import fr.insee.genesis.domain.model.context.DataProcessingContextModel;
 import fr.insee.genesis.domain.model.surveyunit.DataState;
 import fr.insee.genesis.domain.model.surveyunit.Mode;
 import fr.insee.genesis.domain.model.surveyunit.SurveyUnitModel;
@@ -14,6 +16,8 @@
 import fr.insee.genesis.domain.model.surveyunit.rawdata.RawDataModelType;
 import fr.insee.genesis.domain.ports.api.LunaticJsonRawDataApiPort;
 import fr.insee.genesis.domain.ports.spi.LunaticJsonRawDataPersistencePort;
+import fr.insee.genesis.domain.ports.spi.SurveyUnitQualityToolPort;
+import fr.insee.genesis.domain.service.context.DataProcessingContextService;
 import fr.insee.genesis.domain.service.surveyunit.SurveyUnitQualityService;
 import fr.insee.genesis.domain.service.surveyunit.SurveyUnitService;
 import fr.insee.genesis.domain.utils.GroupUtils;
@@ -24,10 +28,13 @@
 import lombok.extern.slf4j.Slf4j;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.beans.factory.annotation.Qualifier;
+import org.springframework.http.ResponseEntity;
 import org.springframework.stereotype.Service;
 
+import java.io.IOException;
 import java.time.LocalDateTime;
 import java.util.ArrayList;
+import java.util.HashMap;
 import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
@@ -41,20 +48,34 @@ public class LunaticJsonRawDataService implements LunaticJsonRawDataApiPort {
     private final MetadataService metadataService;
     private final SurveyUnitService surveyUnitService;
     private final SurveyUnitQualityService surveyUnitQualityService;
+    private final SurveyUnitQualityToolPort surveyUnitQualityToolPort;
+    private final DataProcessingContextService dataProcessingContextService;
     private final FileUtils fileUtils;
-    private static final int BATCH_SIZE = 1000;
+    private final Config config;
 
     @Qualifier("lunaticJsonMongoAdapterNew")
     private final LunaticJsonRawDataPersistencePort lunaticJsonRawDataPersistencePort;
 
     @Autowired
-    public LunaticJsonRawDataService(LunaticJsonRawDataPersistencePort lunaticJsonRawDataNewPersistencePort, ControllerUtils controllerUtils, MetadataService metadataService, SurveyUnitService surveyUnitService, SurveyUnitQualityService surveyUnitQualityService, FileUtils fileUtils) {
+    public LunaticJsonRawDataService(LunaticJsonRawDataPersistencePort lunaticJsonRawDataNewPersistencePort,
+                                     ControllerUtils controllerUtils,
+                                     MetadataService metadataService,
+                                     SurveyUnitService surveyUnitService,
+                                     SurveyUnitQualityService surveyUnitQualityService,
+                                     FileUtils fileUtils,
+                                     DataProcessingContextService dataProcessingContextService,
+                                     SurveyUnitQualityToolPort surveyUnitQualityToolPort,
+                                     Config config
+    ) {
         this.controllerUtils = controllerUtils;
         this.metadataService = metadataService;
         this.surveyUnitService = surveyUnitService;
         this.surveyUnitQualityService = surveyUnitQualityService;
         this.fileUtils = fileUtils;
         this.lunaticJsonRawDataPersistencePort = lunaticJsonRawDataNewPersistencePort;
+        this.surveyUnitQualityToolPort = surveyUnitQualityToolPort;
+        this.dataProcessingContextService = dataProcessingContextService;
+        this.config = config;
     }
 
     @Override
@@ -71,6 +92,8 @@ public List<LunaticJsonRawDataModel> getRawData(String campaignName, Mode mode,
     public DataProcessResult processRawData(String campaignName, List<String> interrogationIdList, List<GenesisError> errors) throws GenesisException {
         int dataCount=0;
         int formattedDataCount=0;
+        DataProcessingContextModel dataProcessingContext =
+                dataProcessingContextService.getContextByPartitionId(campaignName);
         List<Mode> modesList = controllerUtils.getModesList(campaignName, null);
         for (Mode mode : modesList) {
             //Load and save metadata into database, throw exception if none
@@ -82,12 +105,12 @@ public DataProcessResult processRawData(String campaignName, List<String> interr
                                 .formatted(mode, errors.getLast().getMessage())
                 );
             }
-            int totalBatchs = Math.ceilDiv(interrogationIdList.size() , BATCH_SIZE);
+            int totalBatchs = Math.ceilDiv(interrogationIdList.size() , config.getRawDataProcessingBatchSize());
             int batchNumber = 1;
             List<String> interrogationIdListForMode = new ArrayList<>(interrogationIdList);
             while(!interrogationIdListForMode.isEmpty()){
                 log.info("Processing raw data batch {}/{}", batchNumber, totalBatchs);
-                int maxIndex = Math.min(interrogationIdListForMode.size(), BATCH_SIZE);
+                int maxIndex = Math.min(interrogationIdListForMode.size(), config.getRawDataProcessingBatchSize());
                 List<String> interrogationIdToProcess = interrogationIdListForMode.subList(0, maxIndex);
 
                 List<LunaticJsonRawDataModel> rawData = getRawData(campaignName,mode, interrogationIdToProcess);
@@ -110,6 +133,23 @@ public DataProcessResult processRawData(String campaignName, List<String> interr
                         surveyUnitModel -> surveyUnitModel.getState().equals(DataState.FORMATTED)
                 ).toList().size();
 
+                //Send processed ids grouped by questionnaire (if review activated)
+                if(dataProcessingContext != null && dataProcessingContext.isWithReview()) {
+                    try {
+                        ResponseEntity<Object> response =
+                                surveyUnitQualityToolPort.sendProcessedIds(getProcessedIdsMap(surveyUnitModels));
+
+                    if (response.getStatusCode().is2xxSuccessful()) {
+                        log.info("Successfully sent {} ids to quality tool", getProcessedIdsMap(surveyUnitModels).keySet().size());
+                    }else{
+                        log.warn("Survey unit quality tool responded non-2xx code {} and body {}",
+                                response.getStatusCode(), response.getBody());
+                    }
+                    }catch (IOException e){
+                        log.error("Error during Perret call request building : {}", e.toString());
+                    }
+                }
+
                 //Remove processed ids from list
                 interrogationIdListForMode = interrogationIdListForMode.subList(maxIndex, interrogationIdListForMode.size());
 
@@ -119,6 +159,16 @@ public DataProcessResult processRawData(String campaignName, List<String> interr
         return new DataProcessResult(dataCount, formattedDataCount);
     }
 
+    private Map<String, Set<String>> getProcessedIdsMap(List<SurveyUnitModel> surveyUnitModels) {
+        Map<String, Set<String>> processedInterrogationIdsPerQuestionnaire = new HashMap<>();
+        surveyUnitModels.forEach(model ->
+                processedInterrogationIdsPerQuestionnaire
+                        .computeIfAbsent(model.getQuestionnaireId(), k -> new HashSet<>())
+                        .add(model.getInterrogationId())
+        );
+        return processedInterrogationIdsPerQuestionnaire;
+    }
+
     @Override
     public List<SurveyUnitModel> convertRawData(List<LunaticJsonRawDataModel> rawDataList, VariablesMap variablesMap) {
         //Convert to genesis model

src/main/java/fr/insee/genesis/domain/service/surveyunit/SurveyUnitService.java

@@ -505,7 +505,6 @@ private void extractVariables(SurveyUnitModel surveyUnitModel,
             surveyUnitModel.setCollectedVariables(new ArrayList<>());
         }
         for (VariableModel collectedVariable : surveyUnitModel.getCollectedVariables()) {
-            log.info("Variable name : {}", collectedVariable.varId());
             VarIdScopeTuple loopIdTuple = new VarIdScopeTuple(collectedVariable.varId(), collectedVariable.scope(),
                     collectedVariable.iteration());
             VariableDto variableDto = collectedVariableMap.get(loopIdTuple);

src/main/java/fr/insee/genesis/infrastructure/adapter/SurveyQualityToolPerretAdapter.java

@@ -0,0 +1,35 @@
+package fr.insee.genesis.infrastructure.adapter;
+
+import fr.insee.genesis.configuration.Config;
+import fr.insee.genesis.domain.ports.spi.SurveyUnitQualityToolPort;
+import fr.insee.genesis.infrastructure.utils.http.OidcService;
+import fr.insee.genesis.infrastructure.utils.http.HttpUtils;
+import lombok.extern.slf4j.Slf4j;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.beans.factory.annotation.Qualifier;
+import org.springframework.http.HttpMethod;
+import org.springframework.http.ResponseEntity;
+import org.springframework.stereotype.Service;
+
+import java.io.IOException;
+import java.util.Map;
+import java.util.Set;
+
+@Service
+@Qualifier("surveyQualityToolPerretAdapter")
+@Slf4j
+public class SurveyQualityToolPerretAdapter implements SurveyUnitQualityToolPort {
+
+    @Autowired
+    Config config;
+
+    @Autowired
+    OidcService oidcService;
+
+
+    @Override
+    public ResponseEntity<Object> sendProcessedIds(Map<String, Set<String>> processedIdsMap) throws IOException {
+        return HttpUtils.makeApiCall(config.getSurveyQualityToolUrl(), "/interrogations/",HttpMethod.POST, processedIdsMap,
+                Object.class, oidcService);
+    }
+}

src/main/java/fr/insee/genesis/infrastructure/document/surveyunit/SurveyUnitDocument.java

@@ -3,6 +3,7 @@
 import fr.insee.genesis.Constants;
 import lombok.Data;
 import org.springframework.data.mongodb.core.index.CompoundIndex;
+import org.springframework.data.mongodb.core.index.Indexed;
 import org.springframework.data.mongodb.core.mapping.Document;
 
 import java.time.LocalDateTime;
@@ -13,13 +14,12 @@
 @CompoundIndex(name = "campaignId_1_questionnaireId_1", def = "{'campaignId': 1, 'questionnaireId': 1}") //1 = ascending, -1 = descending
 @CompoundIndex(name = "questionnaireId_1_campaignId_1", def = "{'questionnaireId': 1, 'campaignId': 1}")
 @CompoundIndex(name = "questionnaireId_1_interrogationId_1", def = "{'questionnaireId': 1, 'interrogationId': 1}")
+@CompoundIndex(name = "interrogationId_1_questionnaireId_1", def = "{'interrogationId': 1, 'questionnaireId': 1}")
 public class SurveyUnitDocument {
 	private String campaignId;
+	@Indexed
 	private String interrogationId;
 	private String questionnaireId;
-	private String contextualId;
-	private Boolean isCapturedIndirectly;
-	private LocalDateTime validationDate;
 	private String state;
 	private String mode;
 	private LocalDateTime recordDate;

src/main/java/fr/insee/genesis/infrastructure/utils/http/HttpUtils.java

@@ -0,0 +1,36 @@
+package fr.insee.genesis.infrastructure.utils.http;
+
+import lombok.experimental.UtilityClass;
+import org.springframework.http.HttpHeaders;
+import org.springframework.http.HttpMethod;
+import org.springframework.http.MediaType;
+import org.springframework.http.ResponseEntity;
+import org.springframework.web.reactive.function.client.WebClient;
+import reactor.core.publisher.Mono;
+
+import java.io.IOException;
+
+@UtilityClass
+public class HttpUtils {
+    public static <T, R> ResponseEntity<R> makeApiCall(String baseUrl,
+                                                       String path,
+                                                       HttpMethod method,
+                                                       T requestBody,
+                                                       Class<R> responseType,
+                                                       OidcService oidcService
+                                                       ) throws IOException {
+        WebClient webClient = WebClient.builder()
+                .baseUrl(baseUrl)
+                .filter(PerretAuthWebClientFilter.perretAuthFilter(oidcService))
+                .build();
+        WebClient.ResponseSpec responseSpec = webClient
+                .method(method)
+                .uri(path)
+                .bodyValue(requestBody)
+                .header(HttpHeaders.CONTENT_TYPE, MediaType.APPLICATION_JSON_VALUE)
+                .header(HttpHeaders.AUTHORIZATION, "Bearer " + oidcService.getServiceAccountToken())
+                .retrieve();
+        Mono<ResponseEntity<R>> response = responseSpec.toEntity(responseType);
+        return response.block();
+    }
+}

src/main/java/fr/insee/genesis/infrastructure/utils/http/OidcService.java

@@ -0,0 +1,76 @@
+package fr.insee.genesis.infrastructure.utils.http;
+
+import fr.insee.genesis.configuration.Config;
+import lombok.Getter;
+import lombok.extern.slf4j.Slf4j;
+import org.springframework.http.HttpEntity;
+import org.springframework.http.HttpHeaders;
+import org.springframework.http.MediaType;
+import org.springframework.http.ResponseEntity;
+import org.springframework.stereotype.Service;
+import org.springframework.web.client.HttpStatusCodeException;
+import org.springframework.web.client.RestTemplate;
+
+import java.io.IOException;
+import java.util.Map;
+
+@Service
+@Slf4j
+public class OidcService {
+
+    public static final String ACCESS_TOKEN = "access_token";
+    @Getter
+    private final Config config;
+    private String serviceAccountToken;
+    private long tokenExpirationTime = 0;
+    private final RestTemplate restTemplate;
+
+    public OidcService(Config config) {
+        this.restTemplate = new RestTemplate();
+        this.config = config;
+    }
+
+    protected void retrieveServiceAccountToken() throws IOException {
+        String tokenUrl = String.format("%s/realms/%s/protocol/openid-connect/token",
+                config.getAuthServerUrl(),
+                config.getRealm());
+
+        HttpHeaders headers = new HttpHeaders();
+        headers.setContentType(MediaType.APPLICATION_FORM_URLENCODED);
+
+        String body = String.format("grant_type=client_credentials&client_id=%s&client_secret=%s&scope=openid profile roles",
+                config.getServiceClientId(),
+                config.getServiceClientSecret());
+
+        HttpEntity<String> request = new HttpEntity<>(body, headers);
+        try {
+            ResponseEntity<Map> response = restTemplate.postForEntity(tokenUrl, request, Map.class);
+            if (response.getStatusCode().is2xxSuccessful() && response.hasBody() && response.getBody() != null) {
+                Map<String, Object> responseBody = response.getBody();
+
+                if (responseBody == null || !responseBody.containsKey(ACCESS_TOKEN) || !(responseBody.get(ACCESS_TOKEN) instanceof String)) {
+                    throw new IOException("Invalid response: Missing or incorrect 'access_token'");
+                }
+
+                serviceAccountToken = (String) responseBody.get(ACCESS_TOKEN);
+                Integer expiresIn = (Integer) responseBody.get("expires_in");
+                tokenExpirationTime = System.currentTimeMillis() + (expiresIn.longValue() * 1000L);
+            } else {
+                throw new IOException("Failed to retrieve service account token, status: " + response.getStatusCode());
+            }
+        } catch (HttpStatusCodeException e) {
+            throw new IOException("HTTP error: " + e.getStatusCode() + " - " + e.getResponseBodyAsString(), e);
+        } catch (Exception e) {
+            throw new IOException("Unexpected error while fetching token: " + e.getMessage(), e);
+        }
+    }
+
+    public synchronized String getServiceAccountToken() throws IOException {
+        //We had a margin of 5 seconds for the expiration time
+        if (serviceAccountToken == null || System.currentTimeMillis() >= tokenExpirationTime - 5000L) {
+            retrieveServiceAccountToken();
+        }
+        return serviceAccountToken;
+    }
+
+}