Integration of the modelefiliere library

.mvn/wrapper/maven-wrapper.properties

@@ -0,0 +1,2 @@
+distributionUrl=https://repo.maven.apache.org/maven2/org/apache/maven/apache-maven/3.9.11/apache-maven-3.9.11-bin.zip
+wrapperUrl=https://repo.maven.apache.org/maven2/org/apache/maven/wrapper/maven-wrapper/3.3.4/maven-wrapper-3.3.4.jar

pom.xml

@@ -98,7 +98,16 @@
             <artifactId>springdoc-openapi-starter-webmvc-ui</artifactId>
             <version>${springdoc.version}</version>
         </dependency>
-
+        <dependency>
+            <groupId>fr.insee</groupId>
+            <artifactId>modelefiliere</artifactId>
+            <version>2.0.0</version>
+        </dependency>
+        <dependency>
+            <groupId>com.networknt</groupId>
+            <artifactId>json-schema-validator</artifactId>
+            <version>1.5.9</version>
+        </dependency>
         <!-- XML libraries -->
         <!-- XML-XSLT with Saxon -->
         <dependency>
@@ -107,13 +116,6 @@
             <version>12.9</version>
         </dependency>
 
-        <!-- JSON -->
-        <dependency>
-            <groupId>com.networknt</groupId>
-            <artifactId>json-schema-validator</artifactId>
-            <version>2.0.0</version>
-        </dependency>
-
         <!-- generate implementation auto -->
         <dependency>
             <groupId>org.mapstruct</groupId>

src/main/java/fr/insee/genesis/Constants.java

@@ -23,8 +23,7 @@ public class Constants {
     public static final String MONGODB_CONTEXTUAL_EXTERNAL_COLLECTION_NAME = "editedExternal";
     public static final String LOOP_NAME_PREFIX = "BOUCLE";
     public static final String MONGODB_RESPONSE_COLLECTION_NAME = "responses";
-    public static final String MONGODB_RESPONSE_RAW_COLLECTION_NAME = "lunaticjsondata";
-    public static final String MONGODB_VARIABLETYPE_COLLECTION_NAME = "variabletypes";
+    public static final String MONGODB_RAW_RESPONSES_COLLECTION_NAME = "rawResponses";
     public static final String VOLUMETRY_FOLDER_NAME = "genesis_volumetries";
     public static final String VOLUMETRY_FILE_SUFFIX = "_VOLUMETRY";
     public static final String VOLUMETRY_RAW_FILE_SUFFIX = "_RAW_VOLUMETRY";

src/main/java/fr/insee/genesis/controller/adapter/LunaticJsonAdapter.java

@@ -13,7 +13,7 @@ public class LunaticJsonAdapter {
 
     public SurveyUnitModel convert(LunaticJsonSurveyUnit su){
         return SurveyUnitModel.builder()
-                .questionnaireId(su.getQuestionnaireId())
+                .collectionInstrumentId(su.getQuestionnaireId())
                 .campaignId("")
                 .interrogationId(su.getInterrogationId())
                 .state(DataState.COLLECTED)

src/main/java/fr/insee/genesis/controller/adapter/LunaticXmlAdapter.java

@@ -69,7 +69,7 @@ public static List<SurveyUnitModel> convert(LunaticXmlSurveyUnit su, VariablesMa
      */
     private static SurveyUnitModel getStateDataFromSurveyUnit(LunaticXmlSurveyUnit su, VariablesMap variablesMap, String campaignId, DataState dataState, Mode mode) {
         SurveyUnitModel surveyUnitModel = SurveyUnitModel.builder()
-                .questionnaireId(su.getQuestionnaireModelId().toUpperCase())
+                .collectionInstrumentId(su.getQuestionnaireModelId().toUpperCase())
                 .campaignId(campaignId)
                 .interrogationId(su.getId())
                 .state(dataState)

src/main/java/fr/insee/genesis/controller/dto/ScheduleDto.java

@@ -8,6 +8,7 @@
 
 @Builder
 public record ScheduleDto (String surveyName,
+						   String collectionInstrumentId,
 						   LocalDateTime lastExecution,
 						   List<KraftwerkExecutionSchedule> kraftwerkExecutionScheduleList
 ){}

src/main/java/fr/insee/genesis/controller/dto/SurveyUnitSimplified.java

@@ -11,10 +11,14 @@
 @Data
 public class SurveyUnitSimplified {
 
-	private String questionnaireId;
+	private String collectionInstrumentId;
+	/**
+	 * @deprecated We will not reveive this piece of information anymore
+	 */
+	@Deprecated(forRemoval = true)
 	private String campaignId;
 	private String interrogationId;
-	private String surveyUnitId;
+	private String usualSurveyUnitId;
 	private Mode mode;
 	private List<VariableModel> variablesUpdate;
 	private List<VariableModel> externalVariables;

src/main/java/fr/insee/genesis/controller/rest/JsonExtractionController.java

@@ -30,11 +30,11 @@ public class JsonExtractionController {
     @PutMapping(path = "/json")
     @PreAuthorize("hasAnyRole('USER_KRAFTWERK','SCHEDULER')")
     public ResponseEntity<String> saveLastJsonExtractionDate(
-            @RequestParam("questionnaireId") String questionnaireId,
+            @RequestParam("collectionInstrumentId") String collectionInstrumentId,
             @RequestParam(value = "mode", required = false) Mode mode){
         LocalDateTime extractDate = LocalDateTime.now();
         LastJsonExtractionModel extract = LastJsonExtractionModel.builder()
-                .questionnaireModelId(questionnaireId)
+                .collectionInstrumentId(collectionInstrumentId)
                 .mode(mode)
                 .lastExtractionDate(extractDate)
                 .build();
@@ -46,10 +46,10 @@ public ResponseEntity<String> saveLastJsonExtractionDate(
     @GetMapping(path = "/json")
     @PreAuthorize("hasAnyRole('USER_KRAFTWERK','SCHEDULER')")
     public ResponseEntity<LastExtractionResponseDto> getLastJsonExtractionDate(
-            @RequestParam("questionnaireId") String questionnaireId,
+            @RequestParam("collectionInstrumentId") String collectionInstrumentId,
             @RequestParam(value = "mode", required = false) Mode mode){
         try{
-            LastJsonExtractionModel lastJsonExtraction = lastJsonExtractionApiPort.getLastExtractionDate(questionnaireId,mode);
+            LastJsonExtractionModel lastJsonExtraction = lastJsonExtractionApiPort.getLastExtractionDate(collectionInstrumentId,mode);
             return ResponseEntity.ok(new LastExtractionResponseDto(lastJsonExtraction.getLastExtractionDate()));
         } catch (GenesisException e){
             return ResponseEntity.notFound().build();

src/main/java/fr/insee/genesis/controller/rest/responses/ModeController.java

@@ -26,10 +26,10 @@ public ModeController(SurveyUnitApiPort surveyUnitService) {
     }
 
 
-    @Operation(summary = "List sources/modes used for a given questionnaire")
+    @Operation(summary = "List sources/modes used for a given collection instrument (ex questionnaire)")
     @GetMapping(path = "/by-questionnaire")
-    public ResponseEntity<List<Mode>> getModesByQuestionnaire(@RequestParam("questionnaireId") String questionnaireId) {
-        List<Mode> modes = surveyUnitService.findModesByQuestionnaireId(questionnaireId);
+    public ResponseEntity<List<Mode>> getModesByQuestionnaire(@RequestParam("questionnaireId") String collectionInstrumentId) {
+        List<Mode> modes = surveyUnitService.findModesByCollectionInstrumentId(collectionInstrumentId);
         return ResponseEntity.ok(modes);
     }
 

src/main/java/fr/insee/genesis/controller/rest/responses/RawResponseController.java

@@ -1,20 +1,28 @@
 package fr.insee.genesis.controller.rest.responses;
 
 import com.fasterxml.jackson.core.JsonProcessingException;
+import com.fasterxml.jackson.databind.DeserializationFeature;
 import com.fasterxml.jackson.databind.ObjectMapper;
-import com.networknt.schema.Error;
-import com.networknt.schema.Schema;
-import com.networknt.schema.SchemaRegistry;
-import com.networknt.schema.dialect.Dialects;
+import com.fasterxml.jackson.databind.SerializationFeature;
+import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule;
 import fr.insee.genesis.controller.dto.rawdata.LunaticJsonRawDataUnprocessedDto;
+import fr.insee.genesis.controller.utils.ExtendedJsonNormalizer;
+import fr.insee.genesis.controller.utils.JsonSchemaValidator;
+import fr.insee.genesis.controller.utils.SchemaType;
 import fr.insee.genesis.domain.model.surveyunit.Mode;
 import fr.insee.genesis.domain.model.surveyunit.rawdata.DataProcessResult;
 import fr.insee.genesis.domain.model.surveyunit.rawdata.LunaticJsonRawDataModel;
 import fr.insee.genesis.domain.ports.api.LunaticJsonRawDataApiPort;
+import fr.insee.genesis.domain.ports.api.RawResponseApiPort;
 import fr.insee.genesis.exceptions.GenesisError;
 import fr.insee.genesis.exceptions.GenesisException;
+import fr.insee.genesis.exceptions.SchemaValidationException;
+import fr.insee.genesis.infrastructure.repository.RawResponseInputRepository;
+import fr.insee.modelefiliere.RawResponseDto;
 import io.swagger.v3.oas.annotations.Hidden;
 import io.swagger.v3.oas.annotations.Operation;
+import io.swagger.v3.oas.annotations.Parameter;
+import jakarta.validation.Valid;
 import lombok.extern.slf4j.Slf4j;
 import org.springframework.data.domain.Page;
 import org.springframework.data.domain.PageRequest;
@@ -30,9 +38,9 @@
 import org.springframework.web.bind.annotation.PostMapping;
 import org.springframework.web.bind.annotation.PutMapping;
 import org.springframework.web.bind.annotation.RequestBody;
-import org.springframework.web.bind.annotation.RequestMapping;
 import org.springframework.web.bind.annotation.RequestParam;
 
+import java.io.IOException;
 import java.time.Instant;
 import java.time.LocalDateTime;
 import java.util.ArrayList;
@@ -42,20 +50,24 @@
 
 @Slf4j
 @Controller
-@RequestMapping(path = "/responses/raw")
 public class RawResponseController {
 
     private static final String SUCCESS_MESSAGE = "Interrogation %s saved";
     private static final String PARTITION_ID = "partitionId";
     private static final String INTERROGATION_ID = "interrogationId";
     private final LunaticJsonRawDataApiPort lunaticJsonRawDataApiPort;
+    private final RawResponseApiPort rawResponseApiPort;
+    private final RawResponseInputRepository rawRepository;
 
-    public RawResponseController(LunaticJsonRawDataApiPort lunaticJsonRawDataApiPort) {
+
+    public RawResponseController(LunaticJsonRawDataApiPort lunaticJsonRawDataApiPort, RawResponseApiPort rawResponseApiPort, RawResponseInputRepository rawRepository) {
         this.lunaticJsonRawDataApiPort = lunaticJsonRawDataApiPort;
+        this.rawResponseApiPort = rawResponseApiPort;
+        this.rawRepository = rawRepository;
     }
 
     @Operation(summary = "Save lunatic json data from one interrogation in Genesis Database")
-    @PutMapping(path = "/lunatic-json/save")
+    @PutMapping(path = "/responses/raw/lunatic-json/save")
     @PreAuthorize("hasRole('COLLECT_PLATFORM')")
     public ResponseEntity<String> saveRawResponsesFromJsonBody(
 
@@ -86,16 +98,18 @@ public ResponseEntity<String> saveRawResponsesFromJsonBody(
         return ResponseEntity.status(201).body(String.format(SUCCESS_MESSAGE, interrogationId));
     }
 
-    @Operation(summary = "Save lunatic json data from one interrogation in Genesis Database (with json " +
-            "schema validation)")
-    @PutMapping(path = "/lunatic-json")
+/*    @Operation(summary = "Deprecated")
+    @PutMapping(path="/lunatic-json")
     @PreAuthorize("hasRole('COLLECT_PLATFORM')")
-    public ResponseEntity<String> saveRawResponsesFromJsonBodyWithValidation(
+    // Check version when merging
+    @Deprecated(since="1.13.0", forRemoval=true)
+    public ResponseEntity<String> saveRawResponsesFromJsonBodyWithValidationDeprecated(
             @RequestBody Map<String, Object> body
     ) {
+
         SchemaRegistry schemaRegistry = SchemaRegistry.withDialect(Dialects.getDraft202012(), SchemaRegistry.Builder::build);
         Schema jsonSchema = schemaRegistry
-                .getSchema(RawResponseController.class.getResourceAsStream("/jsonSchemas/RawResponse.json")
+                .getSchema(RawResponseController.class.getResourceAsStream("/modele-filiere-spec/RawResponse.json")
         );
         try {
             if (jsonSchema == null) {
@@ -134,19 +148,92 @@ public ResponseEntity<String> saveRawResponsesFromJsonBodyWithValidation(
         log.info("Data saved for interrogationId {} and partition {}", body.get(INTERROGATION_ID).toString(),
                 body.get(PARTITION_ID).toString());
         return ResponseEntity.status(201).body(String.format(SUCCESS_MESSAGE, body.get(INTERROGATION_ID).toString()));
+    }*/
+
+    @Operation(summary = "Save lunatic json data from one interrogation in Genesis Database (with json " +
+            "schema validation)")
+    @PostMapping(path="/raw-responses/debug")
+    @PreAuthorize("hasRole('COLLECT_PLATFORM')")
+    public ResponseEntity<String> saveRawResponsesFromJsonBodyWithValidation(
+            @RequestBody Map<String, Object> body
+    ) throws JsonProcessingException {
+
+        ObjectMapper objectMapperLocal = new ObjectMapper();
+        objectMapperLocal.registerModule(new JavaTimeModule());
+        objectMapperLocal.disable(SerializationFeature.WRITE_DATES_AS_TIMESTAMPS);   // ISO-8601
+        objectMapperLocal.disable(DeserializationFeature.ADJUST_DATES_TO_CONTEXT_TIME_ZONE);
+
+        log.info(objectMapperLocal.writeValueAsString(body));
+        try {
+            RawResponseDto rawResponseDto = JsonSchemaValidator.readAndValidateFromClasspath(
+                    ExtendedJsonNormalizer.normalize(objectMapperLocal.readTree(
+                            objectMapperLocal.writeValueAsString(body))),
+                    SchemaType.RAW_RESPONSE.getSchemaFileName(),
+                    RawResponseDto.class,
+                    objectMapperLocal
+            );
+            rawRepository.saveAsRawJson(rawResponseDto);
+        } catch (SchemaValidationException | IOException e) {
+            return ResponseEntity.status(400).body(e.toString());
+        }
+        return ResponseEntity.status(201).body(String.format(SUCCESS_MESSAGE, body.get(INTERROGATION_ID).toString()));
+    }
+
+    @Operation(summary = "Save lunatic json data from one interrogation in Genesis Database (with json " +
+            "schema validation)")
+    @PostMapping(path="/raw-responses")
+    @PreAuthorize("hasRole('COLLECT_PLATFORM')")
+    public ResponseEntity<String> saveRawResponsesFromRawResponseDto(
+            @Valid @RequestBody RawResponseDto dto
+    ) {
+        rawRepository.saveAsRawJson(dto);
+        return ResponseEntity.status(201).body(String.format(SUCCESS_MESSAGE, dto.getInterrogationId()));
+    }
+
+    //PROCESS
+    @Operation(summary = "Process raw data of a campaign")
+    @PostMapping(path = "/raw-responses/process")
+    @PreAuthorize("hasRole('SCHEDULER')")
+    public ResponseEntity<String> processRawResponses(
+            @Parameter(
+                    description = "Id of the collection instrument (old questionnaireId)",
+                    example = "ENQTEST2025X00"
+            )
+            @RequestParam("collectionInstrumentId") String collectionInstrumentId,
+            @RequestBody List<String> interrogationIdList
+    ) {
+        log.info("Try to process raw responses for questionnaireId {} and {} interrogationIds", collectionInstrumentId, interrogationIdList.size());
+        List<GenesisError> errors = new ArrayList<>();
+
+        try {
+            DataProcessResult result = rawResponseApiPort.processRawResponses(collectionInstrumentId, interrogationIdList, errors);
+            return result.formattedDataCount() == 0 ?
+                    ResponseEntity.ok("%d document(s) processed".formatted(result.dataCount()))
+                    : ResponseEntity.ok("%d document(s) processed, including %d FORMATTED after data verification"
+                    .formatted(result.dataCount(), result.formattedDataCount()));
+        } catch (GenesisException e) {
+            return ResponseEntity.status(e.getStatus()).body(e.getMessage());
+        }
+    }
+    @Operation(summary = "Get the list of collection instruments containing unprocessed interrogations")
+    @GetMapping(path = "/raw-responses/unprocessed/collection-intrument-ids")
+    @PreAuthorize("hasRole('SCHEDULER')")
+    public ResponseEntity<List<String>> getUnprocessedCollectionInstrument(){
+        log.info("Try to get collection instruments containing unprocessed interrogations...");
+        return ResponseEntity.ok(rawResponseApiPort.getUnprocessedCollectionInstrumentIds());
     }
 
     //GET unprocessed
     @Operation(summary = "Get campaign id and interrogationId from all unprocessed raw json data")
-    @GetMapping(path = "/lunatic-json/get/unprocessed")
+    @GetMapping(path = "/responses/raw/lunatic-json/get/unprocessed")
     @PreAuthorize("hasRole('SCHEDULER')")
     public ResponseEntity<List<LunaticJsonRawDataUnprocessedDto>> getUnproccessedJsonRawData() {
         log.info("Try to get unprocessed raw JSON datas...");
         return ResponseEntity.ok(lunaticJsonRawDataApiPort.getUnprocessedDataIds());
     }
 
     @Hidden
-    @GetMapping(path = "/lunatic-json/get/by-interrogation-mode-and-campaign")
+    @GetMapping(path = "/responses/raw/lunatic-json/get/by-interrogation-mode-and-campaign")
     @PreAuthorize("hasRole('ADMIN')")
     public ResponseEntity<LunaticJsonRawDataModel> getJsonRawData(
             @RequestParam(INTERROGATION_ID) String interrogationId,
@@ -159,7 +246,7 @@ public ResponseEntity<LunaticJsonRawDataModel> getJsonRawData(
 
     //PROCESS
     @Operation(summary = "Process raw data of a campaign")
-    @PostMapping(path = "/lunatic-json/process")
+    @PostMapping(path = "/responses/raw/lunatic-json/process")
     @PreAuthorize("hasRole('SCHEDULER')")
     @Deprecated(since = "1.13.0")
     public ResponseEntity<String> processJsonRawData(
@@ -200,7 +287,7 @@ public ResponseEntity<String> processJsonRawData(
     }
 
     @Operation(summary = "Get processed data ids from last n hours (default 24h)")
-    @GetMapping(path = "/lunatic-json/processed/ids")
+    @GetMapping(path = "/responses/raw/lunatic-json/processed/ids")
     @PreAuthorize("hasRole('ADMIN')")
     public ResponseEntity<Map<String, List<String>>> getProcessedDataIdsSinceHours(
             @RequestParam("questionnaireId") String questionnaireId,
@@ -212,7 +299,7 @@ public ResponseEntity<Map<String, List<String>>> getProcessedDataIdsSinceHours(
     }
 
     @Operation(summary = "Get lunatic JSON data from one campaign in Genesis Database, filtered by start and end dates")
-    @GetMapping(path = "/lunatic-json/{campaignId}")
+    @GetMapping(path = "/responses/raw/lunatic-json/{campaignId}")
     @PreAuthorize("hasRole('USER_BATCH_GENERIC')")
     public ResponseEntity<PagedModel<LunaticJsonRawDataModel>> getRawResponsesFromJsonBody(
             @PathVariable String campaignId,