Merge pull request #248 from marcberger/feat/feat_exportrawresponse_generic2

feat: add /responses/raw/lunatic-json/{campaignId} endpoint

Author: alexisszmundy

.gitignore

@@ -48,3 +48,4 @@ src/test/resources/OUT/*
 /src/test/resources/DONE/*
 /src/test/resources/genesis_deleted_schedules/*
 /src/test/resources/IN/WEB/SAMPLETEST/
+/src/main/resources/application-mbg.properties

src/main/java/fr/insee/genesis/controller/rest/responses/RawResponseController.java

@@ -16,34 +16,41 @@
 import io.swagger.v3.oas.annotations.Hidden;
 import io.swagger.v3.oas.annotations.Operation;
 import lombok.extern.slf4j.Slf4j;
+import org.springframework.data.domain.Page;
+import org.springframework.data.domain.PageRequest;
+import org.springframework.data.domain.Pageable;
+import org.springframework.data.web.PagedModel;
+import org.springframework.format.annotation.DateTimeFormat;
+import org.springframework.http.HttpStatus;
 import org.springframework.http.ResponseEntity;
 import org.springframework.security.access.prepost.PreAuthorize;
 import org.springframework.stereotype.Controller;
 import org.springframework.web.bind.annotation.GetMapping;
+import org.springframework.web.bind.annotation.PathVariable;
 import org.springframework.web.bind.annotation.PostMapping;
 import org.springframework.web.bind.annotation.PutMapping;
 import org.springframework.web.bind.annotation.RequestBody;
 import org.springframework.web.bind.annotation.RequestMapping;
 import org.springframework.web.bind.annotation.RequestParam;
 
+import java.time.Instant;
 import java.time.LocalDateTime;
 import java.util.ArrayList;
-import java.util.Arrays;
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
 import java.util.stream.Collectors;
 
 @Slf4j
 @Controller
-@RequestMapping(path = "/responses/raw" )
+@RequestMapping(path = "/responses/raw")
 public class RawResponseController {
 
     private static final String SUCCESS_MESSAGE = "Interrogation %s saved";
     private static final String PARTITION_ID = "partitionId";
     private static final String INTERROGATION_ID = "interrogationId";
     private final LunaticJsonRawDataApiPort lunaticJsonRawDataApiPort;
-    
+
     public RawResponseController(LunaticJsonRawDataApiPort lunaticJsonRawDataApiPort) {
         this.lunaticJsonRawDataApiPort = lunaticJsonRawDataApiPort;
     }
@@ -60,7 +67,7 @@ public ResponseEntity<String> saveRawResponsesFromJsonBody(
             @RequestParam(value = "mode") Mode modeSpecified,
             @RequestBody Map<String, Object> dataJson
     ) {
-        log.info("Try to save interrogationId {} for campaign {}",interrogationId,campaignName);
+        log.info("Try to save interrogationId {} for campaign {}", interrogationId, campaignName);
         LunaticJsonRawDataModel rawData = LunaticJsonRawDataModel.builder()
                 .campaignId(campaignName)
                 .questionnaireId(questionnaireId)
@@ -72,12 +79,12 @@ public ResponseEntity<String> saveRawResponsesFromJsonBody(
                 .build();
         try {
             lunaticJsonRawDataApiPort.save(rawData);
-        } catch (Exception e){
+        } catch (Exception e) {
             return ResponseEntity.status(500).body("Unexpected error");
         }
-        log.info("Data saved for interrogationId {} and campaign {}",interrogationId, campaignName);
+        log.info("Data saved for interrogationId {} and campaign {}", interrogationId, campaignName);
         // Collect platform prefer code 201 in case of success
-        return ResponseEntity.status(201).body(String.format(SUCCESS_MESSAGE,interrogationId));
+        return ResponseEntity.status(201).body(String.format(SUCCESS_MESSAGE, interrogationId));
     }
 
     @Operation(summary = "Save lunatic json data from one interrogation in Genesis Database (with json " +
@@ -90,8 +97,8 @@ public ResponseEntity<String> saveRawResponsesFromJsonBodyWithValidation(
         JsonSchema jsonSchema = JsonSchemaFactory.getInstance(SpecVersion.VersionFlag.V7).getSchema(
                 RawResponseController.class.getResourceAsStream("/jsonSchemas/RawResponse.json")
         );
-        try{
-            if(jsonSchema == null){
+        try {
+            if (jsonSchema == null) {
                 throw new GenesisException(500, "No RawResponse json schema has been found");
             }
             Set<ValidationMessage> errors = jsonSchema.validate(
@@ -103,9 +110,9 @@ public ResponseEntity<String> saveRawResponsesFromJsonBodyWithValidation(
             validate(errors);
             //Check required ids
             checkRequiredIds(body);
-        }catch (JsonProcessingException jpe){
+        } catch (JsonProcessingException jpe) {
             return ResponseEntity.status(400).body(jpe.toString());
-        }catch (GenesisException ge){
+        } catch (GenesisException ge) {
             return ResponseEntity.status(ge.getStatus()).body(ge.getMessage());
         }
 
@@ -124,29 +131,29 @@ public ResponseEntity<String> saveRawResponsesFromJsonBodyWithValidation(
             return ResponseEntity.status(500).body("Unexpected error");
         }
 
-        log.info("Data saved for interrogationId {} and partition {}",body.get(INTERROGATION_ID).toString(),
+        log.info("Data saved for interrogationId {} and partition {}", body.get(INTERROGATION_ID).toString(),
                 body.get(PARTITION_ID).toString());
-        return ResponseEntity.status(201).body(String.format(SUCCESS_MESSAGE,body.get(INTERROGATION_ID).toString()));
+        return ResponseEntity.status(201).body(String.format(SUCCESS_MESSAGE, body.get(INTERROGATION_ID).toString()));
     }
 
     //GET unprocessed
     @Operation(summary = "Get campaign id and interrogationId from all unprocessed raw json data")
     @GetMapping(path = "/lunatic-json/get/unprocessed")
     @PreAuthorize("hasRole('SCHEDULER')")
-    public ResponseEntity<List<LunaticJsonRawDataUnprocessedDto>> getUnproccessedJsonRawData(){
+    public ResponseEntity<List<LunaticJsonRawDataUnprocessedDto>> getUnproccessedJsonRawData() {
         log.info("Try to get unprocessed raw JSON datas...");
         return ResponseEntity.ok(lunaticJsonRawDataApiPort.getUnprocessedDataIds());
     }
 
     @Hidden
-    @GetMapping(path= "/lunatic-json/get/by-interrogation-mode-and-campaign")
+    @GetMapping(path = "/lunatic-json/get/by-interrogation-mode-and-campaign")
     @PreAuthorize("hasRole('ADMIN')")
-            public ResponseEntity<LunaticJsonRawDataModel> getJsonRawData(
-                @RequestParam(INTERROGATION_ID) String interrogationId,
-                @RequestParam("campaignName") String campaignName,
-                @RequestParam(value = "mode") Mode modeSpecified
-            ){
-        List<LunaticJsonRawDataModel> data = lunaticJsonRawDataApiPort.getRawData(campaignName,modeSpecified,List.of(interrogationId));
+    public ResponseEntity<LunaticJsonRawDataModel> getJsonRawData(
+            @RequestParam(INTERROGATION_ID) String interrogationId,
+            @RequestParam("campaignName") String campaignName,
+            @RequestParam(value = "mode") Mode modeSpecified
+    ) {
+        List<LunaticJsonRawDataModel> data = lunaticJsonRawDataApiPort.getRawData(campaignName, modeSpecified, List.of(interrogationId));
         return ResponseEntity.ok(data.getFirst());
     }
 
@@ -158,7 +165,7 @@ public ResponseEntity<String> processJsonRawData(
             @RequestParam("campaignName") String campaignName,
             @RequestParam("questionnaireId") String questionnaireId,
             @RequestBody List<String> interrogationIdList
-    ){
+    ) {
         log.info("Try to process raw JSON datas for campaign {} and {} interrogationIds", campaignName, interrogationIdList.size());
         List<GenesisError> errors = new ArrayList<>();
 
@@ -168,13 +175,30 @@ public ResponseEntity<String> processJsonRawData(
                     ResponseEntity.ok("%d document(s) processed".formatted(result.dataCount()))
                     : ResponseEntity.ok("%d document(s) processed, including %d FORMATTED after data verification"
                     .formatted(result.dataCount(), result.formattedDataCount()));
-        }catch (GenesisException e){
+        } catch (GenesisException e) {
             return ResponseEntity.status(e.getStatus()).body(e.getMessage());
         }
     }
 
+    @Operation(summary = "Get lunatic JSON data from one campaign in Genesis Database, filtered by optional start and end dates")
+    @GetMapping(path = "/lunatic-json/{campaignId}")
+    @PreAuthorize("hasRole('USER_KRAFTWERK')")
+    public ResponseEntity<PagedModel<LunaticJsonRawDataModel>> getRawResponsesFromJsonBody(
+            @PathVariable String campaignId,
+            @RequestParam(value = "startDate", required = false) @DateTimeFormat(iso = DateTimeFormat.ISO.DATE_TIME) Instant startDate,
+            @RequestParam(value = "endDate", required = false) @DateTimeFormat(iso = DateTimeFormat.ISO.DATE_TIME) Instant endDate,
+            @RequestParam(value = "page", defaultValue = "0") int page,
+            @RequestParam(value = "size", defaultValue = "1000") int size
+    ) {
+        log.info("Try to read raw JSONs for campaign {}, with startDate={} and endDate={}", campaignId, startDate, endDate);
+        Pageable pageable = PageRequest.of(page, size);
+        Page<LunaticJsonRawDataModel> rawResponses = lunaticJsonRawDataApiPort.findRawDataByCampaignIdAndDate(campaignId, startDate, endDate, pageable);
+        log.info("rawResponses=" + rawResponses.getContent().size());
+        return ResponseEntity.status(HttpStatus.OK).body(new PagedModel<>(rawResponses));
+    }
+
     private void validate(Set<ValidationMessage> errors) throws GenesisException {
-        if(!errors.isEmpty()){
+        if (!errors.isEmpty()) {
             String errorMessage = errors.stream()
                     .map(ValidationMessage::getMessage)
                     .collect(Collectors.joining(System.lineSeparator() + " - "));
@@ -187,14 +211,14 @@ private void validate(Set<ValidationMessage> errors) throws GenesisException {
     }
 
     private void checkRequiredIds(Map<String, Object> body) throws GenesisException {
-        for(String requiredKey : List.of(
+        for (String requiredKey : List.of(
                 PARTITION_ID,
                 "questionnaireModelId",
                 INTERROGATION_ID,
                 "surveyUnitId",
                 "mode"
-        )){
-            if(body.get(requiredKey) == null){
+        )) {
+            if (body.get(requiredKey) == null) {
                 throw new GenesisException(400, "No %s found in body".formatted(requiredKey));
             }
         }

src/main/java/fr/insee/genesis/domain/ports/api/LunaticJsonRawDataApiPort.java

@@ -8,7 +8,10 @@
 import fr.insee.genesis.domain.model.surveyunit.rawdata.LunaticJsonRawDataModel;
 import fr.insee.genesis.exceptions.GenesisError;
 import fr.insee.genesis.exceptions.GenesisException;
+import org.springframework.data.domain.Page;
+import org.springframework.data.domain.Pageable;
 
+import java.time.Instant;
 import java.util.List;
 import java.util.Set;
 
@@ -21,6 +24,7 @@ public interface LunaticJsonRawDataApiPort {
     void updateProcessDates(List<SurveyUnitModel> surveyUnitModels);
     Set<String> findDistinctQuestionnaireIds();
     long countResponsesByQuestionnaireId(String campaignId);
+    Page<LunaticJsonRawDataModel> findRawDataByCampaignIdAndDate(String campaignId, Instant  startDt, Instant endDt, Pageable pageable);
 
     DataProcessResult processRawData(String campaignName, List<String> interrogationIdList, List<GenesisError> errors) throws GenesisException;
 }

src/main/java/fr/insee/genesis/domain/ports/spi/LunaticJsonRawDataPersistencePort.java

@@ -2,7 +2,10 @@
 
 import fr.insee.genesis.domain.model.surveyunit.Mode;
 import fr.insee.genesis.domain.model.surveyunit.rawdata.LunaticJsonRawDataModel;
+import org.springframework.data.domain.Page;
+import org.springframework.data.domain.Pageable;
 
+import java.time.Instant;
 import java.util.List;
 import java.util.Set;
 
@@ -13,5 +16,6 @@ public interface LunaticJsonRawDataPersistencePort {
     List<LunaticJsonRawDataModel> getAllUnprocessedData();
     void updateProcessDates(String campaignId, Set<String> interrogationIds);
     Set<String> findDistinctQuestionnaireIds();
+    Page<LunaticJsonRawDataModel> findByCampaignIdAndDate(String campaignId, Instant startDt, Instant endDt, Pageable pageable);
     long countResponsesByQuestionnaireId(String questionnaireId);
 }

src/main/java/fr/insee/genesis/domain/service/rawdata/LunaticJsonRawDataService.java

@@ -28,10 +28,13 @@
 import lombok.extern.slf4j.Slf4j;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.beans.factory.annotation.Qualifier;
+import org.springframework.data.domain.Page;
+import org.springframework.data.domain.Pageable;
 import org.springframework.http.ResponseEntity;
 import org.springframework.stereotype.Service;
 
 import java.io.IOException;
+import java.time.Instant;
 import java.time.LocalDateTime;
 import java.util.ArrayList;
 import java.util.HashMap;
@@ -387,4 +390,12 @@ public Set<String> findDistinctQuestionnaireIds() {
     public long countResponsesByQuestionnaireId(String campaignId) {
         return lunaticJsonRawDataPersistencePort.countResponsesByQuestionnaireId(campaignId);
     }
+
+
+    @Override
+    public Page<LunaticJsonRawDataModel> findRawDataByCampaignIdAndDate(String campaignId, Instant startDt, Instant endDt, Pageable pageable){
+        return lunaticJsonRawDataPersistencePort.findByCampaignIdAndDate(campaignId,startDt, endDt,pageable);
+
+    }
+
 }

src/main/java/fr/insee/genesis/infrastructure/adapter/LunaticJsonRawDataMongoAdapter.java

@@ -9,12 +9,16 @@
 import fr.insee.genesis.infrastructure.repository.LunaticJsonMongoDBRepository;
 import lombok.extern.slf4j.Slf4j;
 import org.springframework.beans.factory.annotation.Qualifier;
+import org.springframework.data.domain.Page;
+import org.springframework.data.domain.PageImpl;
+import org.springframework.data.domain.Pageable;
 import org.springframework.data.mongodb.core.MongoTemplate;
 import org.springframework.data.mongodb.core.query.Criteria;
 import org.springframework.data.mongodb.core.query.Query;
 import org.springframework.data.mongodb.core.query.Update;
 import org.springframework.stereotype.Service;
 
+import java.time.Instant;
 import java.time.LocalDateTime;
 import java.util.HashSet;
 import java.util.List;
@@ -68,6 +72,13 @@ public Set<String> findDistinctQuestionnaireIds() {
         return questionnaireIds;
     }
 
+    @Override
+    public Page<LunaticJsonRawDataModel> findByCampaignIdAndDate(String campaignId, Instant startDt, Instant  endDt, Pageable pageable) {
+        Page<LunaticJsonRawDataDocument> rawDataDocsPage =  repository.findByCampaignIdAndRecordDateBetween(campaignId,startDt,endDt,pageable);
+        List<LunaticJsonRawDataModel> modelList = LunaticJsonRawDataDocumentMapper.INSTANCE.listDocumentToListModel(rawDataDocsPage.getContent());
+        return new PageImpl<>(modelList, rawDataDocsPage.getPageable(), rawDataDocsPage.getTotalElements());
+    }
+
     @Override
     public long countResponsesByQuestionnaireId(String questionnaireId) {
         return repository.countByQuestionnaireId(questionnaireId);

src/main/java/fr/insee/genesis/infrastructure/document/rundeck/RundeckExecutionDocument.java

@@ -2,7 +2,9 @@
 
 import com.fasterxml.jackson.annotation.JsonProperty;
 import fr.insee.genesis.domain.model.rundeck.DateStarted;
-import lombok.*;
+import lombok.AllArgsConstructor;
+import lombok.Data;
+import lombok.NoArgsConstructor;
 import org.springframework.data.mongodb.core.mapping.Document;
 
 @Data

src/main/java/fr/insee/genesis/infrastructure/repository/LunaticJsonMongoDBRepository.java

@@ -2,21 +2,28 @@
 
 import fr.insee.genesis.domain.model.surveyunit.Mode;
 import fr.insee.genesis.infrastructure.document.rawdata.LunaticJsonRawDataDocument;
+import org.springframework.data.domain.Page;
+import org.springframework.data.domain.Pageable;
 import org.springframework.data.mongodb.repository.MongoRepository;
 import org.springframework.data.mongodb.repository.Query;
 import org.springframework.stereotype.Repository;
 
+import java.time.Instant;
 import java.util.List;
 
 @Repository
-public interface LunaticJsonMongoDBRepository extends MongoRepository<LunaticJsonRawDataDocument,String> {
+public interface LunaticJsonMongoDBRepository extends MongoRepository<LunaticJsonRawDataDocument, String> {
 
     @Query("{\"processDate\" : null}")
     List<LunaticJsonRawDataDocument> findByNullProcessDate();
+
     @Query(value = "{ 'campaignId' : ?0 }", fields = "{ 'mode' :  1 }")
     List<Mode> findModesByCampaignId(String campaignId);
+
     @Query(value = "{ 'campaignId' : ?0, 'mode' : ?1, 'interrogationId': {$in: ?2} }")
     List<LunaticJsonRawDataDocument> findModesByCampaignIdAndByModeAndinterrogationIdIninterrogationIdList(String campaignName, Mode mode, List<String> interrogationIdList);
+
+    Page<LunaticJsonRawDataDocument> findByCampaignIdAndRecordDateBetween(String campagneId, Instant start, Instant  end, Pageable pageable);
     long countByQuestionnaireId(String questionnaireId);