feat: Collection variables externes

.gitignore

@@ -49,4 +49,5 @@ src/test/resources/OUT/*
 /src/test/resources/genesis_deleted_schedules/*
 /src/test/resources/IN/WEB/SAMPLETEST/
 /src/test/resources/IN/WEB/TEST-EDITED-PREVIOUS/
+/src/test/resources/IN/WEB/TEST-EDITED-EXTERNAL/
 /src/main/resources/application-mbg.properties

src/main/java/fr/insee/genesis/Constants.java

@@ -19,6 +19,7 @@ public class Constants {
     public static final String MONGODB_SCHEDULE_COLLECTION_NAME = "schedules";
     public static final String MONGODB_CONTEXT_COLLECTION_NAME = "dataProcessingContexts";
     public static final String MONGODB_EDITED_PREVIOUS_COLLECTION_NAME = "editedPrevious";
+    public static final String MONGODB_EDITED_EXTERNAL_COLLECTION_NAME = "editedExternal";
     public static final String LOOP_NAME_PREFIX = "BOUCLE";
     public static final String MONGODB_RESPONSE_COLLECTION_NAME = "responses";
     public static final String MONGODB_RESPONSE_RAW_COLLECTION_NAME = "lunaticjsondata";

src/main/java/fr/insee/genesis/controller/rest/responses/EditedExternalResponseController.java

@@ -0,0 +1,74 @@
+package fr.insee.genesis.controller.rest.responses;
+
+import fr.insee.genesis.configuration.Config;
+import fr.insee.genesis.domain.model.surveyunit.Mode;
+import fr.insee.genesis.domain.ports.api.EditedExternalResponseApiPort;
+import fr.insee.genesis.exceptions.GenesisException;
+import fr.insee.genesis.infrastructure.utils.FileUtils;
+import io.swagger.v3.oas.annotations.Operation;
+import lombok.extern.slf4j.Slf4j;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.http.HttpStatusCode;
+import org.springframework.http.ResponseEntity;
+import org.springframework.security.access.prepost.PreAuthorize;
+import org.springframework.stereotype.Controller;
+import org.springframework.web.bind.annotation.PostMapping;
+import org.springframework.web.bind.annotation.RequestMapping;
+import org.springframework.web.bind.annotation.RequestParam;
+
+import java.io.FileInputStream;
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.file.Path;
+
+@RequestMapping(path = "/edited-external" )
+@Controller
+@Slf4j
+public class EditedExternalResponseController {
+    private final EditedExternalResponseApiPort editedExternalResponseApiPort;
+
+    private final Config config;
+
+    @Autowired
+    public EditedExternalResponseController(EditedExternalResponseApiPort editedExternalResponseApiPort, Config config) {
+        this.editedExternalResponseApiPort = editedExternalResponseApiPort;
+        this.config = config;
+    }
+
+    @Operation(summary = "Add edited external json file")
+    @PostMapping(path = "/json")
+    @PreAuthorize("hasRole('USER_KRAFTWERK')")
+    public ResponseEntity<Object> readJson(
+            @RequestParam("questionnaireId") String questionnaireId,
+            @RequestParam("mode") Mode mode,
+            @RequestParam(value = "jsonFileName") String jsonFileName
+    ){
+        try {
+            FileUtils fileUtils = new FileUtils(config);
+
+            String filePath = "%s/%s".formatted(
+                    fileUtils.getDataFolder(questionnaireId, mode.getFolder(), null),
+                    jsonFileName
+            );
+            if (!jsonFileName.toLowerCase().endsWith(".json")) {
+                throw new GenesisException(400, "File must be a JSON file !");
+            }
+            try (InputStream inputStream = new FileInputStream(filePath)) {
+                editedExternalResponseApiPort.readEditedExternalFile(inputStream, questionnaireId);
+            } catch (FileNotFoundException e) {
+                throw new GenesisException(404, "File %s not found".formatted(filePath));
+            } catch (IOException e) {
+                throw new GenesisException(500, e.toString());
+            }
+            try {
+                fileUtils.moveFiles(Path.of(filePath), fileUtils.getDoneFolder(questionnaireId, mode.getFolder()));
+            } catch (IOException e) {
+                throw new GenesisException(500, "Error while moving file to done : %s".formatted(e.toString()));
+            }
+            return ResponseEntity.ok("Edited external variable file %s saved !".formatted(filePath));
+        }catch (GenesisException ge){
+            return ResponseEntity.status(HttpStatusCode.valueOf(ge.getStatus())).body(ge.getMessage());
+        }
+    }
+}

src/main/java/fr/insee/genesis/controller/rest/responses/EditedPreviousResponseController.java

@@ -8,6 +8,7 @@
 import io.swagger.v3.oas.annotations.Operation;
 import lombok.extern.slf4j.Slf4j;
 import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.http.HttpStatusCode;
 import org.springframework.http.ResponseEntity;
 import org.springframework.security.access.prepost.PreAuthorize;
 import org.springframework.stereotype.Controller;
@@ -43,28 +44,32 @@ public ResponseEntity<Object> readJson(
             @RequestParam("mode") Mode mode,
             @RequestParam(value = "sourceState", required = false) String sourceState,
             @RequestParam(value = "jsonFileName") String jsonFileName
-    ) throws GenesisException {
-        FileUtils fileUtils = new FileUtils(config);
-
-        String filePath = "%s/%s".formatted(
-                fileUtils.getDataFolder(questionnaireId, mode.getFolder(), null),
-                jsonFileName
-        );
-        if(!jsonFileName.toLowerCase().endsWith(".json")){
-            throw new GenesisException(400, "File must be a JSON file !");
-        }
-        try(InputStream inputStream = new FileInputStream(filePath)){
-            editedPreviousResponseApiPort.readEditedPreviousFile(inputStream, questionnaireId, sourceState);
-        } catch (FileNotFoundException e) {
-            throw new GenesisException(404, "File %s not found".formatted(filePath));
-        } catch (IOException e) {
-            throw new GenesisException(500, e.toString());
-        }
+    ){
         try {
-            fileUtils.moveFiles(Path.of(filePath), fileUtils.getDoneFolder(questionnaireId, mode.getFolder()));
-        } catch (IOException e) {
-            throw new GenesisException(500, "Error while moving file to done : %s".formatted(e.toString()));
+            FileUtils fileUtils = new FileUtils(config);
+
+            String filePath = "%s/%s".formatted(
+                    fileUtils.getDataFolder(questionnaireId, mode.getFolder(), null),
+                    jsonFileName
+            );
+            if (!jsonFileName.toLowerCase().endsWith(".json")) {
+                throw new GenesisException(400, "File must be a JSON file !");
+            }
+            try (InputStream inputStream = new FileInputStream(filePath)) {
+                editedPreviousResponseApiPort.readEditedPreviousFile(inputStream, questionnaireId, sourceState);
+            } catch (FileNotFoundException e) {
+                throw new GenesisException(404, "File %s not found".formatted(filePath));
+            } catch (IOException e) {
+                throw new GenesisException(500, e.toString());
+            }
+            try {
+                fileUtils.moveFiles(Path.of(filePath), fileUtils.getDoneFolder(questionnaireId, mode.getFolder()));
+            } catch (IOException e) {
+                throw new GenesisException(500, "Error while moving file to done : %s".formatted(e.toString()));
+            }
+            return ResponseEntity.ok("Edited previous variable file %s saved !".formatted(filePath));
+        }catch (GenesisException ge){
+            return ResponseEntity.status(HttpStatusCode.valueOf(ge.getStatus())).body(ge.getMessage());
         }
-        return ResponseEntity.ok("Edited previous variable file %s saved !".formatted(filePath));
     }
 }

src/main/java/fr/insee/genesis/domain/model/editedexternal/EditedExternalResponseModel.java

@@ -0,0 +1,15 @@
+package fr.insee.genesis.domain.model.editedexternal;
+
+import lombok.Builder;
+import lombok.Data;
+
+import java.util.Map;
+
+@Builder
+@Data
+public class EditedExternalResponseModel {
+    String id;
+    String questionnaireId;
+    String interrogationId;
+    Map<String,Object> variables;
+}

src/main/java/fr/insee/genesis/domain/ports/api/EditedExternalResponseApiPort.java

@@ -0,0 +1,9 @@
+package fr.insee.genesis.domain.ports.api;
+
+import fr.insee.genesis.exceptions.GenesisException;
+
+import java.io.InputStream;
+
+public interface EditedExternalResponseApiPort {
+    void readEditedExternalFile(InputStream inputStream, String questionnaireId) throws GenesisException;
+    }

src/main/java/fr/insee/genesis/domain/ports/spi/EditedExternalResponsePersistancePort.java

@@ -0,0 +1,13 @@
+package fr.insee.genesis.domain.ports.spi;
+
+import fr.insee.genesis.domain.model.editedexternal.EditedExternalResponseModel;
+
+import java.util.List;
+
+public interface EditedExternalResponsePersistancePort {
+    void backup(String questionnaireId);
+    void deleteBackup(String questionnaireId);
+    void restoreBackup(String questionnaireId);
+    void saveAll(List<EditedExternalResponseModel> editedPreviousResponseModelList);
+    void delete(String questionnaireId);
+}

src/main/java/fr/insee/genesis/domain/service/editedexternal/EditedExternalResponseJsonService.java

@@ -0,0 +1,144 @@
+package fr.insee.genesis.domain.service.editedexternal;
+
+import com.fasterxml.jackson.core.JsonFactory;
+import com.fasterxml.jackson.core.JsonParseException;
+import com.fasterxml.jackson.core.JsonParser;
+import com.fasterxml.jackson.core.JsonToken;
+import fr.insee.genesis.domain.model.editedexternal.EditedExternalResponseModel;
+import fr.insee.genesis.domain.ports.api.EditedExternalResponseApiPort;
+import fr.insee.genesis.domain.ports.spi.EditedExternalResponsePersistancePort;
+import fr.insee.genesis.domain.utils.JsonUtils;
+import fr.insee.genesis.exceptions.GenesisException;
+import lombok.extern.slf4j.Slf4j;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.stereotype.Service;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+
+@Service
+@Slf4j
+public class EditedExternalResponseJsonService implements EditedExternalResponseApiPort {
+    private final EditedExternalResponsePersistancePort editedExternalResponsePersistancePort;
+
+    private static final int BLOCK_SIZE = 1000;
+
+    @Autowired
+    public EditedExternalResponseJsonService(EditedExternalResponsePersistancePort editedExternalResponsePersistancePort) {
+        this.editedExternalResponsePersistancePort = editedExternalResponsePersistancePort;
+    }
+
+    @Override
+    public void readEditedExternalFile(InputStream inputStream,
+                                       String questionnaireId) throws GenesisException {
+        JsonFactory jsonFactory = new JsonFactory();
+        moveCollectionToBackup(questionnaireId);
+        try(JsonParser jsonParser = jsonFactory.createParser(inputStream)){
+            List<EditedExternalResponseModel> toSave = new ArrayList<>();
+            goToEditedExternalToken(jsonParser);
+            long savedCount = 0;
+            Set<String> savedInterrogationIds = new HashSet<>();
+            jsonParser.nextToken(); //skip field name
+            jsonParser.nextToken(); //skip [
+            while (jsonParser.currentToken() != JsonToken.END_ARRAY) {
+                EditedExternalResponseModel editedExternalResponseModel = readNextEditedExternal(
+                        jsonParser,
+                        questionnaireId
+                );
+
+                checkModel(editedExternalResponseModel, jsonParser, savedInterrogationIds);
+
+                toSave.add(editedExternalResponseModel);
+                savedInterrogationIds.add(editedExternalResponseModel.getInterrogationId());
+
+                if(toSave.size() >= BLOCK_SIZE){
+                    savedCount = saveBlock(toSave, savedCount);
+                }
+                jsonParser.nextToken(); //skip }
+            }
+            editedExternalResponsePersistancePort.saveAll(toSave);
+            savedCount += toSave.size();
+            log.info("Reached end of edited external file, saved %d interrogations".formatted(savedCount));
+            editedExternalResponsePersistancePort.deleteBackup(questionnaireId);
+        }catch (JsonParseException jpe){
+            editedExternalResponsePersistancePort.restoreBackup(questionnaireId);
+            throw new GenesisException(400, "JSON Parsing exception : %s".formatted(jpe.toString()));
+        }catch (IOException ioe){
+            editedExternalResponsePersistancePort.restoreBackup(questionnaireId);
+            throw new GenesisException(500, ioe.toString());
+        }
+    }
+
+    private static void goToEditedExternalToken(JsonParser jsonParser) throws IOException, GenesisException {
+        boolean isTokenFound = false;
+        while (!isTokenFound){
+            jsonParser.nextToken();
+            if(jsonParser.currentToken() == null){
+                throw new GenesisException(400, "editedExternal object not found in JSON");
+            }
+            if(jsonParser.currentToken().equals(JsonToken.FIELD_NAME) && jsonParser.currentName() != null){
+                if (jsonParser.currentName().equals("editedExternal")) {
+                    isTokenFound = true;
+                }
+            }
+        }
+    }
+
+    private void moveCollectionToBackup(String questionnaireId) {
+        editedExternalResponsePersistancePort.backup(questionnaireId);
+        editedExternalResponsePersistancePort.delete(questionnaireId);
+    }
+
+    private long saveBlock(List<EditedExternalResponseModel> toSave, long savedCount) {
+        editedExternalResponsePersistancePort.saveAll(toSave);
+        savedCount += toSave.size();
+        toSave.clear();
+        return savedCount;
+    }
+
+    private static void checkModel(EditedExternalResponseModel editedExternalResponseModel, JsonParser jsonParser, Set<String> savedInterrogationIds) throws GenesisException {
+        if(editedExternalResponseModel.getInterrogationId() == null){
+            throw new GenesisException(400,
+                    "Missing interrogationId on the object that ends on line %d"
+                            .formatted(jsonParser.currentLocation().getLineNr())
+            );
+        }
+        if(savedInterrogationIds.contains(editedExternalResponseModel.getInterrogationId())){
+            throw new GenesisException(400,
+                    "Double interrogationId : %s".formatted(editedExternalResponseModel.getInterrogationId()));
+        }
+    }
+
+    private EditedExternalResponseModel readNextEditedExternal(JsonParser jsonParser,
+                                                               String questionnaireId
+                                                               ) throws IOException {
+        if(jsonParser.currentToken() != JsonToken.START_OBJECT){
+            throw new JsonParseException("Expected { on line %d, got token %s".formatted(jsonParser.currentLocation().getLineNr(), jsonParser.currentToken()));
+        }
+        EditedExternalResponseModel editedExternalResponseModel = EditedExternalResponseModel.builder()
+                .questionnaireId(questionnaireId)
+                .variables(new HashMap<>())
+                .build();
+        jsonParser.nextToken(); // read {
+        while (!jsonParser.currentToken().equals(JsonToken.END_OBJECT)){
+            if(jsonParser.currentToken().equals(JsonToken.FIELD_NAME) && jsonParser.currentName().equals("interrogationId")){
+                jsonParser.nextToken();
+                editedExternalResponseModel.setInterrogationId(jsonParser.getText());
+                jsonParser.nextToken();
+                continue;
+            }
+            jsonParser.nextToken();
+            editedExternalResponseModel.getVariables().put(
+                    jsonParser.currentName(),
+                    JsonUtils.readValue(jsonParser)
+            );
+            jsonParser.nextToken();
+        }
+        return editedExternalResponseModel;
+    }
+}

src/main/java/fr/insee/genesis/domain/service/editedprevious/EditedPreviousResponseJsonService.java

@@ -7,6 +7,7 @@
 import fr.insee.genesis.domain.model.editedprevious.EditedPreviousResponseModel;
 import fr.insee.genesis.domain.ports.api.EditedPreviousResponseApiPort;
 import fr.insee.genesis.domain.ports.spi.EditedPreviousResponsePersistancePort;
+import fr.insee.genesis.domain.utils.JsonUtils;
 import fr.insee.genesis.exceptions.GenesisException;
 import lombok.extern.slf4j.Slf4j;
 import org.springframework.beans.factory.annotation.Autowired;
@@ -42,8 +43,7 @@ public void readEditedPreviousFile(InputStream inputStream,
         JsonFactory jsonFactory = new JsonFactory();
         try(JsonParser jsonParser = jsonFactory.createParser(inputStream)){
             List<EditedPreviousResponseModel> toSave = new ArrayList<>();
-            boolean isTokenFound = false;
-            goToEditedPreviousToken(isTokenFound, jsonParser);
+            goToEditedPreviousToken(jsonParser);
             long savedCount = 0;
             Set<String> savedInterrogationIds = new HashSet<>();
             jsonParser.nextToken(); //skip field name
@@ -95,9 +95,13 @@ private static void checkSourceStateLength(String sourceState) throws GenesisExc
         }
     }
 
-    private static void goToEditedPreviousToken(boolean isTokenFound, JsonParser jsonParser) throws IOException {
+    private static void goToEditedPreviousToken(JsonParser jsonParser) throws IOException, GenesisException {
+        boolean isTokenFound = false;
         while (!isTokenFound){
             jsonParser.nextToken();
+            if(jsonParser.currentToken() == null){
+                throw new GenesisException(400, "editedPrevious object not found in JSON");
+            }
             if(jsonParser.currentToken().equals(JsonToken.FIELD_NAME) && jsonParser.currentName() != null){
                 if (jsonParser.currentName().equals("editedPrevious")) {
                     isTokenFound = true;
@@ -129,7 +133,7 @@ private EditedPreviousResponseModel readNextEditedPrevious(JsonParser jsonParser
             jsonParser.nextToken();
             editedPreviousResponseModel.getVariables().put(
                     jsonParser.currentName(),
-                    readValue(jsonParser)
+                    JsonUtils.readValue(jsonParser)
             );
             jsonParser.nextToken();
         }