V2: External survey tool integration: ngSurvey (#13946)

New Features

    Support for customizable fields in data patches and survey responses; UI shows grouped field names and current values and lets users correct grouped patch entries.
    Improved partial retrieval of customizable field values and extended date/time parsing.

Bug Fixes

    Clearer failure classifications (including invalid custom context and forbidden multi-group updates).
    More robust survey response reprocessing and handling of unknown tokens.

Chores

    Database migration adjusting JSON column handling for external messages.

Author: Pa-Touche

sormas-api/src/main/java/de/symeda/sormas/api/externalmessage/ExternalMessageFacade.java

@@ -2,7 +2,6 @@
 
 import java.util.Date;
 import java.util.List;
-import java.util.Map;
 
 import javax.annotation.Nullable;
 import javax.ejb.Remote;
@@ -13,6 +12,7 @@
 import de.symeda.sormas.api.ReferenceDto;
 import de.symeda.sormas.api.caze.surveillancereport.SurveillanceReportReferenceDto;
 import de.symeda.sormas.api.common.Page;
+import de.symeda.sormas.api.externalmessage.survey.PatchDictionary;
 import de.symeda.sormas.api.patch.partial_retrieval.DisplayablePartialRetrievalResponse;
 import de.symeda.sormas.api.sample.SampleReferenceDto;
 import de.symeda.sormas.api.user.UserReferenceDto;
@@ -96,7 +96,7 @@ default List<ExternalMessageDto> saveAndProcessSurveyResponses() {
 	 *            the corrected field path -> value map to apply
 	 * @return updated ExternalMessageDto after reprocessing
 	 */
-	ExternalMessageDto overwriteSurveyResponse(String uuid, Map<String, Object> correctedDictionary);
+	ExternalMessageDto overwriteSurveyResponse(String uuid, PatchDictionary correctedDictionary);
 
 	/**
 	 * Retrieves display-ready field information (translated names and current case values) for all fields

sormas-api/src/main/java/de/symeda/sormas/api/externalmessage/survey/ExternalMessageSurveyResponseRequest.java

@@ -3,7 +3,6 @@
 import java.io.Serializable;
 import java.util.Date;
 import java.util.List;
-import java.util.Map;
 import java.util.Objects;
 
 import javax.annotation.Nullable;
@@ -43,14 +42,14 @@ public class ExternalMessageSurveyResponseRequest implements Serializable, Compa
 	 * The accepted fields are those from {@link InfoFacade#generateDataDictionary()}.
 	 */
 	@NotNull
-	private Map<String, Object> patchDictionary;
+	private PatchDictionary patchDictionary = new PatchDictionary();
 
 	/**
 	 * Contains fields that were excluded from the patch dictionary, meant for fields that may not start with the prefix.
 	 * The prefix allows to safely exclude fields that are not meant to be mapped into SORMAS.
 	 */
 	@NotNull
-	private Map<String, Object> excludedPatchDictionary;
+	private PatchDictionary excludedPatchDictionary = new PatchDictionary();
 
 	/**
 	 * Origin that wants the patch operation.
@@ -140,11 +139,11 @@ public ExternalMessageSurveyResponseRequest setEmptyValueBehavior(EmptyValueBeha
 		return this;
 	}
 
-	public Map<String, Object> getPatchDictionary() {
+	public PatchDictionary getPatchDictionary() {
 		return patchDictionary;
 	}
 
-	public ExternalMessageSurveyResponseRequest setPatchDictionary(Map<String, Object> patchDictionary) {
+	public ExternalMessageSurveyResponseRequest setPatchDictionary(PatchDictionary patchDictionary) {
 		this.patchDictionary = patchDictionary;
 		return this;
 	}
@@ -187,11 +186,11 @@ public ExternalMessageSurveyResponseRequest setSkipIfAlreadyProcessed(boolean sk
 		return this;
 	}
 
-	public Map<String, Object> getExcludedPatchDictionary() {
+	public PatchDictionary getExcludedPatchDictionary() {
 		return excludedPatchDictionary;
 	}
 
-	public ExternalMessageSurveyResponseRequest setExcludedPatchDictionary(Map<String, Object> excludedPatchDictionary) {
+	public ExternalMessageSurveyResponseRequest setExcludedPatchDictionary(PatchDictionary excludedPatchDictionary) {
 		this.excludedPatchDictionary = excludedPatchDictionary;
 		return this;
 	}

sormas-api/src/main/java/de/symeda/sormas/api/externalmessage/survey/PatchDictionary.java

@@ -0,0 +1,75 @@
+package de.symeda.sormas.api.externalmessage.survey;
+
+import java.io.Serializable;
+import java.util.LinkedHashMap;
+import java.util.Map;
+import java.util.Objects;
+
+import javax.validation.constraints.NotNull;
+
+import com.fasterxml.jackson.annotation.JsonIgnore;
+import de.symeda.sormas.api.audit.AuditedClass;
+
+/**
+ * Wrapper around concrete storage for patch dictionary.
+ */
+@AuditedClass
+public class PatchDictionary implements Serializable {
+
+	@NotNull
+	private LinkedHashMap<PatchField, Object> dictionary = new LinkedHashMap<>();
+
+	/**
+	 * Helper method when you don't have any grouped fields.
+	 * 
+	 * @param key
+	 *            {@link PatchField#getField()}
+	 * @param value
+	 *            value within dictionary
+	 */
+	public void put(String key, Object value) {
+		dictionary.put(new PatchField().setField(key), value);
+	}
+
+	public void put(PatchField key, Object value) {
+		dictionary.put(key, value);
+	}
+
+	public LinkedHashMap<PatchField, Object> getDictionary() {
+		return dictionary;
+	}
+
+	public PatchDictionary setDictionary(LinkedHashMap<PatchField, Object> dictionary) {
+		this.dictionary = dictionary;
+		return this;
+	}
+
+	@JsonIgnore
+	public PatchDictionary setNonTypedPatchDictionary(Map<PatchField, Object> patchDictionary) {
+		this.dictionary = new LinkedHashMap<>(patchDictionary);
+		return this;
+	}
+
+	@JsonIgnore
+	public boolean isEmpty() {
+		return dictionary.isEmpty();
+	}
+
+	@Override
+	public boolean equals(Object o) {
+		if (o == null || getClass() != o.getClass())
+			return false;
+		PatchDictionary that = (PatchDictionary) o;
+		return Objects.equals(dictionary, that.dictionary);
+	}
+
+	@Override
+	public int hashCode() {
+		return Objects.hashCode(dictionary);
+	}
+
+	@Override
+	public String toString() {
+		return "PatchDictionary{" + "dictionary=" + dictionary + '}';
+	}
+}

sormas-api/src/main/java/de/symeda/sormas/api/externalmessage/survey/PatchField.java

@@ -0,0 +1,93 @@
+package de.symeda.sormas.api.externalmessage.survey;
+
+import java.io.Serializable;
+import java.util.Objects;
+import java.util.Optional;
+
+import javax.annotation.Nullable;
+import javax.validation.constraints.NotNull;
+
+import com.fasterxml.jackson.annotation.JsonCreator;
+import com.fasterxml.jackson.annotation.JsonIgnore;
+import com.fasterxml.jackson.annotation.JsonValue;
+
+/**
+ * To be able to repeat some groups and make them belong together, the groupIndex was added.
+ */
+public class PatchField implements Serializable {
+
+	private static final long serialVersionUID = 1L;
+
+	@NotNull
+	private String field;
+
+	/**
+	 * 0-based index of belonging within the repeated occurrence.
+	 */
+	@Nullable
+	private Integer groupIndex;
+
+	public PatchField() {
+	}
+
+	@JsonCreator
+	public PatchField(String field, @Nullable Integer groupIndex) {
+		this.field = field;
+		this.groupIndex = groupIndex;
+	}
+
+	public static PatchField of(String field, Integer groupIndex) {
+		return new PatchField().setField(field).setGroupIndex(groupIndex);
+	}
+
+	public static PatchField of(String field) {
+		return new PatchField().setField(field);
+	}
+
+	public String getField() {
+		return field;
+	}
+
+	public PatchField setField(String field) {
+		this.field = field;
+		return this;
+	}
+
+	@Nullable
+	public Integer getGroupIndex() {
+		return groupIndex;
+	}
+
+	public PatchField setGroupIndex(@Nullable Integer groupIndex) {
+		this.groupIndex = groupIndex;
+		return this;
+	}
+
+	@JsonIgnore
+	public Optional<Integer> getGroupNumber() {
+		return Optional.ofNullable(groupIndex).map(index -> index + 1);
+	}
+
+	@JsonValue
+	public String toJsonValue() {
+		return groupIndex == null ? field : field + "@" + groupIndex;
+	}
+
+	@Override
+	public boolean equals(Object o) {
+		if (o == null || getClass() != o.getClass())
+			return false;
+		PatchField that = (PatchField) o;
+		return Objects.equals(field, that.field) && Objects.equals(groupIndex, that.groupIndex);
+	}
+
+	@Override
+	public int hashCode() {
+		return Objects.hash(field, groupIndex);
+	}
+
+	@Override
+	public String toString() {
+		return "PatchField{" + "field='" + field + '\'' + ", groupIndex=" + groupIndex + '}';
+	}
+}

sormas-api/src/main/java/de/symeda/sormas/api/externalmessage/survey/PatchFieldKeyDeserializer.java

@@ -0,0 +1,35 @@
+package de.symeda.sormas.api.externalmessage.survey;
+
+import java.io.IOException;
+
+import com.fasterxml.jackson.databind.DeserializationContext;
+import com.fasterxml.jackson.databind.KeyDeserializer;
+
+public class PatchFieldKeyDeserializer extends KeyDeserializer {
+
+	@Override
+	public Object deserializeKey(String key, DeserializationContext ctxt) throws IOException {
+		if (key == null || key.isBlank()) {
+			return null;
+		}
+
+		int atIndex = key.lastIndexOf('@');
+		if (atIndex < 0) {
+			return PatchField.of(key);
+		}
+
+		String field = key.substring(0, atIndex);
+		String indexPart = key.substring(atIndex + 1);
+
+		if (field.isBlank()) {
+			throw ctxt.weirdKeyException(PatchField.class, key, "Field part is empty");
+		}
+
+		try {
+			Integer groupIndex = Integer.valueOf(indexPart);
+			return PatchField.of(field, groupIndex);
+		} catch (NumberFormatException e) {
+			throw ctxt.weirdKeyException(PatchField.class, key, String.format("groupIndex must be an integer, was. [%s]", indexPart));
+		}
+	}
+}

sormas-api/src/main/java/de/symeda/sormas/api/patch/CaseDataPatchRequest.java

@@ -8,6 +8,7 @@
 import javax.validation.constraints.NotNull;
 
 import de.symeda.sormas.api.Language;
+import de.symeda.sormas.api.externalmessage.survey.PatchDictionary;
 import de.symeda.sormas.api.info.InfoFacade;
 
 /**
@@ -31,7 +32,7 @@ public class CaseDataPatchRequest {
 	 * The accepted fields are those from {@link InfoFacade#generateDataDictionary()}.
 	 */
 	@NotNull
-	private Map<String, Object> patchDictionary;
+	private PatchDictionary patchDictionary;
 
 	/**
 	 * Origin that wants the patch operation.
@@ -70,15 +71,29 @@ public CaseDataPatchRequest setReplacementStrategy(DataReplacementStrategy repla
 		return this;
 	}
 
-	public Map<String, Object> getPatchDictionary() {
+	public PatchDictionary getPatchDictionary() {
 		return patchDictionary;
 	}
 
-	public CaseDataPatchRequest setPatchDictionary(Map<String, Object> patchDictionary) {
+	public CaseDataPatchRequest setPatchDictionary(PatchDictionary patchDictionary) {
 		this.patchDictionary = patchDictionary;
 		return this;
 	}
 
+	/**
+	 * Meant for convenience purposes when no grouped fields are present.
+	 * 
+	 * @param patchDictionary
+	 *            that must be patched WITHOUT groups
+	 * @return request
+	 */
+	public CaseDataPatchRequest setPatchDictionary(Map<String, Object> patchDictionary) {
+		PatchDictionary patchDictionaryWrapper = new PatchDictionary();
+		patchDictionary.forEach(patchDictionaryWrapper::put);
+		this.patchDictionary = patchDictionaryWrapper;
+		return this;
+	}
+
 	public EmptyValueBehavior getEmptyValueBehavior() {
 		return emptyValueBehavior;
 	}

sormas-api/src/main/java/de/symeda/sormas/api/patch/DataPatchFailureCause.java

@@ -29,6 +29,11 @@ public enum DataPatchFailureCause {
 	 */
 	NOT_PRESENT_IN_REFERENCE_DATA_LIST,
 
+	/**
+	 * Means this reference data requires some configuration / implementation to work.
+	 */
+	UNSUPPORTED_REFERENCE_DATA,
+
 	/**
 	 * Occurs the field is not supported by the disease / country / feature.
 	 * Error message must be somewhat generic to specify the Data Dictionary should be checked.
@@ -75,6 +80,17 @@ public enum DataPatchFailureCause {
 	 */
 	DUPLICATE_FIELD,
 
+	/**
+	 * Means the field does not support being inserted multiple in groups. Most fields are "singulars".
+	 */
+	FORBIDDEN_MULTI_GROUP_FIELD,
+
+	/**
+	 * Tried to insert in a {@link de.symeda.sormas.api.customizablefield.CustomizableFieldContext} that does not exist.
+	 * You can have a look at the class: CustomizableFieldContextPatchMapping.
+	 */
+	INVALID_CUSTOM_CONTEXT,
+
 	/**
 	 * This means there is a hole in the implementation.
 	 */