Skip to content

Commit 0ddee0e

Browse files
Pa-TouchePa-Touche
committed
preparing systemconfig usage
1 parent 5520493 commit 0ddee0e

12 files changed

Lines changed: 221 additions & 93 deletions

File tree

‎sormas-api/src/main/java/de/symeda/sormas/api/feature/FeatureType.java‎

Lines changed: 9 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -14,18 +14,9 @@
1414
*/
1515
package de.symeda.sormas.api.feature;
1616

17-
import static de.symeda.sormas.api.common.DeletableEntityType.CASE;
18-
import static de.symeda.sormas.api.common.DeletableEntityType.CONTACT;
19-
import static de.symeda.sormas.api.common.DeletableEntityType.EVENT;
20-
import static de.symeda.sormas.api.common.DeletableEntityType.EVENT_PARTICIPANT;
21-
import static de.symeda.sormas.api.common.DeletableEntityType.IMMUNIZATION;
22-
import static de.symeda.sormas.api.common.DeletableEntityType.TRAVEL_ENTRY;
17+
import static de.symeda.sormas.api.common.DeletableEntityType.*;
2318

24-
import java.util.ArrayList;
25-
import java.util.Arrays;
26-
import java.util.List;
27-
import java.util.Map;
28-
import java.util.Set;
19+
import java.util.*;
2920

3021
import com.google.common.collect.ImmutableMap;
3122

@@ -128,7 +119,13 @@ public enum FeatureType {
128119
new FeatureType[] {
129120
SAMPLES_LAB },
130121
null,
131-
ImmutableMap.of(FeatureTypeProperty.FETCH_MODE, Boolean.FALSE, FeatureTypeProperty.FORCE_AUTOMATIC_PROCESSING, false)),
122+
ImmutableMap.of(
123+
FeatureTypeProperty.FETCH_MODE,
124+
Boolean.FALSE,
125+
FeatureTypeProperty.FORCE_AUTOMATIC_PROCESSING,
126+
false,
127+
FeatureTypeProperty.SURVEY_FETCH_ENABLED,
128+
false)),
132129
MANUAL_EXTERNAL_MESSAGES(true,
133130
true,
134131
new FeatureType[] {

‎sormas-api/src/main/java/de/symeda/sormas/api/feature/FeatureTypeProperty.java‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,7 @@ public enum FeatureTypeProperty {
3030
SHARE_IMMUNIZATIONS(Boolean.class),
3131
SHARE_REPORTS(Boolean.class),
3232
FETCH_MODE(Boolean.class),
33+
SURVEY_FETCH_ENABLED(Boolean.class),
3334
FORCE_AUTOMATIC_PROCESSING(Boolean.class);
3435

3536
private final Class<?> returnType;

‎sormas-api/src/main/resources/captions.properties‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1737,6 +1737,7 @@ ExternalMessage.personNationalHealthId=National health ID
17371737
ExternalMessage.personPhoneNumberType=Phone number type
17381738
ExternalMessage.personCountry=Country
17391739
externalMessageFetch=Fetch messages
1740+
surveyFetch=Fetch Surveys
17401741
externalMessageProcess=Process
17411742
externalMessageNoDisease=No disease found
17421743
externalMessageNoNewMessages=No new messages available

‎sormas-api/src/main/resources/enum.properties‎

Lines changed: 16 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3062,4 +3062,19 @@ AnimalCategory.WILD=Wild
30623062

30633063
# FomiteTransmissionLocation
30643064
FomiteTransmissionLocation.INSIDE_HOME=Inside home
3065-
FomiteTransmissionLocation.OUTSIDE=Outside
3065+
FomiteTransmissionLocation.OUTSIDE=Outside
3066+
3067+
# DataPatchFailureCause
3068+
DataPatchFailureCause.INVALID_MULTIPLE_FIELDS_FORMAT=Invalid multiple fields format
3069+
DataPatchFailureCause.MISSING_MANDATORY_FIELD_FOR_GROUP=Missing required field for group
3070+
DataPatchFailureCause.FORBIDDEN_NON_UNIQUE_ALIAS=Alias must map to a single unique field
3071+
DataPatchFailureCause.NOT_PRESENT_IN_REFERENCE_DATA_LIST=Value not in allowed reference list
3072+
DataPatchFailureCause.UNSUPPORTED_FIELD_FOR_DISEASE_OR_COUNTRY_OR_FEATURE=Field not supported for this disease, country, or feature
3073+
DataPatchFailureCause.UNSUPPORTED_PREFIX=Unsupported path prefix
3074+
DataPatchFailureCause.FIELD_DOES_NOT_EXIST=Field does not exist
3075+
DataPatchFailureCause.FORBIDDEN_FIELD=Field cannot be modified
3076+
DataPatchFailureCause.FORBIDDEN_VALUE_OVERRIDE=Overriding existing value is not allowed
3077+
DataPatchFailureCause.INVALID_VALUE_TYPE=Invalid value type
3078+
DataPatchFailureCause.UNSUPPORTED_TARGET_TYPE=Unsupported target type
3079+
DataPatchFailureCause.INVALID_PATH_FORMAT=Invalid path format
3080+
DataPatchFailureCause.TECHNICAL=Technical error

‎sormas-api/src/main/resources/strings.properties‎

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2069,4 +2069,11 @@ Vaccine.vaccinationType.PCV15=PCV15
20692069
Vaccine.vaccinationType.PCV13=PCV13
20702070
Vaccine.vaccinationType.PCV20=PCV20
20712071
exposureStartDate = Exposure start date
2072-
exposureEndDate = Exposure end date
2072+
exposureEndDate = Exposure end date
2073+
2074+
headingSurveyResponseDetails=Response Details
2075+
headingSurveyResponseFailures=Validation Issues
2076+
headingSurveyResponseCorrectAndReprocess=Correct & Reprocess
2077+
messageSurveyResponseAllFieldsApplied=All fields were successfully applied
2078+
messageSurveyResponseNotYetProcessed=This response has not been processed yet
2079+
messageSurveyResponseReprocessed=The response has been successfully reprocessed

‎sormas-backend/src/main/java/de/symeda/sormas/backend/common/CronService.java‎

Lines changed: 8 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -225,7 +225,7 @@ public void deleteSystemEvents() {
225225
}
226226
}
227227

228-
@Schedule(hour = "*", minute = "*/30", persistent = false)
228+
@Schedule(hour = "*", minute = "0", second = "0", persistent = false)
229229
public void fetchExternalMessages() {
230230
if (featureConfigurationFacade.isFeatureEnabled(FeatureType.EXTERNAL_MESSAGES)) {
231231
externalMessageFacade.fetchAndSaveExternalMessages(null);
@@ -234,18 +234,19 @@ public void fetchExternalMessages() {
234234

235235
@Schedule(hour = "*", minute = "*/5", persistent = false)
236236
public void fetchSurveyResponses() {
237-
if (!featureConfigurationFacade.isFeatureEnabled(FeatureType.EXTERNAL_MESSAGES)) {
237+
if (!featureConfigurationFacade.isFeatureEnabled(FeatureType.EXTERNAL_MESSAGES)
238+
|| featureConfigurationFacade.isPropertyValueTrue(FeatureType.EXTERNAL_MESSAGES, FeatureTypeProperty.SURVEY_FETCH_ENABLED)) {
238239
logger.info("External messages are disabled, survey responses will not be fetched");
239240
return;
240241
}
241242

242-
logger.error("fetchSurveyResponses is triggered");
243-
244-
List<ExternalMessageDto> externalMessageDtos = externalMessageFacade.saveAndProcessSurveyResponses();
243+
List<ExternalMessageDto> surveyExternalMessages = externalMessageFacade.saveAndProcessSurveyResponses();
245244

246245
if (logger.isInfoEnabled()) {
247-
List<String> reportIds = externalMessageDtos.stream().map(ExternalMessageDto::getReportId).collect(Collectors.toList());
248-
logger.info("Following response ids were saved: [{}]", reportIds);
246+
List<String> reportIds = surveyExternalMessages.stream().map(ExternalMessageDto::getReportId).collect(Collectors.toList());
247+
if (!reportIds.isEmpty()) {
248+
logger.info("Survey responses with following reportIds were saved: [{}]", reportIds);
249+
}
249250
}
250251
}
251252

‎sormas-backend/src/main/java/de/symeda/sormas/backend/externalmessage/ExternalMessageFacadeEjb.java‎

Lines changed: 32 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -114,7 +114,7 @@
114114
UserRight._EXTERNAL_MESSAGE_DOCTOR_DECLARATION_VIEW })
115115
public class ExternalMessageFacadeEjb implements ExternalMessageFacade {
116116

117-
public static final String SURVEY_PERIOD_INTERVAL_HOURS = "SURVEY_PERIOD_INTERVAL_HOURS";
117+
public static final String SURVEY_PERIOD_INTERVAL_DAYS = "SURVEY_PERIOD_INTERVAL_DAYS";
118118
private static final String SURVEY_AS_EXTERNAL_MESSAGE_ADAPTER_JNDI_KEY = "SURVEY_AS_EXTERNAL_MESSAGE_ADAPTER_JNDI_KEY";
119119

120120
private final Logger logger = LoggerFactory.getLogger(getClass());
@@ -317,17 +317,45 @@ public ExternalMessageDto saveAndProcessLabmessage(@Valid ExternalMessageDto lab
317317
public List<ExternalMessageDto> saveAndProcessSurveyResponses(Date since) {
318318

319319
if (since == null) { // TODO: use shorter default range
320-
int hoursRange =
321-
Integer.parseInt(Optional.ofNullable(systemConfigurationValueFacade.getValue(SURVEY_PERIOD_INTERVAL_HOURS)).orElse("50000"));
320+
int dateRange = Integer.parseInt(Optional.ofNullable(systemConfigurationValueFacade.getValue(SURVEY_PERIOD_INTERVAL_DAYS)).orElse("350"));
322321

323-
since = DateHelper.addSeconds(new Date(), -(hoursRange * 3600));
322+
since = DateHelper.addDays(new Date(), -dateRange);
324323
}
325324

326325
logger.error("Since date: [{}]", since);
327326

328327
ExternalMessageAdapterFacade externalLabResultsFacade = getExternalSurveyProviderFacade();
329328
ExternalMessageResult<List<ExternalMessageDto>> externalMessagesResult = externalLabResultsFacade.getExternalMessages(since);
330329
List<ExternalMessageDto> surveyResponses = externalMessagesResult.getValue();
330+
331+
List<String> reportIds = surveyResponses.stream().map(ExternalMessageDto::getReportId).filter(Objects::nonNull).collect(toList());
332+
333+
if (!reportIds.isEmpty()) {
334+
Map<String, de.symeda.sormas.api.utils.Tuple<ExternalMessageStatus, String>> statusUuidTuplesByReportId =
335+
externalMessageService.getUuidsByReportIds(reportIds);
336+
337+
Set<String> unProcessedMessagesReportIds = statusUuidTuplesByReportId.entrySet()
338+
.stream()
339+
.filter(tuple -> tuple.getValue().getFirst() == ExternalMessageStatus.UNPROCESSED)
340+
.map(Map.Entry::getKey)
341+
.collect(Collectors.toSet());
342+
343+
Set<String> alreadyPresentReportIds = statusUuidTuplesByReportId.keySet();
344+
345+
surveyResponses.forEach(dto -> {
346+
de.symeda.sormas.api.utils.Tuple<ExternalMessageStatus, String> tuple = statusUuidTuplesByReportId.get(dto.getReportId());
347+
if (tuple != null && tuple.getFirst() == ExternalMessageStatus.UNPROCESSED) {
348+
dto.setUuid(tuple.getSecond());
349+
}
350+
});
351+
352+
surveyResponses = surveyResponses.stream()
353+
.filter(
354+
externalMessage -> !alreadyPresentReportIds.contains(externalMessage.getReportId())
355+
|| unProcessedMessagesReportIds.contains(externalMessage.getReportId()))
356+
.collect(toList());
357+
}
358+
331359
List<ExternalMessageDto> savedDtos;
332360
try {
333361
List<SurveyResponseProcessingResult> processingResults = automaticSurveyResponseProcessor.processSurveyResponses(surveyResponses);

‎sormas-backend/src/main/java/de/symeda/sormas/backend/externalmessage/ExternalMessageService.java‎

Lines changed: 34 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -1,29 +1,18 @@
11
package de.symeda.sormas.backend.externalmessage;
22

3-
import java.util.ArrayList;
4-
import java.util.Calendar;
5-
import java.util.Collections;
6-
import java.util.List;
7-
8-
import javax.ejb.EJB;
9-
import javax.ejb.LocalBean;
10-
import javax.ejb.Stateless;
11-
import javax.ejb.TransactionAttribute;
12-
import javax.ejb.TransactionAttributeType;
13-
import javax.persistence.criteria.CriteriaBuilder;
14-
import javax.persistence.criteria.CriteriaQuery;
15-
import javax.persistence.criteria.From;
16-
import javax.persistence.criteria.Join;
17-
import javax.persistence.criteria.JoinType;
18-
import javax.persistence.criteria.Path;
19-
import javax.persistence.criteria.Predicate;
20-
import javax.persistence.criteria.Root;
3+
import java.util.*;
4+
import java.util.stream.Collectors;
5+
6+
import javax.ejb.*;
7+
import javax.persistence.Tuple;
8+
import javax.persistence.criteria.*;
219

2210
import org.apache.commons.lang3.StringUtils;
2311

2412
import de.symeda.sormas.api.ReferenceDto;
2513
import de.symeda.sormas.api.caze.surveillancereport.SurveillanceReportReferenceDto;
2614
import de.symeda.sormas.api.externalmessage.ExternalMessageCriteria;
15+
import de.symeda.sormas.api.externalmessage.ExternalMessageStatus;
2716
import de.symeda.sormas.api.externalmessage.ExternalMessageType;
2817
import de.symeda.sormas.api.sample.SampleReferenceDto;
2918
import de.symeda.sormas.api.user.UserRight;
@@ -323,4 +312,31 @@ public ExternalMessage getForSurveillanceReport(SurveillanceReportReferenceDto s
323312

324313
return em.createQuery(cq).getResultStream().findFirst().orElse(null);
325314
}
315+
316+
/**
317+
* Allows to only update (refresh with latest survey data) for messages that are not yet processed:
318+
* {@link ExternalMessageStatus#UNPROCESSED}.
319+
*
320+
* @param reportIds
321+
* dedup / idempotency key composed of the survey id and response id.
322+
* @return UUIds that must be updated / refreshed.
323+
*/
324+
public Map<String, de.symeda.sormas.api.utils.Tuple<ExternalMessageStatus, String>> getUuidsByReportIds(Collection<String> reportIds) {
325+
if (reportIds == null || reportIds.isEmpty()) {
326+
return Collections.emptyMap();
327+
}
328+
CriteriaBuilder cb = em.getCriteriaBuilder();
329+
CriteriaQuery<Tuple> cq = cb.createTupleQuery();
330+
Root<ExternalMessage> root = cq.from(ExternalMessage.class);
331+
cq.multiselect(root.get(AbstractDomainObject.UUID), root.get(ExternalMessage.REPORT_ID), root.get(ExternalMessage.STATUS));
332+
cq.where(root.get(ExternalMessage.REPORT_ID).in(reportIds));
333+
return em.createQuery(cq)
334+
.getResultList()
335+
.stream()
336+
.collect(
337+
Collectors.toMap(
338+
t -> t.get(1, String.class),
339+
t -> new de.symeda.sormas.api.utils.Tuple<>(t.get(2, ExternalMessageStatus.class), t.get(0, String.class)),
340+
(a, b) -> a));
341+
}
326342
}

‎sormas-backend/src/main/java/de/symeda/sormas/backend/externalmessage/survey/AutomaticSurveyResponseProcessor.java‎

Lines changed: 0 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -28,7 +28,6 @@
2828
import de.symeda.sormas.api.survey.SurveyTokenDto;
2929
import de.symeda.sormas.api.utils.Tuple;
3030
import de.symeda.sormas.api.utils.dataprocessing.ProcessingResultStatus;
31-
import de.symeda.sormas.backend.externalmessage.labmessage.ExternalMessageProcessingFacadeEjbLocal;
3231
import de.symeda.sormas.backend.survey.SurveyFacadeEjb;
3332
import de.symeda.sormas.backend.survey.SurveyTokenFacadeEjb;
3433
import de.symeda.sormas.backend.util.CollectorUtils;
@@ -47,9 +46,6 @@ public class AutomaticSurveyResponseProcessor {
4746
@EJB
4847
private SurveyTokenFacadeEjb.SurveyTokenFacadeEjbLocal surveyTokenFacade;
4948

50-
@EJB
51-
private ExternalMessageProcessingFacadeEjbLocal processingFacade;
52-
5349
@Transactional(Transactional.TxType.REQUIRES_NEW)
5450
public List<SurveyResponseProcessingResult> processSurveyResponses(List<ExternalMessageDto> externalMessages)
5551
throws InterruptedException, ExecutionException {

‎sormas-backend/src/main/resources/sql/sormas_schema_next.sql‎

Lines changed: 62 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -28,5 +28,67 @@ ALTER TABLE externalmessage
2828
ALTER TABLE externalmessage
2929
ADD COLUMN IF NOT EXISTS additionalDataJson JSONB;
3030

31+
32+
-- system configuration for surveys
33+
34+
DO
35+
$$ DECLARE
36+
general_category_id bigint;
37+
38+
BEGIN
39+
-- Get GENERAL category id
40+
-- General category should always exist
41+
SELECT id
42+
INTO general_category_id
43+
FROM systemconfigurationcategory
44+
WHERE name = 'GENERAL_CATEGORY';
45+
46+
INSERT INTO systemconfigurationvalue(config_key, config_value, value_description, category_id, value_optional, value_pattern,
47+
value_encrypt, data_provider, validation_message, changedate, creationdate, id,
48+
uuid)
49+
VALUES ('NG_SUVEY_BASE_URI', null, 'i18n/infoSystemConfigurationValueDescriptionNgSurveyBaseURI', general_category_id, true,
50+
'', false, null,
51+
'i18n/systemConfigurationValueInvalidValue', now(), now(), nextval('entity_seq'), generate_base32_uuid());
52+
53+
54+
INSERT INTO systemconfigurationvalue(config_key, config_value, value_description, category_id, value_optional, value_pattern,
55+
value_encrypt, data_provider, validation_message, changedate, creationdate, id,
56+
uuid)
57+
VALUES ('NG_SUVEY_CRYPTED_TOKEN', null, 'i18n/infoSystemConfigurationValueDescriptionNgSurveyCryptedToken', general_category_id, true,
58+
'', true, null,
59+
'i18n/systemConfigurationValueInvalidValue', now(), now(), nextval('entity_seq'), generate_base32_uuid());
60+
61+
INSERT INTO systemconfigurationvalue(config_key, config_value, value_description, category_id, value_optional, value_pattern,
62+
value_encrypt, data_provider, validation_message, changedate, creationdate, id,
63+
uuid)
64+
VALUES ('NG_SUVEY_FIELD_PREFIX', '_so', 'i18n/infoSystemConfigurationValueDescriptionNgSurveyFieldPrefix', general_category_id, true,
65+
'', true, null,
66+
'i18n/systemConfigurationValueInvalidValue', now(), now(), nextval('entity_seq'), generate_base32_uuid());
67+
68+
INSERT INTO systemconfigurationvalue(config_key, config_value, value_description, category_id, value_optional, value_pattern,
69+
value_encrypt, data_provider, validation_message, changedate, creationdate, id,
70+
uuid)
71+
VALUES ('SURVEY_AS_EXTERNAL_MESSAGE_ADAPTER_JNDI_KEY', 'java:global/sormas-esante-adapter/SurveyExternalMessageAdapterFacadeEjb', 'i18n/infoSystemConfigurationValueDescriptionSurveyJDNI', general_category_id, true,
72+
'', true, null,
73+
'i18n/systemConfigurationValueInvalidValue', now(), now(), nextval('entity_seq'), generate_base32_uuid());
74+
75+
76+
77+
INSERT INTO systemconfigurationvalue(config_key, config_value, value_description, category_id, value_optional, value_pattern,
78+
value_encrypt, data_provider, validation_message, changedate, creationdate, id,
79+
uuid)
80+
VALUES ('SURVEY_PERIOD_INTERVAL_DAYS', '4', 'i18n/infoSystemConfigurationValueDescriptionSurveyPeriodIntervalDays', general_category_id, true,
81+
'', true, null,
82+
'i18n/systemConfigurationValueInvalidValue', now(), now(), nextval('entity_seq'), generate_base32_uuid());
83+
84+
85+
86+
END $$
87+
LANGUAGE plpgsql;
88+
89+
-- index to avoid full table scan when checking for survey duplicates
90+
CREATE INDEX idx_externalmessage_report_id
91+
ON externalmessage (reportid);
92+
3193
INSERT INTO schema_version (version_number, comment)
3294
VALUES (609, '#13832 - External Survey facade');

0 commit comments

Comments
 (0)