From 613e78d5558c19d30d46c2b96265fd064f6a0c08 Mon Sep 17 00:00:00 2001 From: Michael Peels Date: Thu, 16 Apr 2026 13:21:41 -0400 Subject: [PATCH 01/16] WIP - passing tests --- .../dto/observation/ObservationCodedKey.java | 9 +- .../model/dto/observation/ObservationKey.java | 14 +- .../observation/ObservationMaterialKey.java | 13 +- .../dto/observation/ParsedObservation.java | 30 + .../service/ObservationService.java | 154 +----- .../observation/NrtObservationWriter.java | 139 +++++ .../observation/ObservationProcessor.java | 145 +++++ .../transformer/ObservationParser.java | 517 ++++++++++++++++++ .../ProcessObservationDataUtil.java | 26 +- .../functional/DataDrivenFunctionalTests.java | 8 +- .../service/ObservationServiceTest.java | 34 +- .../src/test/resources/application-test.yaml | 4 + 12 files changed, 904 insertions(+), 189 deletions(-) create mode 100644 reporting-pipeline-service/src/main/java/gov/cdc/nbs/report/pipeline/observation/model/dto/observation/ParsedObservation.java create mode 100644 reporting-pipeline-service/src/main/java/gov/cdc/nbs/report/pipeline/observation/service/observation/NrtObservationWriter.java create mode 100644 reporting-pipeline-service/src/main/java/gov/cdc/nbs/report/pipeline/observation/service/observation/ObservationProcessor.java create mode 100644 reporting-pipeline-service/src/main/java/gov/cdc/nbs/report/pipeline/observation/transformer/ObservationParser.java diff --git a/reporting-pipeline-service/src/main/java/gov/cdc/nbs/report/pipeline/observation/model/dto/observation/ObservationCodedKey.java b/reporting-pipeline-service/src/main/java/gov/cdc/nbs/report/pipeline/observation/model/dto/observation/ObservationCodedKey.java index 71162fa64..48a894993 100644 --- a/reporting-pipeline-service/src/main/java/gov/cdc/nbs/report/pipeline/observation/model/dto/observation/ObservationCodedKey.java +++ b/reporting-pipeline-service/src/main/java/gov/cdc/nbs/report/pipeline/observation/model/dto/observation/ObservationCodedKey.java @@ -2,13 +2,6 @@ import com.fasterxml.jackson.databind.PropertyNamingStrategies; import com.fasterxml.jackson.databind.annotation.JsonNaming; -import lombok.Data; -import lombok.NoArgsConstructor; -@Data -@NoArgsConstructor @JsonNaming(PropertyNamingStrategies.SnakeCaseStrategy.class) -public class ObservationCodedKey { - private Long observationUid; - private String ovcCode; -} +public record ObservationCodedKey(Long observationUid, String ovcCode) {} diff --git a/reporting-pipeline-service/src/main/java/gov/cdc/nbs/report/pipeline/observation/model/dto/observation/ObservationKey.java b/reporting-pipeline-service/src/main/java/gov/cdc/nbs/report/pipeline/observation/model/dto/observation/ObservationKey.java index fd7e0fdc7..c0b225c1f 100644 --- a/reporting-pipeline-service/src/main/java/gov/cdc/nbs/report/pipeline/observation/model/dto/observation/ObservationKey.java +++ b/reporting-pipeline-service/src/main/java/gov/cdc/nbs/report/pipeline/observation/model/dto/observation/ObservationKey.java @@ -1,17 +1,5 @@ package gov.cdc.nbs.report.pipeline.observation.model.dto.observation; import com.fasterxml.jackson.annotation.JsonProperty; -import com.fasterxml.jackson.databind.PropertyNamingStrategies; -import com.fasterxml.jackson.databind.annotation.JsonNaming; -import lombok.Data; -import lombok.NoArgsConstructor; -import lombok.NonNull; -@Data -@NoArgsConstructor -@JsonNaming(PropertyNamingStrategies.SnakeCaseStrategy.class) -public class ObservationKey { - @NonNull - @JsonProperty("observation_uid") - private Long observationUid; -} +public record ObservationKey(@JsonProperty("observation_uid") Long observationUid) {} diff --git a/reporting-pipeline-service/src/main/java/gov/cdc/nbs/report/pipeline/observation/model/dto/observation/ObservationMaterialKey.java b/reporting-pipeline-service/src/main/java/gov/cdc/nbs/report/pipeline/observation/model/dto/observation/ObservationMaterialKey.java index de2770bfb..391773d7b 100644 --- a/reporting-pipeline-service/src/main/java/gov/cdc/nbs/report/pipeline/observation/model/dto/observation/ObservationMaterialKey.java +++ b/reporting-pipeline-service/src/main/java/gov/cdc/nbs/report/pipeline/observation/model/dto/observation/ObservationMaterialKey.java @@ -1,17 +1,6 @@ package gov.cdc.nbs.report.pipeline.observation.model.dto.observation; import com.fasterxml.jackson.annotation.JsonProperty; -import com.fasterxml.jackson.databind.PropertyNamingStrategies; -import com.fasterxml.jackson.databind.annotation.JsonNaming; -import lombok.Data; -import lombok.NoArgsConstructor; import lombok.NonNull; -@Data -@NoArgsConstructor -@JsonNaming(PropertyNamingStrategies.SnakeCaseStrategy.class) -public class ObservationMaterialKey { - @NonNull - @JsonProperty("material_id") - private Long materialId; -} +public record ObservationMaterialKey(@NonNull @JsonProperty("material_id") Long materialId) {} diff --git a/reporting-pipeline-service/src/main/java/gov/cdc/nbs/report/pipeline/observation/model/dto/observation/ParsedObservation.java b/reporting-pipeline-service/src/main/java/gov/cdc/nbs/report/pipeline/observation/model/dto/observation/ParsedObservation.java new file mode 100644 index 000000000..bd97b18bb --- /dev/null +++ b/reporting-pipeline-service/src/main/java/gov/cdc/nbs/report/pipeline/observation/model/dto/observation/ParsedObservation.java @@ -0,0 +1,30 @@ +package gov.cdc.nbs.report.pipeline.observation.model.dto.observation; + +import java.util.ArrayList; +import java.util.List; + +/** + * Output of the ObservationTransformer that contains the ObservationTransformed object as well as + * lists of entites that need to be persisted to the database + */ +public record ParsedObservation( + ObservationTransformed transformed, + List materialEntries, + List codedEntries, + List dateEntries, + List edxEntries, + List numericEntries, + List reasonEntries, + List textEntries) { + public ParsedObservation(ObservationTransformed observationTransformed) { + this( + observationTransformed, + new ArrayList<>(), + new ArrayList<>(), + new ArrayList<>(), + new ArrayList<>(), + new ArrayList<>(), + new ArrayList<>(), + new ArrayList<>()); + } +} diff --git a/reporting-pipeline-service/src/main/java/gov/cdc/nbs/report/pipeline/observation/service/ObservationService.java b/reporting-pipeline-service/src/main/java/gov/cdc/nbs/report/pipeline/observation/service/ObservationService.java index 57dfb08a1..273dcb540 100644 --- a/reporting-pipeline-service/src/main/java/gov/cdc/nbs/report/pipeline/observation/service/ObservationService.java +++ b/reporting-pipeline-service/src/main/java/gov/cdc/nbs/report/pipeline/observation/service/ObservationService.java @@ -1,38 +1,25 @@ package gov.cdc.nbs.report.pipeline.observation.service; -import static gov.cdc.etldatapipeline.commonutil.UtilHelper.*; +import static gov.cdc.etldatapipeline.commonutil.UtilHelper.errorMessage; +import static gov.cdc.etldatapipeline.commonutil.UtilHelper.extractChangeDataCaptureOperation; +import static gov.cdc.etldatapipeline.commonutil.UtilHelper.extractUid; +import static gov.cdc.etldatapipeline.commonutil.UtilHelper.extractValue; import gov.cdc.etldatapipeline.commonutil.DataProcessingException; import gov.cdc.etldatapipeline.commonutil.NoDataException; -import gov.cdc.etldatapipeline.commonutil.json.CustomJsonGeneratorImpl; -import gov.cdc.etldatapipeline.commonutil.metrics.CustomMetrics; -import gov.cdc.nbs.report.pipeline.observation.model.dto.observation.Observation; -import gov.cdc.nbs.report.pipeline.observation.model.dto.observation.ObservationKey; -import gov.cdc.nbs.report.pipeline.observation.model.dto.observation.ObservationReporting; -import gov.cdc.nbs.report.pipeline.observation.model.dto.observation.ObservationTransformed; -import gov.cdc.nbs.report.pipeline.observation.repository.ObservationRepository; -import gov.cdc.nbs.report.pipeline.observation.transformer.ProcessObservationDataUtil; -import io.micrometer.core.instrument.Counter; -import jakarta.annotation.PostConstruct; -import jakarta.persistence.EntityNotFoundException; +import gov.cdc.nbs.report.pipeline.observation.service.observation.ObservationProcessor; import java.util.NoSuchElementException; -import java.util.Optional; import java.util.concurrent.CompletableFuture; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.function.ToLongFunction; -import lombok.RequiredArgsConstructor; -import lombok.Setter; import org.apache.kafka.clients.consumer.ConsumerRecord; import org.apache.kafka.common.errors.SerializationException; -import org.modelmapper.ModelMapper; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.beans.factory.annotation.Value; import org.springframework.kafka.annotation.KafkaListener; import org.springframework.kafka.annotation.RetryableTopic; -import org.springframework.kafka.core.KafkaTemplate; import org.springframework.kafka.retrytopic.DltStrategy; import org.springframework.kafka.retrytopic.TopicSuffixingStrategy; import org.springframework.kafka.support.serializer.DeserializationException; @@ -57,61 +44,32 @@ * */ @Service -@Setter -@RequiredArgsConstructor public class ObservationService { private static final Logger logger = LoggerFactory.getLogger(ObservationService.class); private static final String BEFORE_PATH = "before"; - - @Value("${spring.kafka.topics.nbs.observation}") - private String observationTopic; - - @Value("${spring.kafka.topics.nbs.act-relationship}") - private String actRelationshipTopic; - - @Value("${spring.kafka.topics.nrt.observation}") - private String observationTopicOutputReporting; - - @Value("${featureFlag.thread-pool-size:1}") - private int threadPoolSize; - - private final ObservationRepository observationRepository; - - @Qualifier("observationKafkaTemplate") - private final KafkaTemplate kafkaTemplate; - - private final ProcessObservationDataUtil processObservationDataUtil; - private final ModelMapper modelMapper = new ModelMapper(); - private final CustomJsonGeneratorImpl jsonGenerator = new CustomJsonGeneratorImpl(); - - private ExecutorService obsExecutor; - - private static String topicDebugLog = "Received Observation with id: {} from topic: {}"; - public static final ToLongFunction> toBatchId = - rec -> rec.timestamp() + rec.offset() + rec.partition(); - - ObservationKey observationKey = new ObservationKey(); - - private static final String SERVICE_NAME = "observation-reporting"; - - private final CustomMetrics metrics; - - private Counter msgProcessed; - private Counter msgSuccess; - private Counter msgFailure; - - @PostConstruct - void initMetrics() { - String[] tags = {"service", SERVICE_NAME}; - - msgProcessed = metrics.counter("obs_msg_processed", tags); - msgSuccess = metrics.counter("obs_msg_success", tags); - msgFailure = metrics.counter("obs_msg_failure", tags); + private static final String TOPIC_DEBUG_LOG = "Received Observation with id: {} from topic: {}"; + + private final String observationTopic; + private final String actRelationshipTopic; + private final ObservationProcessor observationProcessor; + private final ExecutorService obsExecutor; + + public ObservationService( + final ObservationProcessor observationProcessor, + @Value("${spring.kafka.topics.nbs.observation}") final String observationTopic, + @Value("${spring.kafka.topics.nbs.act-relationship}") final String actRelationshipTopic, + @Value("${featureFlag.thread-pool-size:1}") final int threadPoolSize) { + this.observationProcessor = observationProcessor; + this.observationTopic = observationTopic; + this.actRelationshipTopic = actRelationshipTopic; obsExecutor = Executors.newFixedThreadPool(threadPoolSize, new CustomizableThreadFactory("obs-")); } + public static final ToLongFunction> toBatchId = + rec -> rec.timestamp() + rec.offset() + rec.partition(); + @RetryableTopic( attempts = "${spring.kafka.consumer.max-retry}", autoCreateTopics = "false", @@ -140,11 +98,11 @@ public CompletableFuture processMessage(ConsumerRecord rec long batchId = toBatchId.applyAsLong(rec); String topic = rec.topic(); String message = rec.value(); - logger.debug(topicDebugLog, message, topic); + logger.debug(TOPIC_DEBUG_LOG, message, topic); if (topic.equals(observationTopic)) { return CompletableFuture.runAsync( - () -> processObservation(message, batchId, true, ""), obsExecutor); + () -> observationProcessor.process(message, batchId, true, ""), obsExecutor); } else if (topic.equals(actRelationshipTopic) && message != null) { return CompletableFuture.runAsync( () -> processActRelationship(message, batchId), obsExecutor); @@ -155,55 +113,6 @@ public CompletableFuture processMessage(ConsumerRecord rec } } - private void processObservation( - String value, - long batchId, - boolean isFromObservationTopic, - String actRelationshipSourceActUid) { - msgProcessed.increment(); - metrics.recordTime( - "obs_msg_processing_seconds", - () -> { - String observationUid = ""; - try { - observationUid = - isFromObservationTopic - ? extractUid(value, "observation_uid") - : actRelationshipSourceActUid; - observationKey.setObservationUid(Long.valueOf(observationUid)); - logger.info(topicDebugLog, observationUid, observationTopic); - Optional observationData = - observationRepository.computeObservations(observationUid); - if (observationData.isPresent()) { - ObservationReporting reportingModel = - modelMapper.map(observationData.get(), ObservationReporting.class); - ObservationTransformed observationTransformed = - processObservationDataUtil.transformObservationData( - observationData.get(), batchId); - modelMapper.map(observationTransformed, reportingModel); - pushKeyValuePairToKafka( - observationKey, reportingModel, observationTopicOutputReporting); - logger.info( - "Observation data (uid={}) sent to {}", - observationUid, - observationTopicOutputReporting); - msgSuccess.increment(); - } else { - throw new EntityNotFoundException( - "Unable to find Observation with id: " + observationUid); - } - } catch (EntityNotFoundException ex) { - msgFailure.increment(); - throw new NoDataException(ex.getMessage(), ex); - } catch (Exception e) { - msgFailure.increment(); - throw new DataProcessingException(errorMessage("Observation", observationUid, e), e); - } - }, - "service", - SERVICE_NAME); - } - private void processActRelationship(String value, long batchId) { String sourceActUid = ""; @@ -220,26 +129,17 @@ private void processActRelationship(String value, long batchId) { return; } - logger.info(topicDebugLog, "Act_relationship", sourceActUid, actRelationshipTopic); + logger.info(TOPIC_DEBUG_LOG, "Act_relationship", sourceActUid, actRelationshipTopic); // For LabReport values, we only need to trigger if the relationship is deleted (not covered // in updates to Observation) // PHC targets are excluded from the LabReport association updates, as the LabReport will // receive // an update in Observation if (typeCd.equals("LabReport") && targetClassCd.equals("OBS")) { - processObservation(value, batchId, false, sourceActUid); + observationProcessor.process(value, batchId, false, sourceActUid); } } catch (Exception e) { throw new DataProcessingException(errorMessage("ActRelationship", sourceActUid, e), e); } } - - // This same method can be used for elastic search as well and that is why the generic model is - // present - private void pushKeyValuePairToKafka( - ObservationKey observationKey, Object model, String topicName) { - String jsonKey = jsonGenerator.generateStringJson(observationKey); - String jsonValue = jsonGenerator.generateStringJson(model); - kafkaTemplate.send(topicName, jsonKey, jsonValue); - } } diff --git a/reporting-pipeline-service/src/main/java/gov/cdc/nbs/report/pipeline/observation/service/observation/NrtObservationWriter.java b/reporting-pipeline-service/src/main/java/gov/cdc/nbs/report/pipeline/observation/service/observation/NrtObservationWriter.java new file mode 100644 index 000000000..eea545a58 --- /dev/null +++ b/reporting-pipeline-service/src/main/java/gov/cdc/nbs/report/pipeline/observation/service/observation/NrtObservationWriter.java @@ -0,0 +1,139 @@ +package gov.cdc.nbs.report.pipeline.observation.service.observation; + +import gov.cdc.nbs.report.pipeline.observation.model.dto.observation.ObservationMaterial; +import gov.cdc.nbs.report.pipeline.observation.model.dto.observation.ParsedObservation; +import java.util.List; +import org.springframework.jdbc.core.simple.JdbcClient; +import org.springframework.stereotype.Component; + +/** + * Responsible for writing Observation data to the following tables: + * + *
    + *
  • nrt_observation_coded + *
  • nrt_observation_date + *
  • nrt_observation_edx + *
  • nrt_observation_material + *
  • nrt_observation_numeric + *
  • nrt_observation_reason + *
  • nrt_observation_txt + *
+ */ +@Component +public class NrtObservationWriter { + + private final JdbcClient client; + + public NrtObservationWriter(final JdbcClient client) { + this.client = client; + } + + public void persist(ParsedObservation parsedObservation) { + persistMaterials(parsedObservation.materialEntries()); + } + + private static final String UPSERT_MATERIAL = + """ + MERGE INTO nrt_observation_material + USING ( + SELECT + act_uid, + type_cd, + material_id, + subject_class_cd, + record_status, + type_desc_txt, + last_chg_time, + material_cd, + material_nm, + material_details, + material_collection_vol, + material_collection_vol_unit, + material_desc, + risk_cd, + risk_desc_txt + FROM + nrt_observation_material + WHERE + act_uid = :act_uid + AND material_id = :material_id + ) AS source + ON nrt_observation_material.act_uid = source.act_uid AND nrt_observation_material.material_id = source.material_id + WHEN MATCHED THEN + UPDATE SET + type_cd = :type_cd, + subject_class_cd = :subject_class_cd, + record_status = :record_status, + type_desc_txt = :type_desc_txt, + last_chg_time = :last_chg_time, + material_cd = :material_cd, + material_nm = :material_nm, + material_details = :material_details, + material_collection_vol = :material_collection_vol, + material_collection_vol_unit = :material_collection_vol_unit, + material_desc = :material_desc, + risk_cd = :risk_cd, + risk_desc_txt = :risk_desc_txt + WHEN NOT MATCHED THEN + INSERT( + act_uid, + type_cd, + material_id, + subject_class_cd, + record_status, + type_desc_txt, + last_chg_time, + material_cd, + material_nm, + material_details, + material_collection_vol, + material_collection_vol_unit, + material_desc, + risk_cd, + risk_desc_txt + ) values ( + :act_uid, + :type_cd, + :material_id, + :subject_class_cd, + :record_status, + :type_desc_txt, + :last_chg_time, + :material_cd, + :material_nm, + :material_details, + :material_collection_vol, + :material_collection_vol_unit, + :material_desc, + :risk_cd, + :risk_desc_txt + ) + """; + + private void persistMaterials(List materials) { + materials.forEach( + m -> { + client + .sql(UPSERT_MATERIAL) + .param("act_uid", m.getActUid()) + .param("material_id", m.getMaterialId()) + .param("type_cd", m.getTypeCd()) + .param("subject_class_cd", m.getSubjectClassCd()) + .param("record_status", m.getRecordStatus()) + .param("type_desc_txt", m.getTypeDescTxt()) + .param("last_chg_time", m.getLastChgTime()) + .param("material_cd", m.getMaterialCd()) + .param("material_nm", m.getMaterialNm()) + .param("material_details", m.getMaterialDetails()) + .param("material_collection_vol", m.getMaterialCollectionVol()) + .param("material_collection_vol_unit", m.getMaterialCollectionVolUnit()) + .param("material_desc", m.getMaterialDesc()) + .param("risk_cd", m.getRiskCd()) + .param("risk_desc_txt", m.getRiskDescTxt()) + // Are these columns ever set? + // :refresh_datetime + // :max_datetime + .update(); + }); + } +} diff --git a/reporting-pipeline-service/src/main/java/gov/cdc/nbs/report/pipeline/observation/service/observation/ObservationProcessor.java b/reporting-pipeline-service/src/main/java/gov/cdc/nbs/report/pipeline/observation/service/observation/ObservationProcessor.java new file mode 100644 index 000000000..cff0e4716 --- /dev/null +++ b/reporting-pipeline-service/src/main/java/gov/cdc/nbs/report/pipeline/observation/service/observation/ObservationProcessor.java @@ -0,0 +1,145 @@ +package gov.cdc.nbs.report.pipeline.observation.service.observation; + +import static gov.cdc.etldatapipeline.commonutil.UtilHelper.errorMessage; +import static gov.cdc.etldatapipeline.commonutil.UtilHelper.extractUid; + +import gov.cdc.etldatapipeline.commonutil.DataProcessingException; +import gov.cdc.etldatapipeline.commonutil.NoDataException; +import gov.cdc.etldatapipeline.commonutil.json.CustomJsonGeneratorImpl; +import gov.cdc.etldatapipeline.commonutil.metrics.CustomMetrics; +import gov.cdc.nbs.report.pipeline.observation.model.dto.observation.Observation; +import gov.cdc.nbs.report.pipeline.observation.model.dto.observation.ObservationKey; +import gov.cdc.nbs.report.pipeline.observation.model.dto.observation.ObservationReporting; +import gov.cdc.nbs.report.pipeline.observation.model.dto.observation.ParsedObservation; +import gov.cdc.nbs.report.pipeline.observation.repository.ObservationRepository; +import gov.cdc.nbs.report.pipeline.observation.transformer.ObservationParser; +import io.micrometer.core.instrument.Counter; +import jakarta.persistence.EntityNotFoundException; +import java.util.Optional; +import org.modelmapper.ModelMapper; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Qualifier; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.kafka.core.KafkaTemplate; +import org.springframework.stereotype.Component; + +/** Handles the processing of Observation data */ +@Component +public class ObservationProcessor { + private static final Logger logger = LoggerFactory.getLogger(ObservationProcessor.class); + + private final ObservationRepository observationRepository; + private final KafkaTemplate kafkaTemplate; + private final NrtObservationWriter nrtWriter; + + private final String observationTopicOutputReporting; + private final String observationTopic; + + private final ModelMapper modelMapper = new ModelMapper(); + private final CustomJsonGeneratorImpl jsonGenerator = new CustomJsonGeneratorImpl(); + + private final CustomMetrics metrics; + private Counter msgProcessed; + private Counter msgSuccess; + private Counter msgFailure; + + public ObservationProcessor( + final CustomMetrics metrics, + final ObservationRepository observationRepository, + @Qualifier("observationKafkaTemplate") final KafkaTemplate kafkaTemplate, + @Value("${spring.kafka.topics.nrt.observation}") final String observationTopicOutputReporting, + @Value("${spring.kafka.topics.nbs.observation}") final String observationTopic, + final NrtObservationWriter nrtWriter) { + this.metrics = metrics; + this.observationRepository = observationRepository; + this.kafkaTemplate = kafkaTemplate; + this.observationTopicOutputReporting = observationTopicOutputReporting; + this.observationTopic = observationTopic; + this.nrtWriter = nrtWriter; + + String[] tags = {"service", "observation-reporting"}; + + msgProcessed = metrics.counter("obs_msg_processed", tags); + msgSuccess = metrics.counter("obs_msg_success", tags); + msgFailure = metrics.counter("obs_msg_failure", tags); + } + + public void process( + String value, + long batchId, + boolean isFromObservationTopic, + String actRelationshipSourceActUid) { + msgProcessed.increment(); + + metrics.recordTime( + "obs_msg_processing_seconds", + () -> { + String observationUid = ""; + try { + // Get the relevant observation_uid + observationUid = + isFromObservationTopic + ? extractUid(value, "observation_uid") + : actRelationshipSourceActUid; + + ObservationKey observationKey = new ObservationKey(Long.valueOf(observationUid)); + + logger.info( + "Received Observation with id: {} from topic: {}", + observationUid, + observationTopic); + + // Query NBS_ODSE for observation data + Optional observationData = + observationRepository.computeObservations(observationUid); + + // Ensure data is returned + if (observationData.isEmpty()) { + throw new EntityNotFoundException( + "Unable to find Observation with id: " + observationUid); + } + + // Convert Entity to reporting object that will be sent to nrt_observation + ObservationReporting reportingModel = + modelMapper.map(observationData.get(), ObservationReporting.class); + + // Parse all fields from incoming mesage + ParsedObservation parsed = ObservationParser.parse(observationData.get(), batchId); + + // Push parsed fields into reporting object + modelMapper.map(parsed.transformed(), reportingModel); + + // Insert parsed data into nrt_ database + nrtWriter.persist(parsed); + + // Send to reporting object to nrt_observation kafka topic + pushKeyValuePairToKafka( + observationKey, reportingModel, observationTopicOutputReporting); + logger.info( + "Observation data (uid={}) sent to {}", + observationUid, + observationTopicOutputReporting); + + msgSuccess.increment(); + + } catch (EntityNotFoundException ex) { + msgFailure.increment(); + throw new NoDataException(ex.getMessage(), ex); + + } catch (Exception e) { + msgFailure.increment(); + throw new DataProcessingException(errorMessage("Observation", observationUid, e), e); + } + }, + "service", + "observation-reporting"); + } + + private void pushKeyValuePairToKafka( + ObservationKey observationKey, Object model, String topicName) { + String jsonKey = jsonGenerator.generateStringJson(observationKey); + String jsonValue = jsonGenerator.generateStringJson(model); + kafkaTemplate.send(topicName, jsonKey, jsonValue); + } +} diff --git a/reporting-pipeline-service/src/main/java/gov/cdc/nbs/report/pipeline/observation/transformer/ObservationParser.java b/reporting-pipeline-service/src/main/java/gov/cdc/nbs/report/pipeline/observation/transformer/ObservationParser.java new file mode 100644 index 000000000..1009d9d23 --- /dev/null +++ b/reporting-pipeline-service/src/main/java/gov/cdc/nbs/report/pipeline/observation/transformer/ObservationParser.java @@ -0,0 +1,517 @@ +package gov.cdc.nbs.report.pipeline.observation.transformer; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule; +import gov.cdc.nbs.report.pipeline.observation.model.dto.observation.Observation; +import gov.cdc.nbs.report.pipeline.observation.model.dto.observation.ObservationCoded; +import gov.cdc.nbs.report.pipeline.observation.model.dto.observation.ObservationDate; +import gov.cdc.nbs.report.pipeline.observation.model.dto.observation.ObservationEdx; +import gov.cdc.nbs.report.pipeline.observation.model.dto.observation.ObservationMaterial; +import gov.cdc.nbs.report.pipeline.observation.model.dto.observation.ObservationNumeric; +import gov.cdc.nbs.report.pipeline.observation.model.dto.observation.ObservationReason; +import gov.cdc.nbs.report.pipeline.observation.model.dto.observation.ObservationTransformed; +import gov.cdc.nbs.report.pipeline.observation.model.dto.observation.ObservationTxt; +import gov.cdc.nbs.report.pipeline.observation.model.dto.observation.ParsedObservation; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.Optional; +import java.util.function.Consumer; +import java.util.function.Function; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class ObservationParser { + private static final Logger logger = LoggerFactory.getLogger(ObservationParser.class); + private static final ObjectMapper objectMapper = + new ObjectMapper().registerModule(new JavaTimeModule()); + + private static final String SUBJECT_CLASS_CD = "subject_class_cd"; + public static final String TYPE_CD = "type_cd"; + public static final String ENTITY_ID = "entity_id"; + public static final String ORDER = "Order"; + public static final String RESULT = "Result"; + public static final String ACT_ID_SEQ = "act_id_seq"; + public static final String ROOT_EXTENSION_TXT = "root_extension_txt"; + + private ObservationParser() {} + + public static ParsedObservation parse(final Observation observation, final long batchId) { + ParsedObservation parsedObservation = new ParsedObservation(new ObservationTransformed()); + ObservationTransformed observationTransformed = parsedObservation.transformed(); + + observationTransformed.setObservationUid(observation.getObservationUid()); + observationTransformed.setReportObservationUid(observation.getObservationUid()); + observationTransformed.setBatchId(batchId); + + // Person Participations + setPersonParticipations( + observation.getPersonParticipations(), + observation.getObsDomainCdSt1(), + observationTransformed); + + // Organization Participations + setOrganizationParticipations( + observation.getOrganizationParticipations(), + observation.getObsDomainCdSt1(), + observationTransformed); + + // Material Participations + setMaterialParticipations( + observation.getMaterialParticipations(), + observation.getObsDomainCdSt1(), + parsedObservation); + + // Follow up Observations + setFollowupObservations( + observation.getFollowupObservations(), + observation.getObsDomainCdSt1(), + observationTransformed); + + // Parent Observations + setParentObservations(observation.getParentObservations(), observationTransformed); + + // Act Ids + setActIds(observation.getActIds(), observationTransformed); + + // Observation Coded data + setObservationCoded(observation.getObsCode(), parsedObservation); + + // Observation Date data + setObservationDate(observation.getObsDate(), parsedObservation); + + // Observation Edx data + setObservationEdx(observation.getEdxIds(), parsedObservation); + + // Observation Numeric data + setObservationNumeric(observation.getObsNum(), parsedObservation); + + // Observation Reason data + setObservationReasons(observation.getObsReason(), parsedObservation); + + // Observation Text data + setObservationTxt(observation.getObsTxt(), parsedObservation); + + return parsedObservation; + } + + private static void setPersonParticipations( + String personParticipations, String obsDomainCdSt1, ObservationTransformed transformed) { + try { + JsonNode personParticipationsJsonArray = parseJsonArray(personParticipations); + + List orderers = new ArrayList<>(); + for (JsonNode jsonNode : personParticipationsJsonArray) { + assertDomainCdMatches(obsDomainCdSt1, ORDER, RESULT); + + String typeCd = getNodeValue(jsonNode, TYPE_CD, JsonNode::asText); + Long entityId = getNodeValue(jsonNode, ENTITY_ID, JsonNode::asLong); + + if (typeCd.equals("PATSBJ")) { + setPersonParticipationRoles(jsonNode, transformed, entityId); + } + + if (ORDER.equals(obsDomainCdSt1)) { + String subjectClassCd = getNodeValue(jsonNode, SUBJECT_CLASS_CD, JsonNode::asText); + if ("PSN".equals(subjectClassCd)) { + switch (typeCd) { + case "ORD": + orderers.add(String.valueOf(entityId)); + break; + case "PATSBJ", "SubjOfMorbReport": + transformed.setPatientId(entityId); + break; + case "PhysicianOfMorb": + transformed.setMorbPhysicianId(entityId); + break; + case "ReporterOfMorbReport": + transformed.setMorbReporterId(entityId); + break; + case "ENT": + transformed.setTranscriptionistId(entityId); + + ifPresentSet(jsonNode, "first_nm", transformed::setTranscriptionistFirstNm); + ifPresentSet(jsonNode, "last_nm", transformed::setTranscriptionistLastNm); + ifPresentSet(jsonNode, "person_id_val", transformed::setTranscriptionistVal); + ifPresentSet( + jsonNode, + "person_id_assign_auth_cd", + transformed::setTranscriptionistIdAssignAuth); + ifPresentSet( + jsonNode, "person_id_type_desc", transformed::setTranscriptionistAuthType); + break; + case "ASS": + transformed.setAssistantInterpreterId(entityId); + + ifPresentSet(jsonNode, "first_nm", transformed::setAssistantInterpreterFirstNm); + ifPresentSet(jsonNode, "last_nm", transformed::setAssistantInterpreterLastNm); + ifPresentSet(jsonNode, "person_id_val", transformed::setAssistantInterpreterVal); + ifPresentSet( + jsonNode, + "person_id_assign_auth_cd", + transformed::setAssistantInterpreterIdAssignAuth); + ifPresentSet( + jsonNode, "person_id_type_desc", transformed::setAssistantInterpreterAuthType); + break; + case "VRF": + transformed.setResultInterpreterId(entityId); + break; + case "PRF": + transformed.setLabTestTechnicianId(entityId); + break; + default: + } + } + } + } + if (!orderers.isEmpty()) { + transformed.setOrderingPersonId(String.join(",", orderers)); + } + } catch (IllegalArgumentException ex) { + logger.info(ex.getMessage(), "PersonParticipations", personParticipations); + } catch (Exception e) { + logger.error( + "Error processing Person Participation JSON array from observation data: {}", + e.getMessage()); + } + } + + private static void setOrganizationParticipations( + String organizationParticipations, + String obsDomainCdSt1, + ObservationTransformed transformed) { + try { + JsonNode organizationParticipationsJsonArray = parseJsonArray(organizationParticipations); + + for (JsonNode jsonNode : organizationParticipationsJsonArray) { + assertDomainCdMatches(obsDomainCdSt1, RESULT, ORDER); + + String typeCd = getNodeValue(jsonNode, TYPE_CD, JsonNode::asText); + String subjectClassCd = getNodeValue(jsonNode, SUBJECT_CLASS_CD, JsonNode::asText); + Long entityId = getNodeValue(jsonNode, ENTITY_ID, JsonNode::asLong); + + if (subjectClassCd.equals("ORG")) { + + if (RESULT.equals(obsDomainCdSt1) && "PRF".equals(typeCd)) { + transformed.setPerformingOrganizationId(entityId); + } else if (ORDER.equals(obsDomainCdSt1)) { + switch (typeCd) { + case "AUT": + transformed.setAuthorOrganizationId(entityId); + break; + case "ORD": + transformed.setOrderingOrganizationId(entityId); + break; + case "HCFAC": + transformed.setHealthCareId(entityId); + break; + case "ReporterOfMorbReport": + transformed.setMorbHospReporterId(entityId); + break; + case "HospOfMorbObs": + transformed.setMorbHospId(entityId); + break; + default: + break; + } + } + } + } + } catch (IllegalArgumentException ex) { + logger.info(ex.getMessage(), "OrganizationParticipations", organizationParticipations); + } catch (Exception e) { + logger.error( + "Error processing Organization Participation JSON array from observation data: {}", + e.getMessage()); + } + } + + private static void setMaterialParticipations( + String materialParticipations, String obsDomainCdSt1, ParsedObservation parsedObservation) { + try { + JsonNode materialParticipationsJsonArray = parseJsonArray(materialParticipations); + + for (JsonNode jsonNode : materialParticipationsJsonArray) { + String typeCd = getNodeValue(jsonNode, TYPE_CD, JsonNode::asText); + String subjectClassCd = getNodeValue(jsonNode, SUBJECT_CLASS_CD, JsonNode::asText); + + assertDomainCdMatches(obsDomainCdSt1, ORDER); + if ("SPC".equals(typeCd) && "MAT".equals(subjectClassCd)) { + Long materialId = jsonNode.get(ENTITY_ID).asLong(); + parsedObservation.transformed().setMaterialId(materialId); + + ObservationMaterial material = + objectMapper.treeToValue(jsonNode, ObservationMaterial.class); + material.setMaterialId(materialId); + + // Add material to list that will be persisted to the database + parsedObservation.materialEntries().add(material); + } + } + } catch (IllegalArgumentException ex) { + logger.info(ex.getMessage(), "MaterialParticipations", materialParticipations); + } catch (Exception e) { + logger.error( + "Error processing Material Participation JSON array from observation data: {}", + e.getMessage()); + } + } + + private static void setPersonParticipationRoles( + JsonNode node, ObservationTransformed observationTransformed, Long entityId) { + String roleSubject = fieldAsText(node, "role_subject_class_cd"); + if ("PROV".equals(roleSubject)) { + String roleCd = fieldAsText(node, "role_cd"); + if ("SPP".equals(roleCd)) { + String roleScoping = fieldAsText(node, "role_scoping_class_cd"); + if ("PSN".equals(roleScoping)) { + observationTransformed.setSpecimenCollectorId(entityId); + } + } else if ("CT".equals(roleCd)) { + observationTransformed.setCopyToProviderId(entityId); + } + } + } + + private static void setFollowupObservations( + String followupObservations, String obsDomainCdSt1, ObservationTransformed transformed) { + try { + JsonNode followupObservationsJsonArray = parseJsonArray(followupObservations); + + List results = new ArrayList<>(); + List followUps = new ArrayList<>(); + for (JsonNode jsonNode : followupObservationsJsonArray) { + String domainCd = fieldAsText(jsonNode, "domain_cd_st_1"); + assertDomainCdMatches(obsDomainCdSt1, ORDER); + + if (RESULT.equals(domainCd)) { + Optional.ofNullable(jsonNode.get("result_observation_uid")) + .ifPresent(r -> results.add(r.asText())); + } else { + Optional.ofNullable(jsonNode.get("result_observation_uid")) + .ifPresent(r -> followUps.add(r.asText())); + } + } + + if (!results.isEmpty()) { + transformed.setResultObservationUid(String.join(",", results)); + } + if (!followUps.isEmpty()) { + transformed.setFollowUpObservationUid(String.join(",", followUps)); + } + } catch (IllegalArgumentException ex) { + logger.info(ex.getMessage(), "FollowupObservations", followupObservations); + } catch (Exception e) { + logger.error( + "Error processing Followup Observations JSON array from observation data: {}", + e.getMessage()); + } + } + + private static void setParentObservations( + String parentObservations, ObservationTransformed transformed) { + try { + JsonNode parentObservationsJsonArray = parseJsonArray(parentObservations); + + for (JsonNode jsonNode : parentObservationsJsonArray) { + Long parentUid = getNodeValue(jsonNode, "parent_uid", JsonNode::asLong); + String parentTypeCd = fieldAsText(jsonNode, "parent_type_cd"); + String parentDomainCd = fieldAsText(jsonNode, "parent_domain_cd_st_1"); + + if ("SPRT".equals(parentTypeCd)) { + transformed.setReportSprtUid(parentUid); + } else if ("REFR".equals(parentTypeCd)) { + transformed.setReportRefrUid(parentUid); + } + + if (parentDomainCd.contains(ORDER)) { + transformed.setReportObservationUid(parentUid); + } + } + } catch (IllegalArgumentException ex) { + logger.info(ex.getMessage(), "ParentObservations", parentObservations); + } catch (Exception e) { + logger.error( + "Error processing Parent Observations JSON array from observation data: {}", + e.getMessage()); + } + } + + private static void setActIds(String actIds, ObservationTransformed observationTransformed) { + try { + JsonNode actIdsJsonArray = parseJsonArray(actIds); + + for (JsonNode jsonNode : actIdsJsonArray) { + String typeCd = getNodeValue(jsonNode, TYPE_CD, JsonNode::asText); + Integer actIdSeq = getNodeValue(jsonNode, ACT_ID_SEQ, JsonNode::asInt); + if (typeCd.equals("FN")) { + String rootExtTxt = getNodeValue(jsonNode, ROOT_EXTENSION_TXT, JsonNode::asText); + observationTransformed.setAccessionNumber(rootExtTxt); + } + if (typeCd.equals("EII") && actIdSeq.equals(3)) { + String rootExtTxt = getNodeValue(jsonNode, ROOT_EXTENSION_TXT, JsonNode::asText); + observationTransformed.setDeviceInstanceId1(rootExtTxt); + } + if (typeCd.equals("EII") && actIdSeq.equals(4)) { + String rootExtTxt = getNodeValue(jsonNode, ROOT_EXTENSION_TXT, JsonNode::asText); + observationTransformed.setDeviceInstanceId2(rootExtTxt); + } + } + } catch (IllegalArgumentException ex) { + logger.info(ex.getMessage(), "ActIds", actIds); + } catch (Exception e) { + logger.error("Error processing Act Ids JSON array from observation data: {}", e.getMessage()); + } + } + + private static void setObservationCoded( + String observationCoded, ParsedObservation parsedObservation) { + try { + JsonNode observationCodedJsonArray = parseJsonArray(observationCoded); + + for (JsonNode jsonNode : observationCodedJsonArray) { + ObservationCoded coded = objectMapper.treeToValue(jsonNode, ObservationCoded.class); + coded.setBatchId(parsedObservation.transformed().getBatchId()); + + parsedObservation.codedEntries().add(coded); + } + } catch (IllegalArgumentException ex) { + logger.info(ex.getMessage(), "ObservationCoded"); + } catch (Exception e) { + logger.error( + "Error processing Observation Coded JSON array from observation data: {}", + e.getMessage()); + } + } + + private static void setObservationDate( + String observationDate, ParsedObservation parsedObservation) { + try { + JsonNode observationDateJsonArray = parseJsonArray(observationDate); + + for (JsonNode jsonNode : observationDateJsonArray) { + ObservationDate obsDate = objectMapper.treeToValue(jsonNode, ObservationDate.class); + obsDate.setBatchId(parsedObservation.transformed().getBatchId()); + + parsedObservation.dateEntries().add(obsDate); + } + } catch (IllegalArgumentException ex) { + logger.info(ex.getMessage(), "ObservationDate"); + } catch (Exception e) { + logger.error( + "Error processing Observation Date JSON array from observation data: {}", e.getMessage()); + } + } + + private static void setObservationEdx( + String observationEdx, ParsedObservation parsedObservation) { + try { + JsonNode observationEdxJsonArray = parseJsonArray(observationEdx); + for (JsonNode jsonNode : observationEdxJsonArray) { + ObservationEdx edx = objectMapper.treeToValue(jsonNode, ObservationEdx.class); + + parsedObservation.edxEntries().add(edx); + } + } catch (IllegalArgumentException ex) { + logger.info(ex.getMessage(), "ObservationEdx"); + } catch (Exception e) { + logger.error( + "Error processing Observation Edx JSON array from observation data: {}", e.getMessage()); + } + } + + private static void setObservationNumeric( + String observationNumeric, ParsedObservation parsedObservation) { + try { + JsonNode observationNumericJsonArray = parseJsonArray(observationNumeric); + + for (JsonNode jsonNode : observationNumericJsonArray) { + ObservationNumeric numeric = objectMapper.treeToValue(jsonNode, ObservationNumeric.class); + numeric.setBatchId(parsedObservation.transformed().getBatchId()); + parsedObservation.numericEntries().add(numeric); + } + } catch (IllegalArgumentException ex) { + logger.info(ex.getMessage(), "ObservationNumeric"); + } catch (Exception e) { + logger.error( + "Error processing Observation Numeric JSON array from observation data: {}", + e.getMessage()); + } + } + + private static void setObservationReasons( + String observationReasons, ParsedObservation parsedObservation) { + try { + JsonNode observationReasonsJsonArray = parseJsonArray(observationReasons); + + for (JsonNode jsonNode : observationReasonsJsonArray) { + ObservationReason reason = objectMapper.treeToValue(jsonNode, ObservationReason.class); + reason.setBatchId(parsedObservation.transformed().getBatchId()); + parsedObservation.reasonEntries().add(reason); + } + } catch (IllegalArgumentException ex) { + logger.info(ex.getMessage(), "ObservationReasons"); + } catch (Exception e) { + logger.error( + "Error processing Observation Reasons JSON array from observation data: {}", + e.getMessage()); + } + } + + private static void setObservationTxt( + String observationTxt, ParsedObservation parsedObservation) { + try { + JsonNode observationTxtJsonArray = parseJsonArray(observationTxt); + + for (JsonNode jsonNode : observationTxtJsonArray) { + ObservationTxt txt = objectMapper.treeToValue(jsonNode, ObservationTxt.class); + txt.setBatchId(parsedObservation.transformed().getBatchId()); + + parsedObservation.textEntries().add(txt); + } + } catch (IllegalArgumentException ex) { + logger.info(ex.getMessage(), "ObservationTxt"); + } catch (Exception e) { + logger.error( + "Error processing Observation Txt JSON array from observation data: {}", e.getMessage()); + } + } + + private static JsonNode parseJsonArray(String jsonString) + throws JsonProcessingException, IllegalArgumentException { + JsonNode jsonArray = jsonString != null ? objectMapper.readTree(jsonString) : null; + if (jsonArray != null && jsonArray.isArray()) { + return jsonArray; + } else { + throw new IllegalArgumentException("{} array is null."); + } + } + + private static T getNodeValue( + JsonNode jsonNode, String fieldName, Function mapper) { + JsonNode node = jsonNode.get(fieldName); + if (node == null || node.isNull()) { + throw new IllegalArgumentException("Field " + fieldName + " is null or not found in {}: {}"); + } + return mapper.apply(node); + } + + private static void assertDomainCdMatches(String value, String... vals) { + if (Arrays.stream(vals).noneMatch(value::equals)) { + throw new IllegalArgumentException("obsDomainCdSt1: " + value + " is not valid for the {}"); + } + } + + private static String fieldAsText(JsonNode node, String field) { + return node.path(field).asText(null); + } + + private static void ifPresentSet(JsonNode node, String field, Consumer setter) { + String value = fieldAsText(node, field); + if (value != null) { + setter.accept(value); + } + } +} diff --git a/reporting-pipeline-service/src/main/java/gov/cdc/nbs/report/pipeline/observation/transformer/ProcessObservationDataUtil.java b/reporting-pipeline-service/src/main/java/gov/cdc/nbs/report/pipeline/observation/transformer/ProcessObservationDataUtil.java index 9066b9d21..83653c790 100644 --- a/reporting-pipeline-service/src/main/java/gov/cdc/nbs/report/pipeline/observation/transformer/ProcessObservationDataUtil.java +++ b/reporting-pipeline-service/src/main/java/gov/cdc/nbs/report/pipeline/observation/transformer/ProcessObservationDataUtil.java @@ -6,8 +6,6 @@ import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule; import gov.cdc.etldatapipeline.commonutil.json.CustomJsonGeneratorImpl; import gov.cdc.nbs.report.pipeline.observation.model.dto.observation.*; -import gov.cdc.nbs.report.pipeline.observation.model.dto.observation.Observation; -import gov.cdc.nbs.report.pipeline.observation.model.dto.observation.ObservationTransformed; import java.util.ArrayList; import java.util.Arrays; import java.util.List; @@ -56,8 +54,6 @@ public class ProcessObservationDataUtil { @Value("${spring.kafka.topics.nrt.observation-txt}") public String txtTopicName; - ObservationKey observationKey = new ObservationKey(); - private static final String SUBJECT_CLASS_CD = "subject_class_cd"; public static final String TYPE_CD = "type_cd"; public static final String ENTITY_ID = "entity_id"; @@ -67,12 +63,13 @@ public class ProcessObservationDataUtil { public static final String ROOT_EXTENSION_TXT = "root_extension_txt"; public ObservationTransformed transformObservationData(Observation observation, long batchId) { + // Convert to new object ObservationTransformed observationTransformed = new ObservationTransformed(); observationTransformed.setObservationUid(observation.getObservationUid()); observationTransformed.setReportObservationUid(observation.getObservationUid()); observationTransformed.setBatchId(batchId); - observationKey.setObservationUid(observation.getObservationUid()); + ObservationKey observationKey = new ObservationKey(observation.getObservationUid()); String obsDomainCdSt1 = observation.getObsDomainCdSt1(); transformPersonParticipations( @@ -86,9 +83,9 @@ public ObservationTransformed transformObservationData(Observation observation, transformParentObservations(observation.getParentObservations(), observationTransformed); transformActIds(observation.getActIds(), observationTransformed); transformObservationCoded(observation.getObsCode(), batchId); - transformObservationDate(observation.getObsDate(), batchId); + transformObservationDate(observationKey, observation.getObsDate(), batchId); transformObservationEdx(observation.getEdxIds()); - transformObservationNumeric(observation.getObsNum(), batchId); + transformObservationNumeric(observationKey, observation.getObsNum(), batchId); transformObservationReasons(observation.getObsReason(), batchId); transformObservationTxt(observation.getObsTxt(), batchId); @@ -280,8 +277,8 @@ private void transformMaterialParticipations( ObservationMaterial material = objectMapper.treeToValue(jsonNode, ObservationMaterial.class); material.setMaterialId(materialId); - ObservationMaterialKey key = new ObservationMaterialKey(); - key.setMaterialId(observationTransformed.getMaterialId()); + ObservationMaterialKey key = + new ObservationMaterialKey(observationTransformed.getMaterialId()); sendToKafka( key, material, @@ -396,12 +393,11 @@ private void transformObservationCoded(String observationCoded, long batchId) { try { JsonNode observationCodedJsonArray = parseJsonArray(observationCoded); - ObservationCodedKey codedKey = new ObservationCodedKey(); for (JsonNode jsonNode : observationCodedJsonArray) { ObservationCoded coded = objectMapper.treeToValue(jsonNode, ObservationCoded.class); coded.setBatchId(batchId); - codedKey.setObservationUid(coded.getObservationUid()); - codedKey.setOvcCode(coded.getOvcCode()); + ObservationCodedKey codedKey = + new ObservationCodedKey(coded.getObservationUid(), coded.getOvcCode()); sendToKafka( codedKey, coded, @@ -418,7 +414,8 @@ private void transformObservationCoded(String observationCoded, long batchId) { } } - private void transformObservationDate(String observationDate, long batchId) { + private void transformObservationDate( + ObservationKey observationKey, String observationDate, long batchId) { try { JsonNode observationDateJsonArray = parseJsonArray(observationDate); @@ -463,7 +460,8 @@ private void transformObservationEdx(String observationEdx) { } } - private void transformObservationNumeric(String observationNumeric, long batchId) { + private void transformObservationNumeric( + ObservationKey observationKey, String observationNumeric, long batchId) { try { JsonNode observationNumericJsonArray = parseJsonArray(observationNumeric); diff --git a/reporting-pipeline-service/src/test/java/gov/cdc/nbs/report/pipeline/integration/functional/DataDrivenFunctionalTests.java b/reporting-pipeline-service/src/test/java/gov/cdc/nbs/report/pipeline/integration/functional/DataDrivenFunctionalTests.java index fd64757b8..e50c01094 100644 --- a/reporting-pipeline-service/src/test/java/gov/cdc/nbs/report/pipeline/integration/functional/DataDrivenFunctionalTests.java +++ b/reporting-pipeline-service/src/test/java/gov/cdc/nbs/report/pipeline/integration/functional/DataDrivenFunctionalTests.java @@ -22,6 +22,7 @@ import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.MethodSource; import org.skyscreamer.jsonassert.JSONAssert; +import org.skyscreamer.jsonassert.JSONCompare; import org.skyscreamer.jsonassert.JSONCompareMode; import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.jdbc.core.simple.JdbcClient; @@ -75,10 +76,11 @@ static Stream functionalTestDirectoryProvider() throws IOException { * @param testDirectory * @throws IOException If step folder is missing a file, an exception will be thrown. * @throws JSONException + * @throws InterruptedException */ @ParameterizedTest @MethodSource("functionalTestDirectoryProvider") - void testRunner(Path testDirectory) throws IOException, JSONException { + void testRunner(Path testDirectory) throws IOException, JSONException, InterruptedException { System.out.println( "Executing DataDrivenFunctionalTest for directory: " + testDirectory.getFileName()); @@ -114,6 +116,10 @@ void testRunner(Path testDirectory) throws IOException, JSONException { assertThat(results).isPresent(); String actual = mapper.writeValueAsString(results.get()); + // TEMP!! + if (JSONCompare.compareJSON(expectedResult, actual, JSONCompareMode.LENIENT).failed()) { + System.out.println("THINGS FAILED! EXAMINE!!!"); + } JSONAssert.assertEquals(expectedResult, actual, JSONCompareMode.LENIENT); } } diff --git a/reporting-pipeline-service/src/test/java/gov/cdc/nbs/report/pipeline/observation/service/ObservationServiceTest.java b/reporting-pipeline-service/src/test/java/gov/cdc/nbs/report/pipeline/observation/service/ObservationServiceTest.java index 356e65446..f66b8ee7e 100644 --- a/reporting-pipeline-service/src/test/java/gov/cdc/nbs/report/pipeline/observation/service/ObservationServiceTest.java +++ b/reporting-pipeline-service/src/test/java/gov/cdc/nbs/report/pipeline/observation/service/ObservationServiceTest.java @@ -4,6 +4,8 @@ import static gov.cdc.nbs.report.pipeline.observation.service.ObservationService.toBatchId; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.ArgumentMatchers.isNull; import static org.mockito.Mockito.*; import com.fasterxml.jackson.core.JsonProcessingException; @@ -14,6 +16,8 @@ import gov.cdc.nbs.report.pipeline.observation.model.dto.observation.ObservationKey; import gov.cdc.nbs.report.pipeline.observation.model.dto.observation.ObservationReporting; import gov.cdc.nbs.report.pipeline.observation.repository.ObservationRepository; +import gov.cdc.nbs.report.pipeline.observation.service.observation.NrtObservationWriter; +import gov.cdc.nbs.report.pipeline.observation.service.observation.ObservationProcessor; import gov.cdc.nbs.report.pipeline.observation.transformer.ProcessObservationDataUtil; import io.micrometer.core.instrument.simple.SimpleMeterRegistry; import java.util.NoSuchElementException; @@ -33,12 +37,14 @@ class ObservationServiceTest { - @InjectMocks private ObservationService observationService; + private ObservationService observationService; @Mock private ObservationRepository observationRepository; @Mock private KafkaTemplate kafkaTemplate; + @Mock private NrtObservationWriter nrtWrtier; + @Captor private ArgumentCaptor topicCaptor; @Captor private ArgumentCaptor keyCaptor; @@ -60,15 +66,16 @@ void setUp() { transformer.setMaterialTopicName("materialTopic"); observationService = new ObservationService( - observationRepository, - kafkaTemplate, - transformer, - new CustomMetrics(new SimpleMeterRegistry())); - observationService.setObservationTopic(inputTopicNameObservation); - observationService.setActRelationshipTopic(inputTopicNameActRelationship); - observationService.setObservationTopicOutputReporting(outputTopicNameObservation); - observationService.setThreadPoolSize(1); - observationService.initMetrics(); + new ObservationProcessor( + new CustomMetrics(new SimpleMeterRegistry()), + observationRepository, + kafkaTemplate, + outputTopicNameObservation, + inputTopicNameObservation, + nrtWrtier), + inputTopicNameObservation, + inputTopicNameActRelationship, + 1); transformer.setCodedTopicName("ObservationCoded"); transformer.setReasonTopicName("ObservationReason"); @@ -199,8 +206,7 @@ private void validateData(String payload, Observation observation, String inputT ConsumerRecord rec = getRecord(payload, inputTopic); observationService.processMessage(rec); - ObservationKey observationKey = new ObservationKey(); - observationKey.setObservationUid(observation.getObservationUid()); + ObservationKey observationKey = new ObservationKey(observation.getObservationUid()); var reportingModel = constructObservationReporting( @@ -208,10 +214,10 @@ private void validateData(String payload, Observation observation, String inputT reportingModel.setBatchId(toBatchId.applyAsLong(rec)); Awaitility.await() - .atMost(1, TimeUnit.SECONDS) + .atMost(1, TimeUnit.HOURS) .untilAsserted( () -> - verify(kafkaTemplate, times(2)) + verify(kafkaTemplate, times(1)) .send(topicCaptor.capture(), keyCaptor.capture(), messageCaptor.capture())); String actualTopic = topicCaptor.getValue(); String actualKey = keyCaptor.getValue(); diff --git a/reporting-pipeline-service/src/test/resources/application-test.yaml b/reporting-pipeline-service/src/test/resources/application-test.yaml index 6594ffa17..5b42dabbc 100644 --- a/reporting-pipeline-service/src/test/resources/application-test.yaml +++ b/reporting-pipeline-service/src/test/resources/application-test.yaml @@ -24,3 +24,7 @@ spring: value-deserializer: org.springframework.kafka.support.serializer.JsonDeserializer maxPollIntervalMs: 3000 metadata.max.age.ms: 6000 +service: + fixed-delay: + cached-ids: 1000 + datamart: 3000 \ No newline at end of file From 29886bc8855f3ae6592ffb8e0f577ef11b50e13f Mon Sep 17 00:00:00 2001 From: Michael Peels Date: Thu, 16 Apr 2026 14:52:43 -0400 Subject: [PATCH 02/16] WIP upserts --- .../model/dto/observation/ObservationEdx.java | 1 + .../observation/NrtObservationWriter.java | 382 ++++++++++++++---- .../transformer/ObservationParser.java | 1 + 3 files changed, 311 insertions(+), 73 deletions(-) diff --git a/reporting-pipeline-service/src/main/java/gov/cdc/nbs/report/pipeline/observation/model/dto/observation/ObservationEdx.java b/reporting-pipeline-service/src/main/java/gov/cdc/nbs/report/pipeline/observation/model/dto/observation/ObservationEdx.java index 0324e8db7..2e7bfc37d 100644 --- a/reporting-pipeline-service/src/main/java/gov/cdc/nbs/report/pipeline/observation/model/dto/observation/ObservationEdx.java +++ b/reporting-pipeline-service/src/main/java/gov/cdc/nbs/report/pipeline/observation/model/dto/observation/ObservationEdx.java @@ -12,4 +12,5 @@ public class ObservationEdx { private Long edxDocumentUid; private Long edxActUid; private String edxAddTime; + private Long batchId; } diff --git a/reporting-pipeline-service/src/main/java/gov/cdc/nbs/report/pipeline/observation/service/observation/NrtObservationWriter.java b/reporting-pipeline-service/src/main/java/gov/cdc/nbs/report/pipeline/observation/service/observation/NrtObservationWriter.java index eea545a58..44b6e636d 100644 --- a/reporting-pipeline-service/src/main/java/gov/cdc/nbs/report/pipeline/observation/service/observation/NrtObservationWriter.java +++ b/reporting-pipeline-service/src/main/java/gov/cdc/nbs/report/pipeline/observation/service/observation/NrtObservationWriter.java @@ -1,6 +1,10 @@ package gov.cdc.nbs.report.pipeline.observation.service.observation; +import gov.cdc.nbs.report.pipeline.observation.model.dto.observation.ObservationCoded; +import gov.cdc.nbs.report.pipeline.observation.model.dto.observation.ObservationDate; +import gov.cdc.nbs.report.pipeline.observation.model.dto.observation.ObservationEdx; import gov.cdc.nbs.report.pipeline.observation.model.dto.observation.ObservationMaterial; +import gov.cdc.nbs.report.pipeline.observation.model.dto.observation.ObservationNumeric; import gov.cdc.nbs.report.pipeline.observation.model.dto.observation.ParsedObservation; import java.util.List; import org.springframework.jdbc.core.simple.JdbcClient; @@ -30,6 +34,12 @@ public NrtObservationWriter(final JdbcClient client) { public void persist(ParsedObservation parsedObservation) { persistMaterials(parsedObservation.materialEntries()); + persistCoded(parsedObservation.codedEntries()); + persistDate(parsedObservation.dateEntries()); + persistEdx(parsedObservation.edxEntries()); + persistNumeric(parsedObservation.numericEntries()); + // TODO ObservationReason + // TODO ObservationTxt } private static final String UPSERT_MATERIAL = @@ -37,43 +47,38 @@ public void persist(ParsedObservation parsedObservation) { MERGE INTO nrt_observation_material USING ( SELECT - act_uid, - type_cd, - material_id, - subject_class_cd, - record_status, - type_desc_txt, - last_chg_time, - material_cd, - material_nm, - material_details, - material_collection_vol, - material_collection_vol_unit, - material_desc, - risk_cd, - risk_desc_txt - FROM - nrt_observation_material - WHERE - act_uid = :act_uid - AND material_id = :material_id + :act_uid AS act_uid, + :material_id AS material_id, + :type_cd AS type_cd, + :subject_class_cd AS subject_class_cd, + :record_status AS record_status, + :type_desc_txt AS type_desc_txt, + :last_chg_time AS last_chg_time, + :material_cd AS material_cd, + :material_nm AS material_nm, + :material_details AS material_details, + :material_collection_vol AS material_collection_vol, + :material_collection_vol_unit AS material_collection_vol_unit, + :material_desc AS material_desc, + :risk_cd AS risk_cd, + :risk_desc_txt AS risk_desc_txt ) AS source ON nrt_observation_material.act_uid = source.act_uid AND nrt_observation_material.material_id = source.material_id WHEN MATCHED THEN UPDATE SET - type_cd = :type_cd, - subject_class_cd = :subject_class_cd, - record_status = :record_status, - type_desc_txt = :type_desc_txt, - last_chg_time = :last_chg_time, - material_cd = :material_cd, - material_nm = :material_nm, - material_details = :material_details, - material_collection_vol = :material_collection_vol, - material_collection_vol_unit = :material_collection_vol_unit, - material_desc = :material_desc, - risk_cd = :risk_cd, - risk_desc_txt = :risk_desc_txt + type_cd = source.type_cd, + subject_class_cd = source.subject_class_cd, + record_status = source.record_status, + type_desc_txt = source.type_desc_txt, + last_chg_time = source.last_chg_time, + material_cd = source.material_cd, + material_nm = source.material_nm, + material_details = source.material_details, + material_collection_vol = source.material_collection_vol, + material_collection_vol_unit = source.material_collection_vol_unit, + material_desc = source.material_desc, + risk_cd = source.risk_cd, + risk_desc_txt = source.risk_desc_txt WHEN NOT MATCHED THEN INSERT( act_uid, @@ -91,49 +96,280 @@ public void persist(ParsedObservation parsedObservation) { material_desc, risk_cd, risk_desc_txt - ) values ( - :act_uid, - :type_cd, - :material_id, - :subject_class_cd, - :record_status, - :type_desc_txt, - :last_chg_time, - :material_cd, - :material_nm, - :material_details, - :material_collection_vol, - :material_collection_vol_unit, - :material_desc, - :risk_cd, - :risk_desc_txt - ) + ) VALUES ( + source.act_uid, + source.type_cd, + source.material_id, + source.subject_class_cd, + source.record_status, + source.type_desc_txt, + source.last_chg_time, + source.material_cd, + source.material_nm, + source.material_details, + source.material_collection_vol, + source.material_collection_vol_unit, + source.material_desc, + source.risk_cd, + source.risk_desc_txt + ); """; private void persistMaterials(List materials) { materials.forEach( - m -> { - client - .sql(UPSERT_MATERIAL) - .param("act_uid", m.getActUid()) - .param("material_id", m.getMaterialId()) - .param("type_cd", m.getTypeCd()) - .param("subject_class_cd", m.getSubjectClassCd()) - .param("record_status", m.getRecordStatus()) - .param("type_desc_txt", m.getTypeDescTxt()) - .param("last_chg_time", m.getLastChgTime()) - .param("material_cd", m.getMaterialCd()) - .param("material_nm", m.getMaterialNm()) - .param("material_details", m.getMaterialDetails()) - .param("material_collection_vol", m.getMaterialCollectionVol()) - .param("material_collection_vol_unit", m.getMaterialCollectionVolUnit()) - .param("material_desc", m.getMaterialDesc()) - .param("risk_cd", m.getRiskCd()) - .param("risk_desc_txt", m.getRiskDescTxt()) - // Are these columns ever set? - // :refresh_datetime - // :max_datetime - .update(); - }); + m -> + client + .sql(UPSERT_MATERIAL) + .param("act_uid", m.getActUid()) + .param("material_id", m.getMaterialId()) + .param("type_cd", m.getTypeCd()) + .param("subject_class_cd", m.getSubjectClassCd()) + .param("record_status", m.getRecordStatus()) + .param("type_desc_txt", m.getTypeDescTxt()) + .param("last_chg_time", m.getLastChgTime()) + .param("material_cd", m.getMaterialCd()) + .param("material_nm", m.getMaterialNm()) + .param("material_details", m.getMaterialDetails()) + .param("material_collection_vol", m.getMaterialCollectionVol()) + .param("material_collection_vol_unit", m.getMaterialCollectionVolUnit()) + .param("material_desc", m.getMaterialDesc()) + .param("risk_cd", m.getRiskCd()) + .param("risk_desc_txt", m.getRiskDescTxt()) + .update()); + } + + private static final String UPSERT_CODED = + """ + MERGE INTO nrt_observation_coded + USING ( + SELECT + :observation_uid AS observation_uid, + :ovc_code AS ovc_code, + :ovc_code_system_cd AS ovc_code_system_cd, + :ovc_code_system_desc_txt AS ovc_code_system_desc_txt, + :ovc_display_name AS ovc_display_name, + :ovc_alt_cd AS ovc_alt_cd, + :ovc_alt_cd_desc_txt AS ovc_alt_cd_desc_txt, + :ovc_alt_cd_system_cd AS ovc_alt_cd_system_cd, + :ovc_alt_cd_system_desc_txt AS ovc_alt_cd_system_desc_txt, + :batch_id AS batch_id + ) AS source + ON nrt_observation_coded.observation_uid = source.observation_uid AND nrt_observation_coded.ovc_code = source.ovc_code + WHEN MATCHED THEN + UPDATE SET + observation_uid = source.observation_uid + ovc_code = source.ovc_code + ovc_code_system_cd = source.ovc_code_system_cd + ovc_code_system_desc_txt = source.ovc_code_system_desc_txt + ovc_display_name = source.ovc_display_name + ovc_alt_cd = source.ovc_alt_cd + ovc_alt_cd_desc_txt = source.ovc_alt_cd_desc_txt + ovc_alt_cd_system_cd = source.ovc_alt_cd_system_cd + ovc_alt_cd_system_desc_txt = source.ovc_alt_cd_system_desc_txt + batch_id = source.batch_id + WHEN NOT MATCHED THEN + INSERT ( + observation_uid, + ovc_code, + ovc_code_system_cd, + ovc_code_system_desc_txt, + ovc_display_name, + ovc_alt_cd, + ovc_alt_cd_desc_txt, + ovc_alt_cd_system_cd, + ovc_alt_cd_system_desc_txt, + batch_id + ) VALUES ( + source.observation_uid + source.ovc_code + source.ovc_code_system_cd + source.ovc_code_system_desc_txt + source.ovc_display_name + source.ovc_alt_cd + source.ovc_alt_cd_desc_txt + source.ovc_alt_cd_system_cd + source.ovc_alt_cd_system_desc_txt + source.batch_id + ); + """; + + private void persistCoded(List codedEntries) { + codedEntries.forEach( + c -> + client + .sql(UPSERT_CODED) + .param("observation_uid", c.getObservationUid()) + .param("ovc_code", c.getOvcCode()) + .param("ovc_code_system_cd", c.getOvcCodeSystemCd()) + .param("ovc_code_system_desc_txt", c.getOvcCodeSystemDescTxt()) + .param("ovc_display_name", c.getOvcDisplayName()) + .param("ovc_alt_cd", c.getOvcAltCd()) + .param("ovc_alt_cd_desc_txt", c.getOvcAltCdDescTxt()) + .param("ovc_alt_cd_system_cd", c.getOvcAltCdSystemCd()) + .param("ovc_alt_cd_system_desc_txt", c.getOvcAltCdSystemDescTxt()) + .param("batch_id", c.getBatchId()) + .update()); + } + + private static final String UPSERT_DATE = + """ + MERGE INTO nrt_observation_date + USING ( + SELECT + :observation_uid AS observation_uid + :ovd_from_date AS ovd_from_date + :ovd_to_date AS ovd_to_date + :ovd_seq AS ovd_seq + :batch_id AS batch_id + ) AS source + ON nrt_observation_date.observation_uid = source.observation_uid + WHEN MATCHED THEN + UPDATE SET + observation_uid = source.observation_uid + ovd_from_date = source.ovd_from_date + ovd_to_date = source.ovd_to_date + ovd_seq = source.ovd_seq + batch_id = source.batch_id + WHEN NOT MATCHED THEN + INSERT ( + observation_uid, + ovd_from_date, + ovd_to_date, + ovd_seq, + batch_id + ) VALUES ( + source.observation_uid + source.ovd_from_date + source.ovd_to_date + source.ovd_seq + source.batch_id + ); + """; + + private void persistDate(List dateEntries) { + dateEntries.forEach( + d -> + client + .sql(UPSERT_DATE) + .param("observation_uid", d.getObservationUid()) + .param("ovd_from_date", d.getOvdFromDate()) + .param("ovd_to_date", d.getOvdToDate()) + .param("ovd_seq", d.getOvdSeq()) + .param("batch_id", d.getBatchId()) + .update()); + } + + private static final String UPSERT_EDX = + """ + MERGE INTO nrt_observation_edx + USING ( + SELECT + :edx_document_uid AS edx_document_uid + :edx_act_uid AS edx_act_uid + :edx_add_time AS edx_add_time + ) AS source + ON nrt_observation_edx.edx_document_uid = source.edx_document_uid AND nrt_observation_edx.edx_act_uid = source.edx_act_uid + WHEN MATCHED THEN + UPDATE SET + edx_document_uid = source.edx_document_uid + edx_act_uid = source.edx_act_uid + edx_add_time = source.edx_add_time + WHEN NOT MATCHED THEN + INSERT ( + edx_document_uid, + edx_act_uid, + edx_add_time, + batch_id + ) VALUES ( + source.edx_document_uid + source.edx_act_uid + source.edx_add_time + source.batch_id + ); + """; + + private void persistEdx(List edxEntries) { + edxEntries.forEach( + e -> + client + .sql(UPSERT_EDX) + .param("edx_document_uid", e.getEdxDocumentUid()) + .param("edx_act_uid", e.getEdxActUid()) + .param("edx_add_time", e.getEdxAddTime()) + .param("batch_id", e.getBatchId()) + .update()); + } + + private static final String UPSERT_NUMERIC = + """ + MERGE INTO nrt_observation_edx + USING ( + SELECT + :observation_uid AS observation_uid + :ovn_high_range AS ovn_high_range + :ovn_low_range AS ovn_low_range + :ovn_comparator_cd_1 AS ovn_comparator_cd_1 + :ovn_numeric_value_1 AS ovn_numeric_value_1 + :ovn_numeric_value_2 AS ovn_numeric_value_2 + :ovn_numeric_unit_cd AS ovn_numeric_unit_cd + :ovn_separator_cd AS ovn_separator_cd + :ovn_seq AS ovn_seq + :batch_id AS batch_id + ) AS source + ON nrt_observation_edx.observation_uid = source.observation_uid AND nrt_observation_edx.ovn_seq = source.ovn_seq + WHEN MATCHED THEN + UPDATE SET + observation_uid = source.observation_uid + ovn_high_range = source.ovn_high_range + ovn_low_range = source.ovn_low_range + ovn_comparator_cd_1 = source.ovn_comparator_cd_1 + ovn_numeric_value_1 = source.ovn_numeric_value_1 + ovn_numeric_value_2 = source.ovn_numeric_value_2 + ovn_numeric_unit_cd = source.ovn_numeric_unit_cd + ovn_separator_cd = source.ovn_separator_cd + ovn_seq = source.ovn_seq + batch_id = source.batch_id + WHEN NOT MATCHED THEN + INSERT ( + observation_uid, + ovn_high_range, + ovn_low_range, + ovn_comparator_cd_1, + ovn_numeric_value_1, + ovn_numeric_value_2, + ovn_numeric_unit_cd, + ovn_separator_cd, + ovn_seq, + batch_id + ) VALUES ( + source.observation_uid, + source.ovn_high_range, + source.ovn_low_range, + source.ovn_comparator_cd_1, + source.ovn_numeric_value_1, + source.ovn_numeric_value_2, + source.ovn_numeric_unit_cd, + source.ovn_separator_cd, + source.ovn_seq, + source.batch_id + ); + """; + + private void persistNumeric(List numericEntries) { + numericEntries.forEach( + n -> + client + .sql(UPSERT_NUMERIC) + .param("observation_uid", n.getObservationUid()) + .param("ovn_seq", n.getOvnSeq()) + .param("ovn_high_range", n.getOvnHighRange()) + .param("ovn_low_range", n.getOvnLowRange()) + .param("ovn_comparator_cd_1", n.getOvnComparatorCd1()) + .param("ovn_numeric_value_1", n.getOvnNumericValue1()) + .param("ovn_numeric_value_2", n.getOvnNumericValue2()) + .param("ovn_numeric_unit_cd", n.getOvnNumericUnitCd()) + .param("ovn_separator_cd", n.getOvnSeparatorCd()) + .param("batch_id", n.getBatchId()) + .update()); } } diff --git a/reporting-pipeline-service/src/main/java/gov/cdc/nbs/report/pipeline/observation/transformer/ObservationParser.java b/reporting-pipeline-service/src/main/java/gov/cdc/nbs/report/pipeline/observation/transformer/ObservationParser.java index 1009d9d23..5a059ddc9 100644 --- a/reporting-pipeline-service/src/main/java/gov/cdc/nbs/report/pipeline/observation/transformer/ObservationParser.java +++ b/reporting-pipeline-service/src/main/java/gov/cdc/nbs/report/pipeline/observation/transformer/ObservationParser.java @@ -411,6 +411,7 @@ private static void setObservationEdx( JsonNode observationEdxJsonArray = parseJsonArray(observationEdx); for (JsonNode jsonNode : observationEdxJsonArray) { ObservationEdx edx = objectMapper.treeToValue(jsonNode, ObservationEdx.class); + edx.setBatchId(parsedObservation.transformed().getBatchId()); parsedObservation.edxEntries().add(edx); } From ff0f21b652ca3e01b0c120bbbce4804fb96c6de7 Mon Sep 17 00:00:00 2001 From: Michael Peels Date: Fri, 17 Apr 2026 14:26:18 -0400 Subject: [PATCH 03/16] Working tests --- .../observation/NrtObservationWriter.java | 213 ++++++++++---- .../functional/DataDrivenFunctionalTests.java | 8 +- .../ObservationParserTest.java} | 272 +++++------------- 3 files changed, 220 insertions(+), 273 deletions(-) rename reporting-pipeline-service/src/test/java/gov/cdc/nbs/report/pipeline/observation/{ObservationDataProcessTests.java => transformer/ObservationParserTest.java} (55%) diff --git a/reporting-pipeline-service/src/main/java/gov/cdc/nbs/report/pipeline/observation/service/observation/NrtObservationWriter.java b/reporting-pipeline-service/src/main/java/gov/cdc/nbs/report/pipeline/observation/service/observation/NrtObservationWriter.java index 44b6e636d..a6a50f283 100644 --- a/reporting-pipeline-service/src/main/java/gov/cdc/nbs/report/pipeline/observation/service/observation/NrtObservationWriter.java +++ b/reporting-pipeline-service/src/main/java/gov/cdc/nbs/report/pipeline/observation/service/observation/NrtObservationWriter.java @@ -5,6 +5,8 @@ import gov.cdc.nbs.report.pipeline.observation.model.dto.observation.ObservationEdx; import gov.cdc.nbs.report.pipeline.observation.model.dto.observation.ObservationMaterial; import gov.cdc.nbs.report.pipeline.observation.model.dto.observation.ObservationNumeric; +import gov.cdc.nbs.report.pipeline.observation.model.dto.observation.ObservationReason; +import gov.cdc.nbs.report.pipeline.observation.model.dto.observation.ObservationTxt; import gov.cdc.nbs.report.pipeline.observation.model.dto.observation.ParsedObservation; import java.util.List; import org.springframework.jdbc.core.simple.JdbcClient; @@ -14,10 +16,10 @@ * Responsible for writing Observation data to the following tables: * *
    + *
  • nrt_observation_material *
  • nrt_observation_coded *
  • nrt_observation_date *
  • nrt_observation_edx - *
  • nrt_observation_material *
  • nrt_observation_numeric *
  • nrt_observation_reason *
  • nrt_observation_txt @@ -38,8 +40,8 @@ public void persist(ParsedObservation parsedObservation) { persistDate(parsedObservation.dateEntries()); persistEdx(parsedObservation.edxEntries()); persistNumeric(parsedObservation.numericEntries()); - // TODO ObservationReason - // TODO ObservationTxt + persistReason(parsedObservation.reasonEntries()); + persistText(parsedObservation.textEntries()); } private static final String UPSERT_MATERIAL = @@ -157,15 +159,15 @@ private void persistMaterials(List materials) { ON nrt_observation_coded.observation_uid = source.observation_uid AND nrt_observation_coded.ovc_code = source.ovc_code WHEN MATCHED THEN UPDATE SET - observation_uid = source.observation_uid - ovc_code = source.ovc_code - ovc_code_system_cd = source.ovc_code_system_cd - ovc_code_system_desc_txt = source.ovc_code_system_desc_txt - ovc_display_name = source.ovc_display_name - ovc_alt_cd = source.ovc_alt_cd - ovc_alt_cd_desc_txt = source.ovc_alt_cd_desc_txt - ovc_alt_cd_system_cd = source.ovc_alt_cd_system_cd - ovc_alt_cd_system_desc_txt = source.ovc_alt_cd_system_desc_txt + observation_uid = source.observation_uid, + ovc_code = source.ovc_code, + ovc_code_system_cd = source.ovc_code_system_cd, + ovc_code_system_desc_txt = source.ovc_code_system_desc_txt, + ovc_display_name = source.ovc_display_name, + ovc_alt_cd = source.ovc_alt_cd, + ovc_alt_cd_desc_txt = source.ovc_alt_cd_desc_txt, + ovc_alt_cd_system_cd = source.ovc_alt_cd_system_cd, + ovc_alt_cd_system_desc_txt = source.ovc_alt_cd_system_desc_txt, batch_id = source.batch_id WHEN NOT MATCHED THEN INSERT ( @@ -180,15 +182,15 @@ private void persistMaterials(List materials) { ovc_alt_cd_system_desc_txt, batch_id ) VALUES ( - source.observation_uid - source.ovc_code - source.ovc_code_system_cd - source.ovc_code_system_desc_txt - source.ovc_display_name - source.ovc_alt_cd - source.ovc_alt_cd_desc_txt - source.ovc_alt_cd_system_cd - source.ovc_alt_cd_system_desc_txt + source.observation_uid, + source.ovc_code, + source.ovc_code_system_cd, + source.ovc_code_system_desc_txt, + source.ovc_display_name, + source.ovc_alt_cd, + source.ovc_alt_cd_desc_txt, + source.ovc_alt_cd_system_cd, + source.ovc_alt_cd_system_desc_txt, source.batch_id ); """; @@ -216,19 +218,19 @@ private void persistCoded(List codedEntries) { MERGE INTO nrt_observation_date USING ( SELECT - :observation_uid AS observation_uid - :ovd_from_date AS ovd_from_date - :ovd_to_date AS ovd_to_date - :ovd_seq AS ovd_seq + :observation_uid AS observation_uid, + :ovd_from_date AS ovd_from_date, + :ovd_to_date AS ovd_to_date, + :ovd_seq AS ovd_seq, :batch_id AS batch_id ) AS source ON nrt_observation_date.observation_uid = source.observation_uid WHEN MATCHED THEN UPDATE SET - observation_uid = source.observation_uid - ovd_from_date = source.ovd_from_date - ovd_to_date = source.ovd_to_date - ovd_seq = source.ovd_seq + observation_uid = source.observation_uid, + ovd_from_date = source.ovd_from_date, + ovd_to_date = source.ovd_to_date, + ovd_seq = source.ovd_seq, batch_id = source.batch_id WHEN NOT MATCHED THEN INSERT ( @@ -238,10 +240,10 @@ private void persistCoded(List codedEntries) { ovd_seq, batch_id ) VALUES ( - source.observation_uid - source.ovd_from_date - source.ovd_to_date - source.ovd_seq + source.observation_uid, + source.ovd_from_date, + source.ovd_to_date, + source.ovd_seq, source.batch_id ); """; @@ -264,27 +266,25 @@ private void persistDate(List dateEntries) { MERGE INTO nrt_observation_edx USING ( SELECT - :edx_document_uid AS edx_document_uid - :edx_act_uid AS edx_act_uid + :edx_document_uid AS edx_document_uid, + :edx_act_uid AS edx_act_uid, :edx_add_time AS edx_add_time ) AS source ON nrt_observation_edx.edx_document_uid = source.edx_document_uid AND nrt_observation_edx.edx_act_uid = source.edx_act_uid WHEN MATCHED THEN UPDATE SET - edx_document_uid = source.edx_document_uid - edx_act_uid = source.edx_act_uid + edx_document_uid = source.edx_document_uid, + edx_act_uid = source.edx_act_uid, edx_add_time = source.edx_add_time WHEN NOT MATCHED THEN INSERT ( edx_document_uid, edx_act_uid, - edx_add_time, - batch_id + edx_add_time ) VALUES ( - source.edx_document_uid - source.edx_act_uid + source.edx_document_uid, + source.edx_act_uid, source.edx_add_time - source.batch_id ); """; @@ -302,32 +302,32 @@ private void persistEdx(List edxEntries) { private static final String UPSERT_NUMERIC = """ - MERGE INTO nrt_observation_edx + MERGE INTO nrt_observation_edx USING ( SELECT - :observation_uid AS observation_uid - :ovn_high_range AS ovn_high_range - :ovn_low_range AS ovn_low_range - :ovn_comparator_cd_1 AS ovn_comparator_cd_1 - :ovn_numeric_value_1 AS ovn_numeric_value_1 - :ovn_numeric_value_2 AS ovn_numeric_value_2 - :ovn_numeric_unit_cd AS ovn_numeric_unit_cd - :ovn_separator_cd AS ovn_separator_cd - :ovn_seq AS ovn_seq + :observation_uid AS observation_uid, + :ovn_high_range AS ovn_high_range, + :ovn_low_range AS ovn_low_range, + :ovn_comparator_cd_1 AS ovn_comparator_cd_1, + :ovn_numeric_value_1 AS ovn_numeric_value_1, + :ovn_numeric_value_2 AS ovn_numeric_value_2, + :ovn_numeric_unit_cd AS ovn_numeric_unit_cd, + :ovn_separator_cd AS ovn_separator_cd, + :ovn_seq AS ovn_seq, :batch_id AS batch_id ) AS source ON nrt_observation_edx.observation_uid = source.observation_uid AND nrt_observation_edx.ovn_seq = source.ovn_seq WHEN MATCHED THEN UPDATE SET - observation_uid = source.observation_uid - ovn_high_range = source.ovn_high_range - ovn_low_range = source.ovn_low_range - ovn_comparator_cd_1 = source.ovn_comparator_cd_1 - ovn_numeric_value_1 = source.ovn_numeric_value_1 - ovn_numeric_value_2 = source.ovn_numeric_value_2 - ovn_numeric_unit_cd = source.ovn_numeric_unit_cd - ovn_separator_cd = source.ovn_separator_cd - ovn_seq = source.ovn_seq + observation_uid = source.observation_uid, + ovn_high_range = source.ovn_high_range, + ovn_low_range = source.ovn_low_range, + ovn_comparator_cd_1 = source.ovn_comparator_cd_1, + ovn_numeric_value_1 = source.ovn_numeric_value_1, + ovn_numeric_value_2 = source.ovn_numeric_value_2, + ovn_numeric_unit_cd = source.ovn_numeric_unit_cd, + ovn_separator_cd = source.ovn_separator_cd, + ovn_seq = source.ovn_seq, batch_id = source.batch_id WHEN NOT MATCHED THEN INSERT ( @@ -372,4 +372,95 @@ private void persistNumeric(List numericEntries) { .param("batch_id", n.getBatchId()) .update()); } + + private static final String UPSERT_REASON = + """ + MERGE INTO nrt_observation_reason + USING ( + SELECT + :observation_uid as observation_uid, + :reason_cd as reason_cd, + :reason_desc_txt as reason_desc_txt, + :batch_id as batch_id + ) AS source + ON nrt_observation_reason.observation_uid = source.observation_uid AND nrt_observation_reason.reason_cd = source.reason_cd + WHEN MATCHED THEN + UPDATE SET + observation_uid = source.observation_uid, + reason_cd = source.reason_cd, + reason_desc_txt = source.reason_desc_txt, + batch_id = source.batch_id + WHEN NOT MATCHED THEN + INSERT ( + observation_uid, + reason_cd, + reason_desc_txt, + batch_id + ) VALUES ( + source.observation_uid, + source.reason_cd, + source.reason_desc_txt, + source.batch_id + ); + """; + + private void persistReason(List reasonEntries) { + reasonEntries.forEach( + r -> + client + .sql(UPSERT_REASON) + .param("observation_uid", r.getObservationUid()) + .param("reason_cd", r.getReasonCd()) + .param("reason_desc_txt", r.getReasonDescTxt()) + .param("batch_id", r.getBatchId()) + .update()); + } + + private static final String UPSERT_TEXT = + """ + MERGE INTO nrt_observation_txt + USING ( + SELECT + :observation_uid AS observation_uid, + :ovt_seq AS ovt_seq, + :ovt_txt_type_cd AS ovt_txt_type_cd, + :ovt_value_txt AS ovt_value_txt, + :batch_id AS batch_id + ) AS source + ON nrt_observation_txt.observation_uid = source.observation_uid AND nrt_observation_txt.ovt_seq = source.ovt_seq + WHEN MATCHED THEN + UPDATE SET + observation_uid = source.observation_uid, + ovt_seq = source.ovt_seq, + ovt_txt_type_cd = source.ovt_txt_type_cd, + ovt_value_txt = source.ovt_value_txt, + batch_id = source.batch_id + WHEN NOT MATCHED THEN + INSERT ( + observation_uid, + ovt_seq, + ovt_txt_type_cd, + ovt_value_txt, + batch_id + ) VALUES ( + source.observation_uid, + source.ovt_seq, + source.ovt_txt_type_cd, + source.ovt_value_txt, + source.batch_id + ); + """; + + private void persistText(List textEntries) { + textEntries.forEach( + t -> + client + .sql(UPSERT_TEXT) + .param("observation_uid", t.getObservationUid()) + .param("ovt_seq", t.getOvtSeq()) + .param("ovt_txt_type_cd", t.getOvtTxtTypeCd()) + .param("ovt_value_txt", t.getOvtValueTxt()) + .param("batch_id", t.getBatchId()) + .update()); + } } diff --git a/reporting-pipeline-service/src/test/java/gov/cdc/nbs/report/pipeline/integration/functional/DataDrivenFunctionalTests.java b/reporting-pipeline-service/src/test/java/gov/cdc/nbs/report/pipeline/integration/functional/DataDrivenFunctionalTests.java index e50c01094..fd64757b8 100644 --- a/reporting-pipeline-service/src/test/java/gov/cdc/nbs/report/pipeline/integration/functional/DataDrivenFunctionalTests.java +++ b/reporting-pipeline-service/src/test/java/gov/cdc/nbs/report/pipeline/integration/functional/DataDrivenFunctionalTests.java @@ -22,7 +22,6 @@ import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.MethodSource; import org.skyscreamer.jsonassert.JSONAssert; -import org.skyscreamer.jsonassert.JSONCompare; import org.skyscreamer.jsonassert.JSONCompareMode; import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.jdbc.core.simple.JdbcClient; @@ -76,11 +75,10 @@ static Stream functionalTestDirectoryProvider() throws IOException { * @param testDirectory * @throws IOException If step folder is missing a file, an exception will be thrown. * @throws JSONException - * @throws InterruptedException */ @ParameterizedTest @MethodSource("functionalTestDirectoryProvider") - void testRunner(Path testDirectory) throws IOException, JSONException, InterruptedException { + void testRunner(Path testDirectory) throws IOException, JSONException { System.out.println( "Executing DataDrivenFunctionalTest for directory: " + testDirectory.getFileName()); @@ -116,10 +114,6 @@ void testRunner(Path testDirectory) throws IOException, JSONException, Interrupt assertThat(results).isPresent(); String actual = mapper.writeValueAsString(results.get()); - // TEMP!! - if (JSONCompare.compareJSON(expectedResult, actual, JSONCompareMode.LENIENT).failed()) { - System.out.println("THINGS FAILED! EXAMINE!!!"); - } JSONAssert.assertEquals(expectedResult, actual, JSONCompareMode.LENIENT); } } diff --git a/reporting-pipeline-service/src/test/java/gov/cdc/nbs/report/pipeline/observation/ObservationDataProcessTests.java b/reporting-pipeline-service/src/test/java/gov/cdc/nbs/report/pipeline/observation/transformer/ObservationParserTest.java similarity index 55% rename from reporting-pipeline-service/src/test/java/gov/cdc/nbs/report/pipeline/observation/ObservationDataProcessTests.java rename to reporting-pipeline-service/src/test/java/gov/cdc/nbs/report/pipeline/observation/transformer/ObservationParserTest.java index ec9a6f265..e94868432 100644 --- a/reporting-pipeline-service/src/test/java/gov/cdc/nbs/report/pipeline/observation/ObservationDataProcessTests.java +++ b/reporting-pipeline-service/src/test/java/gov/cdc/nbs/report/pipeline/observation/transformer/ObservationParserTest.java @@ -1,21 +1,24 @@ -package gov.cdc.nbs.report.pipeline.observation; +package gov.cdc.nbs.report.pipeline.observation.transformer; import static gov.cdc.etldatapipeline.commonutil.TestUtils.readFileData; -import static org.junit.jupiter.api.Assertions.*; -import static org.mockito.ArgumentMatchers.anyString; -import static org.mockito.Mockito.*; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertTrue; import ch.qos.logback.classic.Logger; import ch.qos.logback.classic.spi.ILoggingEvent; import ch.qos.logback.core.read.ListAppender; -import com.fasterxml.jackson.core.JsonProcessingException; -import com.fasterxml.jackson.databind.ObjectMapper; -import gov.cdc.nbs.report.pipeline.observation.model.dto.observation.*; import gov.cdc.nbs.report.pipeline.observation.model.dto.observation.Observation; +import gov.cdc.nbs.report.pipeline.observation.model.dto.observation.ObservationCoded; +import gov.cdc.nbs.report.pipeline.observation.model.dto.observation.ObservationDate; +import gov.cdc.nbs.report.pipeline.observation.model.dto.observation.ObservationEdx; +import gov.cdc.nbs.report.pipeline.observation.model.dto.observation.ObservationMaterial; +import gov.cdc.nbs.report.pipeline.observation.model.dto.observation.ObservationNumeric; +import gov.cdc.nbs.report.pipeline.observation.model.dto.observation.ObservationReason; import gov.cdc.nbs.report.pipeline.observation.model.dto.observation.ObservationTransformed; -import gov.cdc.nbs.report.pipeline.observation.transformer.ProcessObservationDataUtil; +import gov.cdc.nbs.report.pipeline.observation.model.dto.observation.ObservationTxt; +import gov.cdc.nbs.report.pipeline.observation.model.dto.observation.ParsedObservation; import java.util.List; -import java.util.concurrent.CompletableFuture; import org.jetbrains.annotations.NotNull; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.Assertions; @@ -23,68 +26,26 @@ import org.junit.jupiter.api.Test; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.CsvSource; -import org.mockito.ArgumentCaptor; -import org.mockito.Captor; -import org.mockito.Mock; -import org.mockito.MockitoAnnotations; import org.slf4j.LoggerFactory; -import org.springframework.kafka.core.KafkaTemplate; import org.testcontainers.shaded.org.checkerframework.checker.nullness.qual.NonNull; -class ObservationDataProcessTests { - @Mock KafkaTemplate kafkaTemplate; - - @Captor private ArgumentCaptor topicCaptor; - - @Captor private ArgumentCaptor keyCaptor; - - @Captor private ArgumentCaptor messageCaptor; +class ObservationParserTest { private static final String FILE_PREFIX = "rawDataFiles/observation/"; - private static final String CODED_TOPIC = "codedTopic"; - private static final String DATE_TOPIC = "dateTopic"; - private static final String EDX_TOPIC = "edxTopic"; - private static final String MATERIAL_TOPIC = "materialTopic"; - private static final String NUMERIC_TOPIC = "numericTopic"; - private static final String REASON_TOPIC = "reasonTopic"; - private static final String TXT_TOPIC = "txtTopic"; - private static final Long BATCH_ID = 11L; - - ProcessObservationDataUtil transformer; - - private AutoCloseable closeable; private final ListAppender listAppender = new ListAppender<>(); - private final ObjectMapper objectMapper = new ObjectMapper(); @BeforeEach void setUp() { - closeable = MockitoAnnotations.openMocks(this); - transformer = new ProcessObservationDataUtil(kafkaTemplate); - - transformer.setCodedTopicName(CODED_TOPIC); - transformer.setEdxTopicName(EDX_TOPIC); - transformer.setDateTopicName(DATE_TOPIC); - transformer.setMaterialTopicName(MATERIAL_TOPIC); - transformer.setNumericTopicName(NUMERIC_TOPIC); - transformer.setReasonTopicName(REASON_TOPIC); - transformer.setTxtTopicName(TXT_TOPIC); - - Logger logger = (Logger) LoggerFactory.getLogger(ProcessObservationDataUtil.class); + Logger logger = (Logger) LoggerFactory.getLogger(ObservationParser.class); listAppender.start(); logger.addAppender(listAppender); - - when(kafkaTemplate.send(anyString(), anyString(), isNull())) - .thenReturn(CompletableFuture.completedFuture(null)); - when(kafkaTemplate.send(anyString(), anyString(), anyString())) - .thenReturn(CompletableFuture.completedFuture(null)); } @AfterEach - void tearDown() throws Exception { - Logger logger = (Logger) LoggerFactory.getLogger(ProcessObservationDataUtil.class); + void tearDown() { + Logger logger = (Logger) LoggerFactory.getLogger(ObservationParser.class); logger.detachAppender(listAppender); - closeable.close(); } @Test @@ -100,16 +61,15 @@ void consolidatedDataTransformationTest() { readFileData(FILE_PREFIX + "MaterialParticipations.json")); observation.setFollowupObservations(readFileData(FILE_PREFIX + "FollowupObservations.json")); - ObservationTransformed observationTransformed = - transformer.transformObservationData(observation, BATCH_ID); + ParsedObservation parsedObservation = ObservationParser.parse(observation, BATCH_ID); - Long patId = observationTransformed.getPatientId(); - String ordererId = observationTransformed.getOrderingPersonId(); - Long authorOrgId = observationTransformed.getAuthorOrganizationId(); - Long ordererOrgId = observationTransformed.getOrderingOrganizationId(); - Long performerOrgId = observationTransformed.getPerformingOrganizationId(); - Long materialId = observationTransformed.getMaterialId(); - String resultObsUid = observationTransformed.getResultObservationUid(); + Long patId = parsedObservation.transformed().getPatientId(); + String ordererId = parsedObservation.transformed().getOrderingPersonId(); + Long authorOrgId = parsedObservation.transformed().getAuthorOrganizationId(); + Long ordererOrgId = parsedObservation.transformed().getOrderingOrganizationId(); + Long performerOrgId = parsedObservation.transformed().getPerformingOrganizationId(); + Long materialId = parsedObservation.transformed().getMaterialId(); + String resultObsUid = parsedObservation.transformed().getResultObservationUid(); Assertions.assertEquals("10000055", ordererId); Assertions.assertEquals(10000066L, patId); @@ -129,9 +89,9 @@ void testPersonParticipationTransformation() { final var expected = getObservationTransformed(); observation.setPersonParticipations(readFileData(FILE_PREFIX + "PersonParticipations.json")); - ObservationTransformed observationTransformed = - transformer.transformObservationData(observation, BATCH_ID); - Assertions.assertEquals(expected, observationTransformed); + + ParsedObservation parsedObservation = ObservationParser.parse(observation, BATCH_ID); + Assertions.assertEquals(expected, parsedObservation.transformed()); } @Test @@ -151,9 +111,8 @@ void testMorbReportTransformation() { observation.setPersonParticipations( readFileData(FILE_PREFIX + "PersonParticipationsMorb.json")); - ObservationTransformed observationTransformed = - transformer.transformObservationData(observation, BATCH_ID); - Assertions.assertEquals(expected, observationTransformed); + ParsedObservation parsedObservation = ObservationParser.parse(observation, BATCH_ID); + Assertions.assertEquals(expected, parsedObservation.transformed()); } @Test @@ -165,11 +124,10 @@ void testOrganizationParticipationTransformation() { observation.setOrganizationParticipations( readFileData(FILE_PREFIX + "OrganizationParticipations.json")); - ObservationTransformed observationTransformed = - transformer.transformObservationData(observation, BATCH_ID); - Long authorOrgId = observationTransformed.getAuthorOrganizationId(); - Long ordererOrgId = observationTransformed.getOrderingOrganizationId(); - Long performerOrgId = observationTransformed.getPerformingOrganizationId(); + ParsedObservation parsedObservation = ObservationParser.parse(observation, BATCH_ID); + Long authorOrgId = parsedObservation.transformed().getAuthorOrganizationId(); + Long ordererOrgId = parsedObservation.transformed().getOrderingOrganizationId(); + Long performerOrgId = parsedObservation.transformed().getPerformingOrganizationId(); Assertions.assertNull(authorOrgId); Assertions.assertNull(ordererOrgId); @@ -177,7 +135,7 @@ void testOrganizationParticipationTransformation() { } @Test - void testObservationMaterialTransformation() throws JsonProcessingException { + void testObservationMaterialTransformation() { Observation observation = new Observation(); observation.setObservationUid(100000003L); observation.setObsDomainCdSt1("Order"); @@ -185,27 +143,11 @@ void testObservationMaterialTransformation() throws JsonProcessingException { readFileData(FILE_PREFIX + "MaterialParticipations.json")); ObservationMaterial material = constructObservationMaterial(100000003L); - ObservationTransformed observationTransformed = - transformer.transformObservationData(observation, BATCH_ID); - verify(kafkaTemplate).send(topicCaptor.capture(), keyCaptor.capture(), messageCaptor.capture()); - assertEquals(MATERIAL_TOPIC, topicCaptor.getValue()); - assertEquals(10000005L, observationTransformed.getMaterialId()); + ParsedObservation parsedObservation = ObservationParser.parse(observation, BATCH_ID); - List logs = listAppender.list; - assertTrue( - logs.get(2) - .getFormattedMessage() - .contains("Observation Material data (uid=10000005) sent to " + MATERIAL_TOPIC)); - - var actualMaterial = - objectMapper.readValue( - objectMapper - .readTree(messageCaptor.getAllValues().getFirst()) - .path("payload") - .toString(), - ObservationMaterial.class); - - assertEquals(material, actualMaterial); + assertEquals(10000005L, parsedObservation.transformed().getMaterialId()); + + assertEquals(material, parsedObservation.materialEntries().get(0)); } @ParameterizedTest @@ -217,15 +159,14 @@ void testParentObservationsTransformation(String domainCd) { "[{\"parent_type_cd\":\"MorbFrmQ\",\"parent_uid\":234567888,\"parent_domain_cd_st_1\":\"R_Order\"}]"); observation.setObsDomainCdSt1(domainCd); - ObservationTransformed observationTransformed = - transformer.transformObservationData(observation, BATCH_ID); - assertEquals(234567888L, observationTransformed.getReportObservationUid()); - assertNull(observationTransformed.getReportRefrUid()); - assertNull(observationTransformed.getReportSprtUid()); + ParsedObservation parsedObservation = ObservationParser.parse(observation, BATCH_ID); + assertEquals(234567888L, parsedObservation.transformed().getReportObservationUid()); + assertNull(parsedObservation.transformed().getReportRefrUid()); + assertNull(parsedObservation.transformed().getReportSprtUid()); } @Test - void testObservationCodedTransformation() throws JsonProcessingException { + void testObservationCodedTransformation() { Observation observation = new Observation(); observation.setObservationUid(10001234L); observation.setObsCode(readFileData(FILE_PREFIX + "ObservationCoded.json")); @@ -240,25 +181,13 @@ void testObservationCodedTransformation() throws JsonProcessingException { coded.setOvcAltCdDescTxt("NORMAL"); coded.setBatchId(BATCH_ID); - transformer.transformObservationData(observation, BATCH_ID); - verify(kafkaTemplate).send(topicCaptor.capture(), keyCaptor.capture(), messageCaptor.capture()); - assertEquals(CODED_TOPIC, topicCaptor.getValue()); - List logs = listAppender.list; - assertTrue( - logs.get(6) - .getFormattedMessage() - .contains("Observation Coded data (uid=10001234) sent to " + CODED_TOPIC)); - - var actualCoded = - objectMapper.readValue( - objectMapper.readTree(messageCaptor.getValue()).path("payload").toString(), - ObservationCoded.class); + ParsedObservation parsedObservation = ObservationParser.parse(observation, BATCH_ID); - assertEquals(coded, actualCoded); + assertEquals(coded, parsedObservation.codedEntries().get(0)); } @Test - void testObservationDateTransformation() throws JsonProcessingException { + void testObservationDateTransformation() { Observation observation = new Observation(); observation.setObservationUid(10001234L); observation.setObsDate(readFileData(FILE_PREFIX + "ObservationDate.json")); @@ -269,25 +198,13 @@ void testObservationDateTransformation() throws JsonProcessingException { obd.setOvdSeq(1); obd.setBatchId(BATCH_ID); - transformer.transformObservationData(observation, BATCH_ID); - verify(kafkaTemplate).send(topicCaptor.capture(), keyCaptor.capture(), messageCaptor.capture()); - assertEquals(DATE_TOPIC, topicCaptor.getValue()); - List logs = listAppender.list; - assertTrue( - logs.get(7) - .getFormattedMessage() - .contains("Observation Date data (uid=10001234) sent to " + DATE_TOPIC)); - - var actualObd = - objectMapper.readValue( - objectMapper.readTree(messageCaptor.getValue()).path("payload").toString(), - ObservationDate.class); + ParsedObservation parsedObservation = ObservationParser.parse(observation, BATCH_ID); - assertEquals(obd, actualObd); + assertEquals(obd, parsedObservation.dateEntries().get(0)); } @Test - void testObservationEdxTransformation() throws JsonProcessingException { + void testObservationEdxTransformation() { Observation observation = new Observation(); observation.setActUid(10001234L); observation.setObservationUid(10001234L); @@ -297,30 +214,15 @@ void testObservationEdxTransformation() throws JsonProcessingException { edx.setEdxDocumentUid(10101L); edx.setEdxActUid(observation.getActUid()); edx.setEdxAddTime("2024-09-30T21:08:19.017"); + edx.setBatchId(BATCH_ID); - transformer.transformObservationData(observation, BATCH_ID); - verify(kafkaTemplate, times(2)) - .send(topicCaptor.capture(), keyCaptor.capture(), messageCaptor.capture()); - assertEquals(EDX_TOPIC, topicCaptor.getValue()); - List logs = listAppender.list; - assertTrue( - logs.get(8) - .getFormattedMessage() - .contains("Observation Edx data (edx doc uid=10101) sent to " + EDX_TOPIC)); - - var actualEdx = - objectMapper.readValue( - objectMapper - .readTree(messageCaptor.getAllValues().getFirst()) - .path("payload") - .toString(), - ObservationEdx.class); - - assertEquals(edx, actualEdx); + ParsedObservation parsedObservation = ObservationParser.parse(observation, BATCH_ID); + + assertEquals(edx, parsedObservation.edxEntries().get(0)); } @Test - void testObservationNumericTransformation() throws JsonProcessingException { + void testObservationNumericTransformation() { Observation observation = new Observation(); observation.setObservationUid(10001234L); observation.setObsNum(readFileData(FILE_PREFIX + "ObservationNumeric.json")); @@ -337,25 +239,13 @@ void testObservationNumericTransformation() throws JsonProcessingException { numeric.setOvnSeq(1); numeric.setBatchId(BATCH_ID); - transformer.transformObservationData(observation, BATCH_ID); - verify(kafkaTemplate).send(topicCaptor.capture(), keyCaptor.capture(), messageCaptor.capture()); - assertEquals(NUMERIC_TOPIC, topicCaptor.getValue()); - List logs = listAppender.list; - assertTrue( - logs.get(9) - .getFormattedMessage() - .contains("Observation Numeric data (uid=10001234) sent to " + NUMERIC_TOPIC)); + ParsedObservation parsedObservation = ObservationParser.parse(observation, BATCH_ID); - var actualNumeric = - objectMapper.readValue( - objectMapper.readTree(messageCaptor.getValue()).path("payload").toString(), - ObservationNumeric.class); - - assertEquals(numeric, actualNumeric); + assertEquals(numeric, parsedObservation.numericEntries().get(0)); } @Test - void testObservationReasonTransformation() throws JsonProcessingException { + void testObservationReasonTransformation() { Observation observation = new Observation(); observation.setObservationUid(10001234L); observation.setObsReason(readFileData(FILE_PREFIX + "ObservationReason.json")); @@ -366,25 +256,13 @@ void testObservationReasonTransformation() throws JsonProcessingException { reason.setReasonDescTxt("PRESENCE OF REASON"); reason.setBatchId(BATCH_ID); - transformer.transformObservationData(observation, BATCH_ID); - verify(kafkaTemplate).send(topicCaptor.capture(), keyCaptor.capture(), messageCaptor.capture()); - assertEquals(REASON_TOPIC, topicCaptor.getValue()); - List logs = listAppender.list; - assertTrue( - logs.get(10) - .getFormattedMessage() - .contains("Observation Reason data (uid=10001234) sent to " + REASON_TOPIC)); - - var actualReason = - objectMapper.readValue( - objectMapper.readTree(messageCaptor.getValue()).path("payload").toString(), - ObservationReason.class); + ParsedObservation parsedObservation = ObservationParser.parse(observation, BATCH_ID); - assertEquals(reason, actualReason); + assertEquals(reason, parsedObservation.reasonEntries().get(0)); } @Test - void testObservationTxtTransformation() throws JsonProcessingException { + void testObservationTxtTransformation() { Observation observation = new Observation(); observation.setObservationUid(10001234L); observation.setObsTxt(readFileData(FILE_PREFIX + "ObservationTxt.json")); @@ -396,25 +274,9 @@ void testObservationTxtTransformation() throws JsonProcessingException { txt.setOvtValueTxt("RECOMMENDED IN SUCH INSTANCES."); txt.setBatchId(BATCH_ID); - transformer.transformObservationData(observation, BATCH_ID); - verify(kafkaTemplate, times(2)) - .send(topicCaptor.capture(), keyCaptor.capture(), messageCaptor.capture()); - assertEquals(TXT_TOPIC, topicCaptor.getValue()); - List logs = listAppender.list; - assertTrue( - logs.get(11) - .getFormattedMessage() - .contains("Observation Txt data (uid=10001234) sent to " + TXT_TOPIC)); - - var actualTxt = - objectMapper.readValue( - objectMapper - .readTree(messageCaptor.getAllValues().getFirst()) - .path("payload") - .toString(), - ObservationTxt.class); - - assertEquals(txt, actualTxt); + ParsedObservation parsedObservation = ObservationParser.parse(observation, BATCH_ID); + + assertEquals(txt, parsedObservation.textEntries().get(0)); } @Test @@ -422,7 +284,7 @@ void testTransformNoObservationData() { Observation observation = new Observation(); observation.setObservationUid(10001234L); observation.setOrganizationParticipations("{\"act_uid\": 10000003}"); - transformer.transformObservationData(observation, BATCH_ID); + ObservationParser.parse(observation, BATCH_ID); List logs = listAppender.list; logs.forEach(le -> assertTrue(le.getFormattedMessage().matches("^\\w+ array is null."))); @@ -447,7 +309,7 @@ void testTransformObservationDataError() { observation.setObsReason(invalidJSON); observation.setObsTxt(invalidJSON); - transformer.transformObservationData(observation, BATCH_ID); + ObservationParser.parse(observation, BATCH_ID); List logs = listAppender.list; logs.forEach(le -> assertTrue(le.getFormattedMessage().contains(invalidJSON))); @@ -467,7 +329,7 @@ void testTransformObservationInvalidDomainError() { observation.setMaterialParticipations(dummyJSON); observation.setFollowupObservations(dummyJSON); - transformer.transformObservationData(observation, BATCH_ID); + ObservationParser.parse(observation, BATCH_ID); List logs = listAppender.list.subList(0, 4); logs.forEach( @@ -491,7 +353,7 @@ void testTransformObservationNullError(String payload) { observation.setFollowupObservations(payload); observation.setParentObservations(payload); - transformer.transformObservationData(observation, BATCH_ID); + ObservationParser.parse(observation, BATCH_ID); List logs = listAppender.list.subList(0, 4); logs.forEach( From a2eaa1fc90c2d6c7d3dcc3ae0c702038e057cff3 Mon Sep 17 00:00:00 2001 From: Michael Peels Date: Fri, 17 Apr 2026 16:44:25 -0400 Subject: [PATCH 04/16] Testing --- .../service/ObservationService.java | 20 +- .../observation/NrtObservationWriter.java | 2 +- .../observation/ObservationProcessor.java | 54 +-- .../service/ObservationServiceTest.java | 392 +++++------------- .../observation/ObservationProcessorTest.java | 165 ++++++++ 5 files changed, 308 insertions(+), 325 deletions(-) create mode 100644 reporting-pipeline-service/src/test/java/gov/cdc/nbs/report/pipeline/observation/service/observation/ObservationProcessorTest.java diff --git a/reporting-pipeline-service/src/main/java/gov/cdc/nbs/report/pipeline/observation/service/ObservationService.java b/reporting-pipeline-service/src/main/java/gov/cdc/nbs/report/pipeline/observation/service/ObservationService.java index 273dcb540..c0b3262c4 100644 --- a/reporting-pipeline-service/src/main/java/gov/cdc/nbs/report/pipeline/observation/service/ObservationService.java +++ b/reporting-pipeline-service/src/main/java/gov/cdc/nbs/report/pipeline/observation/service/ObservationService.java @@ -5,6 +5,7 @@ import static gov.cdc.etldatapipeline.commonutil.UtilHelper.extractUid; import static gov.cdc.etldatapipeline.commonutil.UtilHelper.extractValue; +import com.fasterxml.jackson.core.JsonProcessingException; import gov.cdc.etldatapipeline.commonutil.DataProcessingException; import gov.cdc.etldatapipeline.commonutil.NoDataException; import gov.cdc.nbs.report.pipeline.observation.service.observation.ObservationProcessor; @@ -101,11 +102,9 @@ public CompletableFuture processMessage(ConsumerRecord rec logger.debug(TOPIC_DEBUG_LOG, message, topic); if (topic.equals(observationTopic)) { - return CompletableFuture.runAsync( - () -> observationProcessor.process(message, batchId, true, ""), obsExecutor); + return CompletableFuture.runAsync(() -> handleObservation(message, batchId), obsExecutor); } else if (topic.equals(actRelationshipTopic) && message != null) { - return CompletableFuture.runAsync( - () -> processActRelationship(message, batchId), obsExecutor); + return CompletableFuture.runAsync(() -> handleActRelationship(message, batchId), obsExecutor); } else { return CompletableFuture.failedFuture( new DataProcessingException( @@ -113,7 +112,16 @@ public CompletableFuture processMessage(ConsumerRecord rec } } - private void processActRelationship(String value, long batchId) { + private void handleObservation(String message, long batchId) { + try { + String observationUid = extractUid(message, "observation_uid"); + observationProcessor.process(batchId, observationUid); + } catch (JsonProcessingException e) { + throw new DataProcessingException(errorMessage("Observation", "", e), e); + } + } + + private void handleActRelationship(String value, long batchId) { String sourceActUid = ""; try { @@ -136,7 +144,7 @@ private void processActRelationship(String value, long batchId) { // receive // an update in Observation if (typeCd.equals("LabReport") && targetClassCd.equals("OBS")) { - observationProcessor.process(value, batchId, false, sourceActUid); + observationProcessor.process(batchId, sourceActUid); } } catch (Exception e) { throw new DataProcessingException(errorMessage("ActRelationship", sourceActUid, e), e); diff --git a/reporting-pipeline-service/src/main/java/gov/cdc/nbs/report/pipeline/observation/service/observation/NrtObservationWriter.java b/reporting-pipeline-service/src/main/java/gov/cdc/nbs/report/pipeline/observation/service/observation/NrtObservationWriter.java index a6a50f283..16b7e6135 100644 --- a/reporting-pipeline-service/src/main/java/gov/cdc/nbs/report/pipeline/observation/service/observation/NrtObservationWriter.java +++ b/reporting-pipeline-service/src/main/java/gov/cdc/nbs/report/pipeline/observation/service/observation/NrtObservationWriter.java @@ -13,7 +13,7 @@ import org.springframework.stereotype.Component; /** - * Responsible for writing Observation data to the following tables: + * Responsible for upserting Observation data to the following tables: * *
      *
    • nrt_observation_material diff --git a/reporting-pipeline-service/src/main/java/gov/cdc/nbs/report/pipeline/observation/service/observation/ObservationProcessor.java b/reporting-pipeline-service/src/main/java/gov/cdc/nbs/report/pipeline/observation/service/observation/ObservationProcessor.java index cff0e4716..ebe50a448 100644 --- a/reporting-pipeline-service/src/main/java/gov/cdc/nbs/report/pipeline/observation/service/observation/ObservationProcessor.java +++ b/reporting-pipeline-service/src/main/java/gov/cdc/nbs/report/pipeline/observation/service/observation/ObservationProcessor.java @@ -1,7 +1,6 @@ package gov.cdc.nbs.report.pipeline.observation.service.observation; import static gov.cdc.etldatapipeline.commonutil.UtilHelper.errorMessage; -import static gov.cdc.etldatapipeline.commonutil.UtilHelper.extractUid; import gov.cdc.etldatapipeline.commonutil.DataProcessingException; import gov.cdc.etldatapipeline.commonutil.NoDataException; @@ -33,13 +32,13 @@ public class ObservationProcessor { private final KafkaTemplate kafkaTemplate; private final NrtObservationWriter nrtWriter; - private final String observationTopicOutputReporting; - private final String observationTopic; + private final String nrtObservationTopic; private final ModelMapper modelMapper = new ModelMapper(); private final CustomJsonGeneratorImpl jsonGenerator = new CustomJsonGeneratorImpl(); private final CustomMetrics metrics; + private static final String[] TAGS = {"service", "observation-reporting"}; private Counter msgProcessed; private Counter msgSuccess; private Counter msgFailure; @@ -48,47 +47,27 @@ public ObservationProcessor( final CustomMetrics metrics, final ObservationRepository observationRepository, @Qualifier("observationKafkaTemplate") final KafkaTemplate kafkaTemplate, - @Value("${spring.kafka.topics.nrt.observation}") final String observationTopicOutputReporting, - @Value("${spring.kafka.topics.nbs.observation}") final String observationTopic, + @Value("${spring.kafka.topics.nrt.observation}") final String nrtObservationTopic, final NrtObservationWriter nrtWriter) { this.metrics = metrics; this.observationRepository = observationRepository; this.kafkaTemplate = kafkaTemplate; - this.observationTopicOutputReporting = observationTopicOutputReporting; - this.observationTopic = observationTopic; + this.nrtObservationTopic = nrtObservationTopic; this.nrtWriter = nrtWriter; - String[] tags = {"service", "observation-reporting"}; - - msgProcessed = metrics.counter("obs_msg_processed", tags); - msgSuccess = metrics.counter("obs_msg_success", tags); - msgFailure = metrics.counter("obs_msg_failure", tags); + msgProcessed = metrics.counter("obs_msg_processed", TAGS); + msgSuccess = metrics.counter("obs_msg_success", TAGS); + msgFailure = metrics.counter("obs_msg_failure", TAGS); } - public void process( - String value, - long batchId, - boolean isFromObservationTopic, - String actRelationshipSourceActUid) { + public void process(final long batchId, final String observationUid) { msgProcessed.increment(); metrics.recordTime( "obs_msg_processing_seconds", () -> { - String observationUid = ""; try { - // Get the relevant observation_uid - observationUid = - isFromObservationTopic - ? extractUid(value, "observation_uid") - : actRelationshipSourceActUid; - - ObservationKey observationKey = new ObservationKey(Long.valueOf(observationUid)); - - logger.info( - "Received Observation with id: {} from topic: {}", - observationUid, - observationTopic); + logger.info("Received Observation with id: {}", observationUid); // Query NBS_ODSE for observation data Optional observationData = @@ -110,16 +89,14 @@ public void process( // Push parsed fields into reporting object modelMapper.map(parsed.transformed(), reportingModel); - // Insert parsed data into nrt_ database + // Insert parsed data into nrt_observation_* database tables nrtWriter.persist(parsed); - // Send to reporting object to nrt_observation kafka topic - pushKeyValuePairToKafka( - observationKey, reportingModel, observationTopicOutputReporting); + // Send reporting object to nrt_observation kafka topic + ObservationKey observationKey = new ObservationKey(Long.valueOf(observationUid)); + pushKeyValuePairToKafka(observationKey, reportingModel, nrtObservationTopic); logger.info( - "Observation data (uid={}) sent to {}", - observationUid, - observationTopicOutputReporting); + "Observation data (uid={}) sent to {}", observationUid, nrtObservationTopic); msgSuccess.increment(); @@ -132,8 +109,7 @@ public void process( throw new DataProcessingException(errorMessage("Observation", observationUid, e), e); } }, - "service", - "observation-reporting"); + TAGS); } private void pushKeyValuePairToKafka( diff --git a/reporting-pipeline-service/src/test/java/gov/cdc/nbs/report/pipeline/observation/service/ObservationServiceTest.java b/reporting-pipeline-service/src/test/java/gov/cdc/nbs/report/pipeline/observation/service/ObservationServiceTest.java index f66b8ee7e..3ef69c478 100644 --- a/reporting-pipeline-service/src/test/java/gov/cdc/nbs/report/pipeline/observation/service/ObservationServiceTest.java +++ b/reporting-pipeline-service/src/test/java/gov/cdc/nbs/report/pipeline/observation/service/ObservationServiceTest.java @@ -1,315 +1,149 @@ package gov.cdc.nbs.report.pipeline.observation.service; -import static gov.cdc.etldatapipeline.commonutil.TestUtils.readFileData; -import static gov.cdc.nbs.report.pipeline.observation.service.ObservationService.toBatchId; -import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.assertj.core.api.Assertions.assertThat; import static org.junit.jupiter.api.Assertions.assertThrows; -import static org.mockito.ArgumentMatchers.anyString; -import static org.mockito.ArgumentMatchers.isNull; import static org.mockito.Mockito.*; -import com.fasterxml.jackson.core.JsonProcessingException; -import com.fasterxml.jackson.databind.ObjectMapper; -import gov.cdc.etldatapipeline.commonutil.NoDataException; -import gov.cdc.etldatapipeline.commonutil.metrics.CustomMetrics; -import gov.cdc.nbs.report.pipeline.observation.model.dto.observation.Observation; -import gov.cdc.nbs.report.pipeline.observation.model.dto.observation.ObservationKey; -import gov.cdc.nbs.report.pipeline.observation.model.dto.observation.ObservationReporting; -import gov.cdc.nbs.report.pipeline.observation.repository.ObservationRepository; -import gov.cdc.nbs.report.pipeline.observation.service.observation.NrtObservationWriter; import gov.cdc.nbs.report.pipeline.observation.service.observation.ObservationProcessor; -import gov.cdc.nbs.report.pipeline.observation.transformer.ProcessObservationDataUtil; -import io.micrometer.core.instrument.simple.SimpleMeterRegistry; -import java.util.NoSuchElementException; -import java.util.Optional; import java.util.concurrent.CompletableFuture; import java.util.concurrent.CompletionException; -import java.util.concurrent.TimeUnit; import org.apache.kafka.clients.consumer.ConsumerRecord; -import org.awaitility.Awaitility; -import org.junit.jupiter.api.AfterEach; -import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; -import org.junit.jupiter.params.ParameterizedTest; -import org.junit.jupiter.params.provider.CsvSource; import org.mockito.*; -import org.springframework.kafka.core.KafkaTemplate; class ObservationServiceTest { - private ObservationService observationService; + private final String observationTopic = "Observation"; + private final String actRelationshipTopic = "Act_relationship"; - @Mock private ObservationRepository observationRepository; - - @Mock private KafkaTemplate kafkaTemplate; - - @Mock private NrtObservationWriter nrtWrtier; - - @Captor private ArgumentCaptor topicCaptor; - - @Captor private ArgumentCaptor keyCaptor; - - @Captor private ArgumentCaptor messageCaptor; - - private final ObjectMapper objectMapper = new ObjectMapper(); - private AutoCloseable closeable; - - private final String inputTopicNameObservation = "Observation"; - private final String outputTopicNameObservation = "ObservationOutput"; - - private final String inputTopicNameActRelationship = "Act_relationship"; - - @BeforeEach - void setUp() { - closeable = MockitoAnnotations.openMocks(this); - ProcessObservationDataUtil transformer = new ProcessObservationDataUtil(kafkaTemplate); - transformer.setMaterialTopicName("materialTopic"); - observationService = - new ObservationService( - new ObservationProcessor( - new CustomMetrics(new SimpleMeterRegistry()), - observationRepository, - kafkaTemplate, - outputTopicNameObservation, - inputTopicNameObservation, - nrtWrtier), - inputTopicNameObservation, - inputTopicNameActRelationship, - 1); - - transformer.setCodedTopicName("ObservationCoded"); - transformer.setReasonTopicName("ObservationReason"); - transformer.setTxtTopicName("ObservationTxt"); - } - - @AfterEach - void closeService() throws Exception { - closeable.close(); - } + @Mock ObservationProcessor processor = Mockito.mock(ObservationProcessor.class); + private ObservationService service = + new ObservationService(processor, observationTopic, actRelationshipTopic, 1); @Test - void testProcessMessage() throws JsonProcessingException { - // Mocked input data - Long observationUid = 123456789L; - String obsDomainCdSt = "Order"; - String payload = - "{\"payload\": {\"after\": {\"observation_uid\": \"" + observationUid + "\"}}}"; - - Observation observation = constructObservation(observationUid, obsDomainCdSt); - when(observationRepository.computeObservations(String.valueOf(observationUid))) - .thenReturn(Optional.of(observation)); - when(kafkaTemplate.send(anyString(), anyString(), anyString())) - .thenReturn(CompletableFuture.completedFuture(null)); - when(kafkaTemplate.send(anyString(), anyString(), isNull())) - .thenReturn(CompletableFuture.completedFuture(null)); - - validateData(payload, observation, inputTopicNameObservation); - - verify(observationRepository).computeObservations(String.valueOf(observationUid)); + void processesObservationMessage() { // observation_uid + String message = + """ + { + "payload" : { + "op": "c", + "after": { + "observation_uid": "123" + } + } + } + """; + ConsumerRecord consumerRecord = + new ConsumerRecord<>(observationTopic, 0, 1l, null, message); + // receives valid observation message + service.processMessage(consumerRecord).join(); + + // sends to ObservationProcessor + verify(processor, times(1)).process(0, "123"); } - @ParameterizedTest - @CsvSource({ - "d,LabReport,OBS", - "d,LabReport,OTHER", - "c,LabReport,OBS", - "c,LabReport,OTHER", - "d,OTHER,OBS", - "d,OTHER,OTHER" - }) - void testProcessActRelationship(String op, String typeCd, String targetClassCd) - throws JsonProcessingException { - Long sourceActUid = 123456789L; - String obsDomainCdSt = "Order"; - String payload = - "{\"payload\": {\"before\": {\"source_act_uid\": \"" - + sourceActUid - + "\", \"type_cd\": \"" - + typeCd - + "\", \"target_class_cd\": \"" - + targetClassCd - + "\"}," - + "\"after\": {\"source_act_uid\": \"123\"}," - + "\"op\": \"" - + op - + "\"}}"; - - if (typeCd.equals("OTHER") || !op.equals("d") || targetClassCd.equals("OTHER")) { - ConsumerRecord rec = getRecord(payload, inputTopicNameActRelationship); - - observationService.processMessage(rec); - verify(kafkaTemplate, never()).send(anyString(), anyString(), anyString()); - } else { - Observation observation = constructObservation(sourceActUid, obsDomainCdSt); - when(observationRepository.computeObservations(String.valueOf(sourceActUid))) - .thenReturn(Optional.of(observation)); - when(kafkaTemplate.send(anyString(), anyString(), anyString())) - .thenReturn(CompletableFuture.completedFuture(null)); - when(kafkaTemplate.send(anyString(), anyString(), isNull())) - .thenReturn(CompletableFuture.completedFuture(null)); - - validateData(payload, observation, inputTopicNameActRelationship); - - verify(observationRepository).computeObservations(String.valueOf(sourceActUid)); - } + @Test + void processesActRelationshipMessage() { + String message = + """ + { + "payload" : { + "op": "d", + "before": { + "source_act_uid": "1", + "type_cd": "LabReport", + "target_class_cd": "OBS" + } + } + } + """; + ConsumerRecord consumerRecord = + new ConsumerRecord<>(actRelationshipTopic, 0, 1l, null, message); + // receives valid act_relationship message + service.processMessage(consumerRecord).join(); + + // sends to ObservationProcessor + verify(processor, times(1)).process(0, "1"); } @Test - void testProcessActRelationshipNullPayload() { - ConsumerRecord rec = getRecord(null, inputTopicNameActRelationship); - - observationService.processMessage(rec); - - verify(kafkaTemplate, never()).send(anyString(), anyString(), anyString()); + void doesNotProcessActRelationshipMessageBadOp() { + String message = + """ + { + "payload" : { + "op": "c", + "before": { + "source_act_uid": "1", + "type_cd": "LabReport", + "target_class_cd": "OBS" + } + } + } + """; + ConsumerRecord consumerRecord = + new ConsumerRecord<>(actRelationshipTopic, 0, 1l, null, message); + // receives non 'delete' act_relationship message + service.processMessage(consumerRecord).join(); + + // does not send to ObservationProcessor + verifyNoInteractions(processor); } @Test - void testProcessMessageUnknownTopic() { - ConsumerRecord rec = getRecord(null, "dummyTopicName"); - - observationService.processMessage(rec); - - verify(kafkaTemplate, never()).send(anyString(), anyString(), anyString()); + void doesNotProcessActRelationshipMessageBadTypeCd() { + String message = + """ + { + "payload" : { + "op": "d", + "before": { + "source_act_uid": "1", + "type_cd": "BadValue", + "target_class_cd": "OBS" + } + } + } + """; + ConsumerRecord consumerRecord = + new ConsumerRecord<>(actRelationshipTopic, 0, 1l, null, message); + // receives act_relationship message with a type_cd other than 'LabReport' + service.processMessage(consumerRecord).join(); + + // does not send to ObservationProcessor + verifyNoInteractions(processor); } - @ParameterizedTest - @CsvSource({ - "{\"payload\": {\"after\": {}}},Error", - "{\"payload\": {\"after\": {}}},Observation", - "{\"payload\": {\"after\": {\"source_act_uid\": \"123\"}, \"op\": \"d\"}}}, Act_relationship" - }) - void testProcessMessageException(String payload, String topic) { - - ConsumerRecord rec = getRecord(payload, topic); + @Test + void throwsExceptionForBadTopic() { + ConsumerRecord consumerRecord = + new ConsumerRecord<>("bad_topic", 0, 1l, null, ""); + CompletableFuture future = service.processMessage(consumerRecord); - CompletableFuture future = observationService.processMessage(rec); CompletionException ex = assertThrows(CompletionException.class, future::join); - assertEquals(NoSuchElementException.class, ex.getCause().getCause().getClass()); + assertThat(ex.getCause().getMessage()) + .isEqualTo("Received data from an unknown topic: bad_topic"); } @Test - void testProcessMessageNoDataException() { - Long observationUid = 123456789L; - String payload = - "{\"payload\": {\"after\": {\"observation_uid\": \"" + observationUid + "\"}}}"; - ConsumerRecord rec = getRecord(payload, inputTopicNameObservation); + void throwsExceptionForBadActRelationshipMessage() { + String message = + """ + { + "payload" : { + "op": "d", + "before": { + } + } + } + """; + ConsumerRecord consumerRecord = + new ConsumerRecord<>(actRelationshipTopic, 0, 1l, null, message); + CompletableFuture future = service.processMessage(consumerRecord); - when(observationRepository.computeObservations(String.valueOf(observationUid))) - .thenReturn(Optional.empty()); - CompletableFuture future = observationService.processMessage(rec); CompletionException ex = assertThrows(CompletionException.class, future::join); - assertEquals(NoDataException.class, ex.getCause().getClass()); - } - - private void validateData(String payload, Observation observation, String inputTopic) - throws JsonProcessingException { - ConsumerRecord rec = getRecord(payload, inputTopic); - observationService.processMessage(rec); - - ObservationKey observationKey = new ObservationKey(observation.getObservationUid()); - - var reportingModel = - constructObservationReporting( - observation.getObservationUid(), observation.getObsDomainCdSt1()); - reportingModel.setBatchId(toBatchId.applyAsLong(rec)); - - Awaitility.await() - .atMost(1, TimeUnit.HOURS) - .untilAsserted( - () -> - verify(kafkaTemplate, times(1)) - .send(topicCaptor.capture(), keyCaptor.capture(), messageCaptor.capture())); - String actualTopic = topicCaptor.getValue(); - String actualKey = keyCaptor.getValue(); - String actualValue = messageCaptor.getValue(); - - var actualReporting = - objectMapper.readValue( - objectMapper.readTree(actualValue).path("payload").toString(), - ObservationReporting.class); - - var actualObservationKey = - objectMapper.readValue( - objectMapper.readTree(actualKey).path("payload").toString(), ObservationKey.class); - - assertEquals(outputTopicNameObservation, actualTopic); - assertEquals(observationKey, actualObservationKey); - assertEquals(reportingModel, actualReporting); - } - - private Observation constructObservation(Long observationUid, String obsDomainCdSt1) { - String filePathPrefix = "rawDataFiles/observation/"; - Observation observation = new Observation(); - observation.setObservationUid(observationUid); - observation.setActUid(observationUid); - observation.setClassCd("OBS"); - observation.setMoodCd("ENV"); - observation.setLocalId("OBS10003388MA01"); - observation.setActivityFromTime("2021-01-28 16:06:03.000"); - observation.setObsDomainCdSt1(obsDomainCdSt1); - observation.setPersonParticipations(readFileData(filePathPrefix + "PersonParticipations.json")); - observation.setOrganizationParticipations( - readFileData(filePathPrefix + "OrganizationParticipations.json")); - observation.setMaterialParticipations( - readFileData(filePathPrefix + "MaterialParticipations.json")); - observation.setFollowupObservations(readFileData(filePathPrefix + "FollowupObservations.json")); - observation.setParentObservations(readFileData(filePathPrefix + "ParentObservations.json")); - observation.setActIds(readFileData(filePathPrefix + "ActIds.json")); - return observation; - } - - private ObservationReporting constructObservationReporting( - Long observationUid, String obsDomainCdSt1) { - ObservationReporting observation = new ObservationReporting(); - observation.setObservationUid(observationUid); - observation.setObsDomainCdSt1(obsDomainCdSt1); - observation.setActUid(observationUid); - observation.setClassCd("OBS"); - observation.setMoodCd("ENV"); - observation.setLocalId("OBS10003388MA01"); - observation.setOrderingPersonId("10000055"); - observation.setPatientId(10000066L); - observation.setPerformingOrganizationId(null); // not null when obsDomainCdSt1=Result - observation.setAuthorOrganizationId(34567890L); // null when obsDomainCdSt1=Result - observation.setOrderingOrganizationId(23456789L); // null when obsDomainCdSt1=Result - observation.setHealthCareId(56789012L); // null when obsDomainCdSt1=Result - observation.setMorbHospReporterId(67890123L); // null when obsDomainCdSt1=Result - observation.setMorbHospId(78901234L); // null when obsDomainCdSt1=Result - observation.setMaterialId(10000005L); - observation.setResultObservationUid("56789012,56789013"); - observation.setFollowupObservationUid("56789014,56789015"); - observation.setReportObservationUid(123456788L); - observation.setReportRefrUid(123456790L); - observation.setReportSprtUid(123456788L); - - observation.setAssistantInterpreterId(10000077L); - observation.setAssistantInterpreterVal("22582"); - observation.setAssistantInterpreterFirstNm("Cara"); - observation.setAssistantInterpreterLastNm("Dune"); - observation.setAssistantInterpreterIdAssignAuth("22D7377772"); - observation.setAssistantInterpreterAuthType("Employee number"); - - observation.setTranscriptionistId(10000088L); - observation.setTranscriptionistVal("34344355455144"); - observation.setTranscriptionistFirstNm("Moff"); - observation.setTranscriptionistLastNm("Gideon"); - observation.setTranscriptionistIdAssignAuth("18D8181818"); - observation.setTranscriptionistAuthType("Employee number"); - - observation.setResultInterpreterId(10000022L); - observation.setLabTestTechnicianId(10000011L); - - observation.setSpecimenCollectorId(10000033L); - observation.setCopyToProviderId(10000044L); - observation.setAccessionNumber("20120601114"); - observation.setActivityFromTime("2021-01-28 16:06:03.000"); - observation.setDeviceInstanceId1("No Equipment"); - observation.setDeviceInstanceId2("NEW TOOLS"); - - return observation; - } - - private ConsumerRecord getRecord(String payload, String inputTopic) { - return new ConsumerRecord<>(inputTopic, 0, 11L, null, payload); + assertThat(ex.getCause().getMessage()) + .isEqualTo( + "Error processing ActRelationship data: The source_act_uid field is missing in the message payload."); } } diff --git a/reporting-pipeline-service/src/test/java/gov/cdc/nbs/report/pipeline/observation/service/observation/ObservationProcessorTest.java b/reporting-pipeline-service/src/test/java/gov/cdc/nbs/report/pipeline/observation/service/observation/ObservationProcessorTest.java new file mode 100644 index 000000000..78ac06243 --- /dev/null +++ b/reporting-pipeline-service/src/test/java/gov/cdc/nbs/report/pipeline/observation/service/observation/ObservationProcessorTest.java @@ -0,0 +1,165 @@ +package gov.cdc.nbs.report.pipeline.observation.service.observation; + +import static gov.cdc.etldatapipeline.commonutil.TestUtils.readFileData; +import static org.assertj.core.api.Assertions.assertThat; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.ObjectMapper; +import gov.cdc.etldatapipeline.commonutil.NoDataException; +import gov.cdc.etldatapipeline.commonutil.metrics.CustomMetrics; +import gov.cdc.nbs.report.pipeline.observation.model.dto.observation.Observation; +import gov.cdc.nbs.report.pipeline.observation.model.dto.observation.ObservationKey; +import gov.cdc.nbs.report.pipeline.observation.model.dto.observation.ObservationReporting; +import gov.cdc.nbs.report.pipeline.observation.repository.ObservationRepository; +import io.micrometer.core.instrument.simple.SimpleMeterRegistry; +import java.util.Optional; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.ArgumentCaptor; +import org.mockito.Captor; +import org.mockito.Mock; +import org.mockito.Spy; +import org.mockito.junit.jupiter.MockitoExtension; +import org.springframework.kafka.core.KafkaTemplate; + +@ExtendWith(MockitoExtension.class) +class ObservationProcessorTest { + + @Spy private CustomMetrics metrics = new CustomMetrics(new SimpleMeterRegistry()); + @Mock private ObservationRepository repository; + @Mock private NrtObservationWriter writer; + @Mock private KafkaTemplate kafkaTemplate; + @Captor private ArgumentCaptor keyCaptor; + @Captor private ArgumentCaptor messageCaptor; + private final ObjectMapper mapper = new ObjectMapper(); + private final String nrtObservationTopic = "nrt_observation"; + + private ObservationProcessor processor; + + @BeforeEach + void init() { + processor = + new ObservationProcessor(metrics, repository, kafkaTemplate, nrtObservationTopic, writer); + } + + @Test + void successfullyProcessesObservation() throws JsonProcessingException { + // Mock database response + Observation observation = constructObservation(123L, "Order"); + when(repository.computeObservations("123")).thenReturn(Optional.of(observation)); + + // Act + processor.process(1l, "123"); + + // Verify + verify(repository, times(1)).computeObservations("123"); + verify(kafkaTemplate, times(1)) + .send(eq("nrt_observation"), keyCaptor.capture(), messageCaptor.capture()); + + ObservationKey expectedKey = new ObservationKey(observation.getObservationUid()); + ObservationKey actualKey = + mapper.readValue( + mapper.readTree(keyCaptor.getValue()).path("payload").toString(), ObservationKey.class); + assertThat(actualKey).isEqualTo(expectedKey); + + var reportingModel = + constructObservationReporting( + observation.getObservationUid(), observation.getObsDomainCdSt1()); + + var actualReporting = + mapper.readValue( + mapper.readTree(messageCaptor.getValue()).path("payload").toString(), + ObservationReporting.class); + + assertThat(reportingModel).isEqualTo(actualReporting); + } + + @Test + void verifyThrowsExceptionWhenNoDataFound() { + // Mock database response + when(repository.computeObservations("123")).thenReturn(Optional.empty()); + + // Act + Verify + NoDataException ex = assertThrows(NoDataException.class, () -> processor.process(1l, "123")); + assertThat(ex.getMessage()).isEqualTo("Unable to find Observation with id: 123"); + } + + private Observation constructObservation(Long observationUid, String obsDomainCdSt1) { + String filePathPrefix = "rawDataFiles/observation/"; + Observation observation = new Observation(); + observation.setObservationUid(observationUid); + observation.setActUid(observationUid); + observation.setClassCd("OBS"); + observation.setMoodCd("ENV"); + observation.setLocalId("OBS10003388MA01"); + observation.setActivityFromTime("2021-01-28 16:06:03.000"); + observation.setObsDomainCdSt1(obsDomainCdSt1); + observation.setPersonParticipations(readFileData(filePathPrefix + "PersonParticipations.json")); + observation.setOrganizationParticipations( + readFileData(filePathPrefix + "OrganizationParticipations.json")); + observation.setMaterialParticipations( + readFileData(filePathPrefix + "MaterialParticipations.json")); + observation.setFollowupObservations(readFileData(filePathPrefix + "FollowupObservations.json")); + observation.setParentObservations(readFileData(filePathPrefix + "ParentObservations.json")); + observation.setActIds(readFileData(filePathPrefix + "ActIds.json")); + return observation; + } + + private ObservationReporting constructObservationReporting( + Long observationUid, String obsDomainCdSt1) { + ObservationReporting observation = new ObservationReporting(); + observation.setBatchId(1l); + observation.setObservationUid(observationUid); + observation.setObsDomainCdSt1(obsDomainCdSt1); + observation.setActUid(observationUid); + observation.setClassCd("OBS"); + observation.setMoodCd("ENV"); + observation.setLocalId("OBS10003388MA01"); + observation.setOrderingPersonId("10000055"); + observation.setPatientId(10000066L); + observation.setPerformingOrganizationId(null); // not null when obsDomainCdSt1=Result + observation.setAuthorOrganizationId(34567890L); // null when obsDomainCdSt1=Result + observation.setOrderingOrganizationId(23456789L); // null when obsDomainCdSt1=Result + observation.setHealthCareId(56789012L); // null when obsDomainCdSt1=Result + observation.setMorbHospReporterId(67890123L); // null when obsDomainCdSt1=Result + observation.setMorbHospId(78901234L); // null when obsDomainCdSt1=Result + observation.setMaterialId(10000005L); + observation.setResultObservationUid("56789012,56789013"); + observation.setFollowupObservationUid("56789014,56789015"); + observation.setReportObservationUid(123456788L); + observation.setReportRefrUid(123456790L); + observation.setReportSprtUid(123456788L); + + observation.setAssistantInterpreterId(10000077L); + observation.setAssistantInterpreterVal("22582"); + observation.setAssistantInterpreterFirstNm("Cara"); + observation.setAssistantInterpreterLastNm("Dune"); + observation.setAssistantInterpreterIdAssignAuth("22D7377772"); + observation.setAssistantInterpreterAuthType("Employee number"); + + observation.setTranscriptionistId(10000088L); + observation.setTranscriptionistVal("34344355455144"); + observation.setTranscriptionistFirstNm("Moff"); + observation.setTranscriptionistLastNm("Gideon"); + observation.setTranscriptionistIdAssignAuth("18D8181818"); + observation.setTranscriptionistAuthType("Employee number"); + + observation.setResultInterpreterId(10000022L); + observation.setLabTestTechnicianId(10000011L); + + observation.setSpecimenCollectorId(10000033L); + observation.setCopyToProviderId(10000044L); + observation.setAccessionNumber("20120601114"); + observation.setActivityFromTime("2021-01-28 16:06:03.000"); + observation.setDeviceInstanceId1("No Equipment"); + observation.setDeviceInstanceId2("NEW TOOLS"); + + return observation; + } +} From 84ae3713db9df06c4cd8933df87aaa8ba0618623 Mon Sep 17 00:00:00 2001 From: Michael Peels Date: Fri, 17 Apr 2026 17:02:05 -0400 Subject: [PATCH 05/16] Remove old util --- .../ProcessObservationDataUtil.java | 575 ------------------ 1 file changed, 575 deletions(-) delete mode 100644 reporting-pipeline-service/src/main/java/gov/cdc/nbs/report/pipeline/observation/transformer/ProcessObservationDataUtil.java diff --git a/reporting-pipeline-service/src/main/java/gov/cdc/nbs/report/pipeline/observation/transformer/ProcessObservationDataUtil.java b/reporting-pipeline-service/src/main/java/gov/cdc/nbs/report/pipeline/observation/transformer/ProcessObservationDataUtil.java deleted file mode 100644 index 83653c790..000000000 --- a/reporting-pipeline-service/src/main/java/gov/cdc/nbs/report/pipeline/observation/transformer/ProcessObservationDataUtil.java +++ /dev/null @@ -1,575 +0,0 @@ -package gov.cdc.nbs.report.pipeline.observation.transformer; - -import com.fasterxml.jackson.core.JsonProcessingException; -import com.fasterxml.jackson.databind.JsonNode; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule; -import gov.cdc.etldatapipeline.commonutil.json.CustomJsonGeneratorImpl; -import gov.cdc.nbs.report.pipeline.observation.model.dto.observation.*; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.List; -import java.util.Optional; -import java.util.function.Function; -import lombok.RequiredArgsConstructor; -import lombok.Setter; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.springframework.beans.factory.annotation.Qualifier; -import org.springframework.beans.factory.annotation.Value; -import org.springframework.kafka.core.KafkaTemplate; -import org.springframework.stereotype.Component; - -@Component -@RequiredArgsConstructor -@Setter -public class ProcessObservationDataUtil { - private static final Logger logger = LoggerFactory.getLogger(ProcessObservationDataUtil.class); - private static final ObjectMapper objectMapper = - new ObjectMapper().registerModule(new JavaTimeModule()); - - @Qualifier("observationKafkaTemplate") - private final KafkaTemplate kafkaTemplate; - - private final CustomJsonGeneratorImpl jsonGenerator = new CustomJsonGeneratorImpl(); - - @Value("${spring.kafka.topics.nrt.observation-coded}") - public String codedTopicName; - - @Value("${spring.kafka.topics.nrt.observation-date}") - public String dateTopicName; - - @Value("${spring.kafka.topics.nrt.observation-edx}") - public String edxTopicName; - - @Value("${spring.kafka.topics.nrt.observation-material}") - public String materialTopicName; - - @Value("${spring.kafka.topics.nrt.observation-numeric}") - public String numericTopicName; - - @Value("${spring.kafka.topics.nrt.observation-reason}") - public String reasonTopicName; - - @Value("${spring.kafka.topics.nrt.observation-txt}") - public String txtTopicName; - - private static final String SUBJECT_CLASS_CD = "subject_class_cd"; - public static final String TYPE_CD = "type_cd"; - public static final String ENTITY_ID = "entity_id"; - public static final String ORDER = "Order"; - public static final String RESULT = "Result"; - public static final String ACT_ID_SEQ = "act_id_seq"; - public static final String ROOT_EXTENSION_TXT = "root_extension_txt"; - - public ObservationTransformed transformObservationData(Observation observation, long batchId) { - // Convert to new object - ObservationTransformed observationTransformed = new ObservationTransformed(); - observationTransformed.setObservationUid(observation.getObservationUid()); - observationTransformed.setReportObservationUid(observation.getObservationUid()); - observationTransformed.setBatchId(batchId); - - ObservationKey observationKey = new ObservationKey(observation.getObservationUid()); - String obsDomainCdSt1 = observation.getObsDomainCdSt1(); - - transformPersonParticipations( - observation.getPersonParticipations(), obsDomainCdSt1, observationTransformed); - transformOrganizationParticipations( - observation.getOrganizationParticipations(), obsDomainCdSt1, observationTransformed); - transformMaterialParticipations( - observation.getMaterialParticipations(), obsDomainCdSt1, observationTransformed); - transformFollowupObservations( - observation.getFollowupObservations(), obsDomainCdSt1, observationTransformed); - transformParentObservations(observation.getParentObservations(), observationTransformed); - transformActIds(observation.getActIds(), observationTransformed); - transformObservationCoded(observation.getObsCode(), batchId); - transformObservationDate(observationKey, observation.getObsDate(), batchId); - transformObservationEdx(observation.getEdxIds()); - transformObservationNumeric(observationKey, observation.getObsNum(), batchId); - transformObservationReasons(observation.getObsReason(), batchId); - transformObservationTxt(observation.getObsTxt(), batchId); - - return observationTransformed; - } - - private void transformPersonParticipations( - String personParticipations, - String obsDomainCdSt1, - ObservationTransformed observationTransformed) { - try { - JsonNode personParticipationsJsonArray = parseJsonArray(personParticipations); - - List orderers = new ArrayList<>(); - for (JsonNode jsonNode : personParticipationsJsonArray) { - assertDomainCdMatches(obsDomainCdSt1, ORDER, RESULT); - - String typeCd = getNodeValue(jsonNode, TYPE_CD, JsonNode::asText); - Long entityId = getNodeValue(jsonNode, ENTITY_ID, JsonNode::asLong); - - if (typeCd.equals("PATSBJ")) { - transformPersonParticipationRoles(jsonNode, observationTransformed, entityId); - } - - if (ORDER.equals(obsDomainCdSt1)) { - String subjectClassCd = getNodeValue(jsonNode, SUBJECT_CLASS_CD, JsonNode::asText); - if ("PSN".equals(subjectClassCd)) { - switch (typeCd) { - case "ORD": - orderers.add(String.valueOf(entityId)); - break; - case "PATSBJ", "SubjOfMorbReport": - observationTransformed.setPatientId(entityId); - break; - case "PhysicianOfMorb": - observationTransformed.setMorbPhysicianId(entityId); - break; - case "ReporterOfMorbReport": - observationTransformed.setMorbReporterId(entityId); - break; - case "ENT": - observationTransformed.setTranscriptionistId(entityId); - Optional.ofNullable(jsonNode.get("first_nm")) - .filter(n -> !n.isNull()) - .ifPresent(n -> observationTransformed.setTranscriptionistFirstNm(n.asText())); - Optional.ofNullable(jsonNode.get("last_nm")) - .filter(n -> !n.isNull()) - .ifPresent(n -> observationTransformed.setTranscriptionistLastNm(n.asText())); - Optional.ofNullable(jsonNode.get("person_id_val")) - .filter(n -> !n.isNull()) - .ifPresent(n -> observationTransformed.setTranscriptionistVal(n.asText())); - Optional.ofNullable(jsonNode.get("person_id_assign_auth_cd")) - .filter(n -> !n.isNull()) - .ifPresent( - n -> observationTransformed.setTranscriptionistIdAssignAuth(n.asText())); - Optional.ofNullable(jsonNode.get("person_id_type_desc")) - .filter(n -> !n.isNull()) - .ifPresent(n -> observationTransformed.setTranscriptionistAuthType(n.asText())); - break; - case "ASS": - observationTransformed.setAssistantInterpreterId(entityId); - Optional.ofNullable(jsonNode.get("first_nm")) - .filter(n -> !n.isNull()) - .ifPresent( - n -> observationTransformed.setAssistantInterpreterFirstNm(n.asText())); - Optional.ofNullable(jsonNode.get("last_nm")) - .filter(n -> !n.isNull()) - .ifPresent( - n -> observationTransformed.setAssistantInterpreterLastNm(n.asText())); - Optional.ofNullable(jsonNode.get("person_id_val")) - .filter(n -> !n.isNull()) - .ifPresent(n -> observationTransformed.setAssistantInterpreterVal(n.asText())); - Optional.ofNullable(jsonNode.get("person_id_assign_auth_cd")) - .filter(n -> !n.isNull()) - .ifPresent( - n -> - observationTransformed.setAssistantInterpreterIdAssignAuth(n.asText())); - Optional.ofNullable(jsonNode.get("person_id_type_desc")) - .filter(n -> !n.isNull()) - .ifPresent( - n -> observationTransformed.setAssistantInterpreterAuthType(n.asText())); - break; - case "VRF": - observationTransformed.setResultInterpreterId(entityId); - break; - case "PRF": - observationTransformed.setLabTestTechnicianId(entityId); - break; - default: - } - } - } - } - if (!orderers.isEmpty()) { - observationTransformed.setOrderingPersonId(String.join(",", orderers)); - } - } catch (IllegalArgumentException ex) { - logger.info(ex.getMessage(), "PersonParticipations", personParticipations); - } catch (Exception e) { - logger.error( - "Error processing Person Participation JSON array from observation data: {}", - e.getMessage()); - } - } - - private void transformPersonParticipationRoles( - JsonNode node, ObservationTransformed observationTransformed, Long entityId) { - String roleSubject = node.path("role_subject_class_cd").asText(); - if ("PROV".equals(roleSubject)) { - String roleCd = node.path("role_cd").asText(); - if ("SPP".equals(roleCd)) { - String roleScoping = node.path("role_scoping_class_cd").asText(); - if ("PSN".equals(roleScoping)) { - observationTransformed.setSpecimenCollectorId(entityId); - } - } else if ("CT".equals(roleCd)) { - observationTransformed.setCopyToProviderId(entityId); - } - } - } - - private void transformOrganizationParticipations( - String organizationParticipations, - String obsDomainCdSt1, - ObservationTransformed observationTransformed) { - try { - JsonNode organizationParticipationsJsonArray = parseJsonArray(organizationParticipations); - - for (JsonNode jsonNode : organizationParticipationsJsonArray) { - assertDomainCdMatches(obsDomainCdSt1, RESULT, ORDER); - - String typeCd = getNodeValue(jsonNode, TYPE_CD, JsonNode::asText); - String subjectClassCd = getNodeValue(jsonNode, SUBJECT_CLASS_CD, JsonNode::asText); - Long entityId = getNodeValue(jsonNode, ENTITY_ID, JsonNode::asLong); - - if (subjectClassCd.equals("ORG")) { - if (RESULT.equals(obsDomainCdSt1)) { - if ("PRF".equals(typeCd)) { - observationTransformed.setPerformingOrganizationId(entityId); - } - } else if (ORDER.equals(obsDomainCdSt1)) { - switch (typeCd) { - case "AUT": - observationTransformed.setAuthorOrganizationId(entityId); - break; - case "ORD": - observationTransformed.setOrderingOrganizationId(entityId); - break; - case "HCFAC": - observationTransformed.setHealthCareId(entityId); - break; - case "ReporterOfMorbReport": - observationTransformed.setMorbHospReporterId(entityId); - break; - case "HospOfMorbObs": - observationTransformed.setMorbHospId(entityId); - break; - default: - break; - } - } - } - } - } catch (IllegalArgumentException ex) { - logger.info(ex.getMessage(), "OrganizationParticipations", organizationParticipations); - } catch (Exception e) { - logger.error( - "Error processing Organization Participation JSON array from observation data: {}", - e.getMessage()); - } - } - - private void transformMaterialParticipations( - String materialParticipations, - String obsDomainCdSt1, - ObservationTransformed observationTransformed) { - try { - JsonNode materialParticipationsJsonArray = parseJsonArray(materialParticipations); - - for (JsonNode jsonNode : materialParticipationsJsonArray) { - String typeCd = getNodeValue(jsonNode, TYPE_CD, JsonNode::asText); - String subjectClassCd = getNodeValue(jsonNode, SUBJECT_CLASS_CD, JsonNode::asText); - - assertDomainCdMatches(obsDomainCdSt1, ORDER); - if ("SPC".equals(typeCd) && "MAT".equals(subjectClassCd)) { - Long materialId = jsonNode.get(ENTITY_ID).asLong(); - observationTransformed.setMaterialId(materialId); - - ObservationMaterial material = - objectMapper.treeToValue(jsonNode, ObservationMaterial.class); - material.setMaterialId(materialId); - ObservationMaterialKey key = - new ObservationMaterialKey(observationTransformed.getMaterialId()); - sendToKafka( - key, - material, - materialTopicName, - materialId, - "Observation Material data (uid={}) sent to {}"); - } - } - } catch (IllegalArgumentException ex) { - logger.info(ex.getMessage(), "MaterialParticipations", materialParticipations); - } catch (Exception e) { - logger.error( - "Error processing Material Participation JSON array from observation data: {}", - e.getMessage()); - } - } - - private void transformFollowupObservations( - String followupObservations, - String obsDomainCdSt1, - ObservationTransformed observationTransformed) { - try { - JsonNode followupObservationsJsonArray = parseJsonArray(followupObservations); - - List results = new ArrayList<>(); - List followUps = new ArrayList<>(); - for (JsonNode jsonNode : followupObservationsJsonArray) { - Optional domainCd = Optional.ofNullable(jsonNode.get("domain_cd_st_1")); - assertDomainCdMatches(obsDomainCdSt1, ORDER); - - if (domainCd.isPresent() && RESULT.equals(domainCd.get().asText())) { - Optional.ofNullable(jsonNode.get("result_observation_uid")) - .ifPresent(r -> results.add(r.asText())); - } else { - Optional.ofNullable(jsonNode.get("result_observation_uid")) - .ifPresent(r -> followUps.add(r.asText())); - } - } - - if (!results.isEmpty()) { - observationTransformed.setResultObservationUid(String.join(",", results)); - } - if (!followUps.isEmpty()) { - observationTransformed.setFollowUpObservationUid(String.join(",", followUps)); - } - } catch (IllegalArgumentException ex) { - logger.info(ex.getMessage(), "FollowupObservations", followupObservations); - } catch (Exception e) { - logger.error( - "Error processing Followup Observations JSON array from observation data: {}", - e.getMessage()); - } - } - - private void transformParentObservations( - String parentObservations, ObservationTransformed observationTransformed) { - try { - JsonNode parentObservationsJsonArray = parseJsonArray(parentObservations); - - for (JsonNode jsonNode : parentObservationsJsonArray) { - Long parentUid = getNodeValue(jsonNode, "parent_uid", JsonNode::asLong); - String parentTypeCd = jsonNode.path("parent_type_cd").asText(); - String parentDomainCd = jsonNode.path("parent_domain_cd_st_1").asText(); - - if (parentTypeCd.equals("SPRT")) { - observationTransformed.setReportSprtUid(parentUid); - } else if (parentTypeCd.equals("REFR")) { - observationTransformed.setReportRefrUid(parentUid); - } - - if (parentDomainCd.contains(ORDER)) { - observationTransformed.setReportObservationUid(parentUid); - } - } - } catch (IllegalArgumentException ex) { - logger.info(ex.getMessage(), "ParentObservations", parentObservations); - } catch (Exception e) { - logger.error( - "Error processing Parent Observations JSON array from observation data: {}", - e.getMessage()); - } - } - - private void transformActIds(String actIds, ObservationTransformed observationTransformed) { - try { - JsonNode actIdsJsonArray = parseJsonArray(actIds); - - for (JsonNode jsonNode : actIdsJsonArray) { - String typeCd = getNodeValue(jsonNode, TYPE_CD, JsonNode::asText); - Integer actIdSeq = getNodeValue(jsonNode, ACT_ID_SEQ, JsonNode::asInt); - if (typeCd.equals("FN")) { - String rootExtTxt = getNodeValue(jsonNode, ROOT_EXTENSION_TXT, JsonNode::asText); - observationTransformed.setAccessionNumber(rootExtTxt); - } - if (typeCd.equals("EII") && actIdSeq.equals(3)) { - String rootExtTxt = getNodeValue(jsonNode, ROOT_EXTENSION_TXT, JsonNode::asText); - observationTransformed.setDeviceInstanceId1(rootExtTxt); - } - if (typeCd.equals("EII") && actIdSeq.equals(4)) { - String rootExtTxt = getNodeValue(jsonNode, ROOT_EXTENSION_TXT, JsonNode::asText); - observationTransformed.setDeviceInstanceId2(rootExtTxt); - } - } - } catch (IllegalArgumentException ex) { - logger.info(ex.getMessage(), "ActIds", actIds); - } catch (Exception e) { - logger.error("Error processing Act Ids JSON array from observation data: {}", e.getMessage()); - } - } - - private void transformObservationCoded(String observationCoded, long batchId) { - try { - JsonNode observationCodedJsonArray = parseJsonArray(observationCoded); - - for (JsonNode jsonNode : observationCodedJsonArray) { - ObservationCoded coded = objectMapper.treeToValue(jsonNode, ObservationCoded.class); - coded.setBatchId(batchId); - ObservationCodedKey codedKey = - new ObservationCodedKey(coded.getObservationUid(), coded.getOvcCode()); - sendToKafka( - codedKey, - coded, - codedTopicName, - coded.getObservationUid(), - "Observation Coded data (uid={}) sent to {}"); - } - } catch (IllegalArgumentException ex) { - logger.info(ex.getMessage(), "ObservationCoded"); - } catch (Exception e) { - logger.error( - "Error processing Observation Coded JSON array from observation data: {}", - e.getMessage()); - } - } - - private void transformObservationDate( - ObservationKey observationKey, String observationDate, long batchId) { - try { - JsonNode observationDateJsonArray = parseJsonArray(observationDate); - - for (JsonNode jsonNode : observationDateJsonArray) { - ObservationDate obsDate = objectMapper.treeToValue(jsonNode, ObservationDate.class); - obsDate.setBatchId(batchId); - sendToKafka( - observationKey, - obsDate, - dateTopicName, - obsDate.getObservationUid(), - "Observation Date data (uid={}) sent to {}"); - } - } catch (IllegalArgumentException ex) { - logger.info(ex.getMessage(), "ObservationDate"); - } catch (Exception e) { - logger.error( - "Error processing Observation Date JSON array from observation data: {}", e.getMessage()); - } - } - - private void transformObservationEdx(String observationEdx) { - try { - JsonNode observationEdxJsonArray = parseJsonArray(observationEdx); - ObservationEdxKey edxKey = new ObservationEdxKey(); - - for (JsonNode jsonNode : observationEdxJsonArray) { - ObservationEdx edx = objectMapper.treeToValue(jsonNode, ObservationEdx.class); - edxKey.setEdxDocumentUid(edx.getEdxDocumentUid()); - sendToKafka( - edxKey, - edx, - edxTopicName, - edx.getEdxDocumentUid(), - "Observation Edx data (edx doc uid={}) sent to {}"); - } - } catch (IllegalArgumentException ex) { - logger.info(ex.getMessage(), "ObservationEdx"); - } catch (Exception e) { - logger.error( - "Error processing Observation Edx JSON array from observation data: {}", e.getMessage()); - } - } - - private void transformObservationNumeric( - ObservationKey observationKey, String observationNumeric, long batchId) { - try { - JsonNode observationNumericJsonArray = parseJsonArray(observationNumeric); - - for (JsonNode jsonNode : observationNumericJsonArray) { - ObservationNumeric numeric = objectMapper.treeToValue(jsonNode, ObservationNumeric.class); - numeric.setBatchId(batchId); - sendToKafka( - observationKey, - numeric, - numericTopicName, - numeric.getObservationUid(), - "Observation Numeric data (uid={}) sent to {}"); - } - } catch (IllegalArgumentException ex) { - logger.info(ex.getMessage(), "ObservationNumeric"); - } catch (Exception e) { - logger.error( - "Error processing Observation Numeric JSON array from observation data: {}", - e.getMessage()); - } - } - - private void transformObservationReasons(String observationReasons, long batchId) { - try { - JsonNode observationReasonsJsonArray = parseJsonArray(observationReasons); - - ObservationReasonKey reasonKey = new ObservationReasonKey(); - for (JsonNode jsonNode : observationReasonsJsonArray) { - ObservationReason reason = objectMapper.treeToValue(jsonNode, ObservationReason.class); - reason.setBatchId(batchId); - reasonKey.setObservationUid(reason.getObservationUid()); - reasonKey.setReasonCd(reason.getReasonCd()); - sendToKafka( - reasonKey, - reason, - reasonTopicName, - reason.getObservationUid(), - "Observation Reason data (uid={}) sent to {}"); - } - } catch (IllegalArgumentException ex) { - logger.info(ex.getMessage(), "ObservationReasons"); - } catch (Exception e) { - logger.error( - "Error processing Observation Reasons JSON array from observation data: {}", - e.getMessage()); - } - } - - private void transformObservationTxt(String observationTxt, long batchId) { - try { - JsonNode observationTxtJsonArray = parseJsonArray(observationTxt); - - ObservationTxtKey txtKey = new ObservationTxtKey(); - for (JsonNode jsonNode : observationTxtJsonArray) { - ObservationTxt txt = objectMapper.treeToValue(jsonNode, ObservationTxt.class); - txt.setBatchId(batchId); - txtKey.setObservationUid(txt.getObservationUid()); - txtKey.setOvtSeq(txt.getOvtSeq()); - sendToKafka( - txtKey, - txt, - txtTopicName, - txt.getObservationUid(), - "Observation Txt data (uid={}) sent to {}"); - } - } catch (IllegalArgumentException ex) { - logger.info(ex.getMessage(), "ObservationTxt"); - } catch (Exception e) { - logger.error( - "Error processing Observation Txt JSON array from observation data: {}", e.getMessage()); - } - } - - private void sendToKafka(Object key, Object value, String topicName, Long uid, String message) { - String jsonKey = jsonGenerator.generateStringJson(key); - String jsonValue = - Optional.ofNullable(value).map(jsonGenerator::generateStringJson).orElse(null); - kafkaTemplate - .send(topicName, jsonKey, jsonValue) - .whenComplete( - (res, e) -> { - if (message != null) { - logger.info(message, uid, topicName); - } - }); - } - - private JsonNode parseJsonArray(String jsonString) - throws JsonProcessingException, IllegalArgumentException { - JsonNode jsonArray = jsonString != null ? objectMapper.readTree(jsonString) : null; - if (jsonArray != null && jsonArray.isArray()) { - return jsonArray; - } else { - throw new IllegalArgumentException("{} array is null."); - } - } - - private T getNodeValue(JsonNode jsonNode, String fieldName, Function mapper) { - JsonNode node = jsonNode.get(fieldName); - if (node == null || node.isNull()) { - throw new IllegalArgumentException("Field " + fieldName + " is null or not found in {}: {}"); - } - return mapper.apply(node); - } - - private void assertDomainCdMatches(String value, String... vals) { - if (Arrays.stream(vals).noneMatch(value::equals)) { - throw new IllegalArgumentException("obsDomainCdSt1: " + value + " is not valid for the {}"); - } - } -} From 8068015123f4beccbc6d8d5ec192f12b7d8c5158 Mon Sep 17 00:00:00 2001 From: Michael Peels Date: Mon, 20 Apr 2026 10:05:40 -0400 Subject: [PATCH 06/16] Finish tests --- .../model/dto/observation/ObservationEdx.java | 1 - .../observation/NrtObservationWriter.java | 21 +- .../observation/ObservationProcessor.java | 2 +- .../transformer/ObservationParser.java | 1 - .../support/config/DataSourceConfig.java | 2 +- .../observation/NrtObservationWriterTest.java | 603 ++++++++++++++++++ .../transformer/ObservationParserTest.java | 1 - .../src/test/resources/application-test.yaml | 2 +- 8 files changed, 617 insertions(+), 16 deletions(-) create mode 100644 reporting-pipeline-service/src/test/java/gov/cdc/nbs/report/pipeline/observation/service/observation/NrtObservationWriterTest.java diff --git a/reporting-pipeline-service/src/main/java/gov/cdc/nbs/report/pipeline/observation/model/dto/observation/ObservationEdx.java b/reporting-pipeline-service/src/main/java/gov/cdc/nbs/report/pipeline/observation/model/dto/observation/ObservationEdx.java index 2e7bfc37d..0324e8db7 100644 --- a/reporting-pipeline-service/src/main/java/gov/cdc/nbs/report/pipeline/observation/model/dto/observation/ObservationEdx.java +++ b/reporting-pipeline-service/src/main/java/gov/cdc/nbs/report/pipeline/observation/model/dto/observation/ObservationEdx.java @@ -12,5 +12,4 @@ public class ObservationEdx { private Long edxDocumentUid; private Long edxActUid; private String edxAddTime; - private Long batchId; } diff --git a/reporting-pipeline-service/src/main/java/gov/cdc/nbs/report/pipeline/observation/service/observation/NrtObservationWriter.java b/reporting-pipeline-service/src/main/java/gov/cdc/nbs/report/pipeline/observation/service/observation/NrtObservationWriter.java index 16b7e6135..be7234004 100644 --- a/reporting-pipeline-service/src/main/java/gov/cdc/nbs/report/pipeline/observation/service/observation/NrtObservationWriter.java +++ b/reporting-pipeline-service/src/main/java/gov/cdc/nbs/report/pipeline/observation/service/observation/NrtObservationWriter.java @@ -11,6 +11,7 @@ import java.util.List; import org.springframework.jdbc.core.simple.JdbcClient; import org.springframework.stereotype.Component; +import org.springframework.transaction.annotation.Transactional; /** * Responsible for upserting Observation data to the following tables: @@ -26,6 +27,7 @@ *
    */ @Component +@Transactional public class NrtObservationWriter { private final JdbcClient client; @@ -117,7 +119,7 @@ public void persist(ParsedObservation parsedObservation) { ); """; - private void persistMaterials(List materials) { + void persistMaterials(List materials) { materials.forEach( m -> client @@ -195,7 +197,7 @@ private void persistMaterials(List materials) { ); """; - private void persistCoded(List codedEntries) { + void persistCoded(List codedEntries) { codedEntries.forEach( c -> client @@ -248,7 +250,7 @@ private void persistCoded(List codedEntries) { ); """; - private void persistDate(List dateEntries) { + void persistDate(List dateEntries) { dateEntries.forEach( d -> client @@ -288,7 +290,7 @@ private void persistDate(List dateEntries) { ); """; - private void persistEdx(List edxEntries) { + void persistEdx(List edxEntries) { edxEntries.forEach( e -> client @@ -296,13 +298,12 @@ private void persistEdx(List edxEntries) { .param("edx_document_uid", e.getEdxDocumentUid()) .param("edx_act_uid", e.getEdxActUid()) .param("edx_add_time", e.getEdxAddTime()) - .param("batch_id", e.getBatchId()) .update()); } private static final String UPSERT_NUMERIC = """ - MERGE INTO nrt_observation_edx + MERGE INTO nrt_observation_numeric USING ( SELECT :observation_uid AS observation_uid, @@ -316,7 +317,7 @@ private void persistEdx(List edxEntries) { :ovn_seq AS ovn_seq, :batch_id AS batch_id ) AS source - ON nrt_observation_edx.observation_uid = source.observation_uid AND nrt_observation_edx.ovn_seq = source.ovn_seq + ON nrt_observation_numeric.observation_uid = source.observation_uid AND nrt_observation_numeric.ovn_seq = source.ovn_seq WHEN MATCHED THEN UPDATE SET observation_uid = source.observation_uid, @@ -355,7 +356,7 @@ private void persistEdx(List edxEntries) { ); """; - private void persistNumeric(List numericEntries) { + void persistNumeric(List numericEntries) { numericEntries.forEach( n -> client @@ -404,7 +405,7 @@ private void persistNumeric(List numericEntries) { ); """; - private void persistReason(List reasonEntries) { + void persistReason(List reasonEntries) { reasonEntries.forEach( r -> client @@ -451,7 +452,7 @@ private void persistReason(List reasonEntries) { ); """; - private void persistText(List textEntries) { + void persistText(List textEntries) { textEntries.forEach( t -> client diff --git a/reporting-pipeline-service/src/main/java/gov/cdc/nbs/report/pipeline/observation/service/observation/ObservationProcessor.java b/reporting-pipeline-service/src/main/java/gov/cdc/nbs/report/pipeline/observation/service/observation/ObservationProcessor.java index ebe50a448..7c0d80852 100644 --- a/reporting-pipeline-service/src/main/java/gov/cdc/nbs/report/pipeline/observation/service/observation/ObservationProcessor.java +++ b/reporting-pipeline-service/src/main/java/gov/cdc/nbs/report/pipeline/observation/service/observation/ObservationProcessor.java @@ -92,7 +92,7 @@ public void process(final long batchId, final String observationUid) { // Insert parsed data into nrt_observation_* database tables nrtWriter.persist(parsed); - // Send reporting object to nrt_observation kafka topic + // Send observation data to nrt_observation kafka topic ObservationKey observationKey = new ObservationKey(Long.valueOf(observationUid)); pushKeyValuePairToKafka(observationKey, reportingModel, nrtObservationTopic); logger.info( diff --git a/reporting-pipeline-service/src/main/java/gov/cdc/nbs/report/pipeline/observation/transformer/ObservationParser.java b/reporting-pipeline-service/src/main/java/gov/cdc/nbs/report/pipeline/observation/transformer/ObservationParser.java index 5a059ddc9..1009d9d23 100644 --- a/reporting-pipeline-service/src/main/java/gov/cdc/nbs/report/pipeline/observation/transformer/ObservationParser.java +++ b/reporting-pipeline-service/src/main/java/gov/cdc/nbs/report/pipeline/observation/transformer/ObservationParser.java @@ -411,7 +411,6 @@ private static void setObservationEdx( JsonNode observationEdxJsonArray = parseJsonArray(observationEdx); for (JsonNode jsonNode : observationEdxJsonArray) { ObservationEdx edx = objectMapper.treeToValue(jsonNode, ObservationEdx.class); - edx.setBatchId(parsedObservation.transformed().getBatchId()); parsedObservation.edxEntries().add(edx); } diff --git a/reporting-pipeline-service/src/test/java/gov/cdc/nbs/report/pipeline/integration/support/config/DataSourceConfig.java b/reporting-pipeline-service/src/test/java/gov/cdc/nbs/report/pipeline/integration/support/config/DataSourceConfig.java index b2b34176f..4740dd1b9 100644 --- a/reporting-pipeline-service/src/test/java/gov/cdc/nbs/report/pipeline/integration/support/config/DataSourceConfig.java +++ b/reporting-pipeline-service/src/test/java/gov/cdc/nbs/report/pipeline/integration/support/config/DataSourceConfig.java @@ -28,7 +28,7 @@ public DataSource dataSource(DataSourceProperties properties) { } @Primary - @Bean + @Bean("rtrClient") public JdbcClient jdbcClient(DataSource dataSource) { return JdbcClient.create(dataSource); } diff --git a/reporting-pipeline-service/src/test/java/gov/cdc/nbs/report/pipeline/observation/service/observation/NrtObservationWriterTest.java b/reporting-pipeline-service/src/test/java/gov/cdc/nbs/report/pipeline/observation/service/observation/NrtObservationWriterTest.java new file mode 100644 index 000000000..a8e823dea --- /dev/null +++ b/reporting-pipeline-service/src/test/java/gov/cdc/nbs/report/pipeline/observation/service/observation/NrtObservationWriterTest.java @@ -0,0 +1,603 @@ +package gov.cdc.nbs.report.pipeline.observation.service.observation; + +import static org.assertj.core.api.Assertions.assertThat; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.SerializationFeature; +import gov.cdc.nbs.report.pipeline.integration.unit.UnitTest; +import gov.cdc.nbs.report.pipeline.observation.model.dto.observation.ObservationCoded; +import gov.cdc.nbs.report.pipeline.observation.model.dto.observation.ObservationDate; +import gov.cdc.nbs.report.pipeline.observation.model.dto.observation.ObservationEdx; +import gov.cdc.nbs.report.pipeline.observation.model.dto.observation.ObservationMaterial; +import gov.cdc.nbs.report.pipeline.observation.model.dto.observation.ObservationNumeric; +import gov.cdc.nbs.report.pipeline.observation.model.dto.observation.ObservationReason; +import gov.cdc.nbs.report.pipeline.observation.model.dto.observation.ObservationTxt; +import java.text.SimpleDateFormat; +import java.util.List; +import java.util.Map; +import org.json.JSONException; +import org.junit.jupiter.api.Test; +import org.skyscreamer.jsonassert.Customization; +import org.skyscreamer.jsonassert.JSONAssert; +import org.skyscreamer.jsonassert.JSONCompareMode; +import org.skyscreamer.jsonassert.comparator.CustomComparator; +import org.springframework.beans.factory.annotation.Qualifier; +import org.springframework.jdbc.core.simple.JdbcClient; + +class NrtObservationWriterTest extends UnitTest { + + private final JdbcClient client; + private final NrtObservationWriter writer; + private static final ObjectMapper mapper = + new ObjectMapper() + .enable(SerializationFeature.INDENT_OUTPUT) + .disable(SerializationFeature.WRITE_DATES_AS_TIMESTAMPS) + .setDateFormat(new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSS")); + + public NrtObservationWriterTest(@Qualifier("rtrClient") final JdbcClient client) { + this.client = client; + this.writer = new NrtObservationWriter(client); + } + + @Test + void insertsMaterialData() throws JsonProcessingException, JSONException { + // Insert material data + ObservationMaterial material = new ObservationMaterial(); + material.setActUid(2L); + material.setTypeCd("SPC"); + material.setMaterialId(2L); + material.setSubjectClassCd("MAT"); + material.setRecordStatus("ACTIVE"); + material.setTypeDescTxt("Specimen"); + material.setLastChgTime("2024-01-01T00:00:00.000"); + material.setMaterialCd("UNK"); + material.setMaterialNm("name"); + material.setMaterialDetails("Details"); + material.setMaterialCollectionVol("36"); + material.setMaterialCollectionVolUnit("ML"); + material.setMaterialDesc("Lymphocytes"); + material.setRiskCd("rsk"); + material.setRiskDescTxt("rskDesc"); + + writer.persistMaterials(List.of(material)); + + // Verify data is as expected + Map data = + client + .sql("SELECT * FROM nrt_observation_material WHERE act_uid = 2 AND material_id = 2") + .query() + .singleRow(); + + String actual = mapper.writeValueAsString(data); + String expected = mapper.writeValueAsString(material); + JSONAssert.assertEquals( + expected, + actual, + new CustomComparator( + JSONCompareMode.LENIENT, new Customization("refresh_datetime", (a, b) -> true))); + } + + @Test + void updatesMaterialData() throws JsonProcessingException, JSONException { + // Insert material data + ObservationMaterial material = new ObservationMaterial(); + material.setActUid(3L); + material.setTypeCd("SPC"); + material.setMaterialId(3L); + material.setSubjectClassCd("MAT"); + material.setRecordStatus("ACTIVE"); + material.setTypeDescTxt("Specimen"); + material.setLastChgTime("2024-01-01T00:00:00.000"); + material.setMaterialCd("UNK"); + material.setMaterialNm("name"); + material.setMaterialDetails("Details"); + material.setMaterialCollectionVol("36"); + material.setMaterialCollectionVolUnit("ML"); + material.setMaterialDesc("Lymphocytes"); + material.setRiskCd("rsk"); + material.setRiskDescTxt("rskDesc"); + + writer.persistMaterials(List.of(material)); + + // Upsert material data + material.setTypeCd("CPS"); + material.setSubjectClassCd("TAM"); + material.setRecordStatus("INACTIVE"); + material.setTypeDescTxt("NEW"); + material.setLastChgTime("2025-01-01T00:00:00.000"); + material.setMaterialCd("ST"); + material.setMaterialNm("NEW_NAME"); + material.setMaterialDetails("Updated Details"); + material.setMaterialCollectionVol("34"); + material.setMaterialCollectionVolUnit("LM"); + material.setMaterialDesc("Something"); + material.setRiskCd("UpdatedRisk"); + material.setRiskDescTxt("NewRiskDesc"); + writer.persistMaterials(List.of(material)); + + // Verify data is as expected + Integer count = + client + .sql( + "SELECT COUNT(*) FROM nrt_observation_material WHERE act_uid = 3 AND material_id = 3") + .query(Integer.class) + .single(); + + assertThat(count).isEqualTo(1); + + Map data = + client + .sql("SELECT * FROM nrt_observation_material WHERE act_uid = 3 AND material_id = 3") + .query() + .singleRow(); + + String actual = mapper.writeValueAsString(data); + String expected = mapper.writeValueAsString(material); + JSONAssert.assertEquals( + expected, + actual, + new CustomComparator( + JSONCompareMode.LENIENT, new Customization("refresh_datetime", (a, b) -> true))); + } + + @Test + void insertsCodedEntry() throws JSONException, JsonProcessingException { + // Insert coded data + ObservationCoded coded = new ObservationCoded(); + coded.setObservationUid(4L); + coded.setBatchId(0L); + coded.setOvcCode("CE04"); + coded.setOvcCodeSystemCd("SNM"); + coded.setOvcCodeSystemDescTxt("SNOMED"); + coded.setOvcDisplayName("Normal]"); + coded.setOvcAltCd("A-124"); + coded.setOvcAltCdDescTxt("NORMAL"); + + writer.persistCoded(List.of(coded)); + + // Verify data is as expected + Map data = + client + .sql( + "SELECT * FROM nrt_observation_coded WHERE observation_uid = 4 AND ovc_code = 'CE04'") + .query() + .singleRow(); + + String actual = mapper.writeValueAsString(data); + String expected = mapper.writeValueAsString(coded); + JSONAssert.assertEquals( + expected, + actual, + new CustomComparator( + JSONCompareMode.LENIENT, new Customization("refresh_datetime", (a, b) -> true))); + } + + @Test + void updatesCodedEntry() throws JSONException, JsonProcessingException { + // Insert coded data + ObservationCoded coded = new ObservationCoded(); + coded.setObservationUid(5L); + coded.setBatchId(0L); + coded.setOvcCode("CE05"); + coded.setOvcCodeSystemCd("SNM"); + coded.setOvcCodeSystemDescTxt("SNOMED"); + coded.setOvcDisplayName("Normal]"); + coded.setOvcAltCd("A-124"); + coded.setOvcAltCdDescTxt("NORMAL"); + + writer.persistCoded(List.of(coded)); + + // Upsert coded data + coded.setBatchId(1L); + coded.setOvcCodeSystemCd("MNS"); + coded.setOvcCodeSystemDescTxt("LOINC"); + coded.setOvcDisplayName("NotNormal"); + coded.setOvcAltCd("D-444"); + coded.setOvcAltCdDescTxt("ABOVE"); + + writer.persistCoded(List.of(coded)); + + // Verify data is as expected + Integer count = + client + .sql( + "SELECT COUNT(*) FROM nrt_observation_coded WHERE observation_uid = 5 AND ovc_code = 'CE05'") + .query(Integer.class) + .single(); + + assertThat(count).isEqualTo(1); + + Map data = + client + .sql( + "SELECT * FROM nrt_observation_coded WHERE observation_uid = 5 AND ovc_code = 'CE05'") + .query() + .singleRow(); + + String actual = mapper.writeValueAsString(data); + String expected = mapper.writeValueAsString(coded); + JSONAssert.assertEquals( + expected, + actual, + new CustomComparator( + JSONCompareMode.LENIENT, new Customization("refresh_datetime", (a, b) -> true))); + } + + @Test + void insertDateEntry() throws JsonProcessingException, JSONException { + // Insert Date data + ObservationDate date = new ObservationDate(); + date.setObservationUid(6L); + date.setBatchId(0L); + date.setOvdFromDate("2024-08-16T00:00:00.000"); + date.setOvdSeq(1); + + writer.persistDate(List.of(date)); + + // Verify data is as expected + Integer count = + client + .sql("SELECT COUNT(*) FROM nrt_observation_date WHERE observation_uid = 6") + .query(Integer.class) + .single(); + + assertThat(count).isEqualTo(1); + + Map data = + client + .sql("SELECT * FROM nrt_observation_date WHERE observation_uid = 6") + .query() + .singleRow(); + + String actual = mapper.writeValueAsString(data); + String expected = mapper.writeValueAsString(date); + JSONAssert.assertEquals( + expected, + actual, + new CustomComparator( + JSONCompareMode.LENIENT, new Customization("refresh_datetime", (a, b) -> true))); + } + + @Test + void updatesDateEntry() throws JsonProcessingException, JSONException { + // Insert Date data + ObservationDate date = new ObservationDate(); + date.setObservationUid(7L); + date.setBatchId(0L); + date.setOvdFromDate("2024-08-16T00:00:00.000"); + date.setOvdSeq(1); + + writer.persistDate(List.of(date)); + + // Upsert date data + date.setBatchId(2L); + date.setOvdFromDate("2025-08-16T00:00:00.000"); + date.setOvdSeq(2); + + writer.persistDate(List.of(date)); + + // Verify data is as expected + Map data = + client + .sql("SELECT * FROM nrt_observation_date WHERE observation_uid = 7") + .query() + .singleRow(); + + String actual = mapper.writeValueAsString(data); + String expected = mapper.writeValueAsString(date); + JSONAssert.assertEquals( + expected, + actual, + new CustomComparator( + JSONCompareMode.LENIENT, new Customization("refresh_datetime", (a, b) -> true))); + } + + @Test + void insertEdx() throws JsonProcessingException, JSONException { + // Insert edx data + ObservationEdx edx = new ObservationEdx(); + edx.setEdxActUid(8l); + edx.setEdxDocumentUid(9L); + edx.setEdxAddTime("2024-09-30T21:08:19.017"); + + writer.persistEdx(List.of(edx)); + + // Verify data is as expected + Map data = + client + .sql("SELECT * FROM nrt_observation_edx WHERE edx_act_uid = 8 AND edx_document_uid = 9") + .query() + .singleRow(); + + String actual = mapper.writeValueAsString(data); + String expected = mapper.writeValueAsString(edx); + JSONAssert.assertEquals( + expected, + actual, + new CustomComparator( + JSONCompareMode.LENIENT, new Customization("refresh_datetime", (a, b) -> true))); + } + + @Test + void updatesEdx() throws JsonProcessingException, JSONException { + // Insert edx data + ObservationEdx edx = new ObservationEdx(); + edx.setEdxActUid(9l); + edx.setEdxDocumentUid(10L); + edx.setEdxAddTime("2024-09-30T21:08:19.017"); + + writer.persistEdx(List.of(edx)); + + // Upsert edx data + edx.setEdxAddTime("2025-10-02T20:04:09.000"); + + writer.persistEdx(List.of(edx)); + + // Verify data is as expected + Integer count = + client + .sql( + "SELECT COUNT(*) FROM nrt_observation_edx WHERE edx_act_uid = 9 AND edx_document_uid = 10") + .query(Integer.class) + .single(); + + assertThat(count).isEqualTo(1); + + Map data = + client + .sql( + "SELECT * FROM nrt_observation_edx WHERE edx_act_uid = 9 AND edx_document_uid = 10") + .query() + .singleRow(); + + String actual = mapper.writeValueAsString(data); + String expected = mapper.writeValueAsString(edx); + JSONAssert.assertEquals( + expected, + actual, + new CustomComparator( + JSONCompareMode.LENIENT, new Customization("refresh_datetime", (a, b) -> true))); + } + + @Test + void insertNumeric() { + // Insert numeric data + ObservationNumeric numeric = new ObservationNumeric(); + numeric.setObservationUid(10L); + numeric.setOvnSeq(1); + numeric.setBatchId(0l); + numeric.setOvnComparatorCd1("100"); + numeric.setOvnLowRange("10-100"); + numeric.setOvnHighRange("100-1000"); + numeric.setOvnNumericValue1("23.10000"); + numeric.setOvnNumericValue2("1.00000"); + numeric.setOvnNumericUnitCd("mL"); + numeric.setOvnSeparatorCd(":"); + + writer.persistNumeric(List.of(numeric)); + + // Verify data is as expected + Map data = + client + .sql("SELECT * FROM nrt_observation_numeric WHERE observation_uid = 10 AND ovn_seq = 1") + .query() + .singleRow(); + + // Field comparison due to type mismatch from String to numeric(15,5) of numeric value fields + assertThat(data) + .containsEntry("observation_uid", numeric.getObservationUid()) + .containsEntry("ovn_high_range", numeric.getOvnHighRange()) + .containsEntry("ovn_low_range", numeric.getOvnLowRange()) + .containsEntry("ovn_comparator_cd_1", numeric.getOvnComparatorCd1()) + .containsEntry("ovn_numeric_unit_cd", numeric.getOvnNumericUnitCd()) + .containsEntry("ovn_separator_cd", numeric.getOvnSeparatorCd()) + .containsEntry("batch_id", numeric.getBatchId()); + + assertThat(data.get("ovn_seq")).hasToString(numeric.getOvnSeq().toString()); + assertThat(data.get("ovn_numeric_value_1")).hasToString(numeric.getOvnNumericValue1()); + assertThat(data.get("ovn_numeric_value_2")).hasToString(numeric.getOvnNumericValue2()); + } + + @Test + void updatesNumeric() { + // Insert numeric data + ObservationNumeric numeric = new ObservationNumeric(); + numeric.setObservationUid(11L); + numeric.setOvnSeq(2); + numeric.setBatchId(0l); + numeric.setOvnComparatorCd1("100"); + numeric.setOvnLowRange("10-100"); + numeric.setOvnHighRange("100-1000"); + numeric.setOvnNumericValue1("23"); + numeric.setOvnNumericValue2("1.0"); + numeric.setOvnNumericUnitCd("mL"); + numeric.setOvnSeparatorCd(":"); + + writer.persistNumeric(List.of(numeric)); + + // Upsert numeric data + numeric.setBatchId(1l); + numeric.setOvnComparatorCd1("200"); + numeric.setOvnLowRange("20-200"); + numeric.setOvnHighRange("200-2000"); + numeric.setOvnNumericValue1("34.00000"); + numeric.setOvnNumericValue2("2.00000"); + numeric.setOvnNumericUnitCd("LM"); + numeric.setOvnSeparatorCd("!"); + + writer.persistNumeric(List.of(numeric)); + + // Verify data is as expected + Integer count = + client + .sql( + "SELECT COUNT(*) FROM nrt_observation_numeric WHERE observation_uid = 11 AND ovn_seq = 2") + .query(Integer.class) + .single(); + + assertThat(count).isEqualTo(1); + + Map data = + client + .sql("SELECT * FROM nrt_observation_numeric WHERE observation_uid = 11 AND ovn_seq = 2") + .query() + .singleRow(); + + assertThat(data) + .containsEntry("observation_uid", numeric.getObservationUid()) + .containsEntry("ovn_high_range", numeric.getOvnHighRange()) + .containsEntry("ovn_low_range", numeric.getOvnLowRange()) + .containsEntry("ovn_comparator_cd_1", numeric.getOvnComparatorCd1()) + .containsEntry("ovn_numeric_unit_cd", numeric.getOvnNumericUnitCd()) + .containsEntry("ovn_separator_cd", numeric.getOvnSeparatorCd()) + .containsEntry("batch_id", numeric.getBatchId()); + + assertThat(data.get("ovn_seq")).hasToString(numeric.getOvnSeq().toString()); + assertThat(data.get("ovn_numeric_value_1")).hasToString(numeric.getOvnNumericValue1()); + assertThat(data.get("ovn_numeric_value_2")).hasToString(numeric.getOvnNumericValue2()); + } + + @Test + void insertsReason() throws JsonProcessingException, JSONException { + // Insert reason data + ObservationReason reason = new ObservationReason(); + reason.setObservationUid(12l); + reason.setReasonCd("80008"); + reason.setReasonDescTxt("PRESENCE OF REASON"); + reason.setBatchId(12l); + + writer.persistReason(List.of(reason)); + + // Verify data is as expected + Map data = + client + .sql( + "SELECT * FROM nrt_observation_reason WHERE observation_uid = 12 AND reason_cd = '80008'") + .query() + .singleRow(); + + String actual = mapper.writeValueAsString(data); + String expected = mapper.writeValueAsString(reason); + JSONAssert.assertEquals( + expected, + actual, + new CustomComparator( + JSONCompareMode.LENIENT, new Customization("refresh_datetime", (a, b) -> true))); + } + + @Test + void updatesReason() throws JsonProcessingException, JSONException { + // Insert reason data + ObservationReason reason = new ObservationReason(); + reason.setObservationUid(13l); + reason.setReasonCd("9009"); + reason.setReasonDescTxt("PRESENCE OF REASON"); + reason.setBatchId(13l); + + writer.persistReason(List.of(reason)); + + // Upsert reason data + reason.setReasonDescTxt("A DIFFERENT REASON"); + reason.setBatchId(14l); + + writer.persistReason(List.of(reason)); + + // Verify data is as expected + Integer count = + client + .sql( + "SELECT COUNT(*) FROM nrt_observation_reason WHERE observation_uid = 13 AND reason_cd = '9009'") + .query(Integer.class) + .single(); + + assertThat(count).isEqualTo(1); + + Map data = + client + .sql( + "SELECT * FROM nrt_observation_reason WHERE observation_uid = 13 AND reason_cd = '9009'") + .query() + .singleRow(); + + String actual = mapper.writeValueAsString(data); + String expected = mapper.writeValueAsString(reason); + JSONAssert.assertEquals( + expected, + actual, + new CustomComparator( + JSONCompareMode.LENIENT, new Customization("refresh_datetime", (a, b) -> true))); + } + + @Test + void insertsText() throws JsonProcessingException, JSONException { + // Insert text data + ObservationTxt txt = new ObservationTxt(); + txt.setObservationUid(14L); + txt.setOvtSeq(1); + txt.setBatchId(14L); + txt.setOvtTxtTypeCd("N"); + txt.setOvtValueTxt("RECOMMENDED IN SUCH INSTANCES."); + + writer.persistText(List.of(txt)); + + // Verify data is as expected + Map data = + client + .sql("SELECT * FROM nrt_observation_txt WHERE observation_uid = 14 AND ovt_seq = 1") + .query() + .singleRow(); + + String actual = mapper.writeValueAsString(data); + String expected = mapper.writeValueAsString(txt); + JSONAssert.assertEquals( + expected, + actual, + new CustomComparator( + JSONCompareMode.LENIENT, new Customization("refresh_datetime", (a, b) -> true))); + } + + @Test + void updatesText() throws JsonProcessingException, JSONException { + // Insert text data + ObservationTxt txt = new ObservationTxt(); + txt.setObservationUid(15L); + txt.setOvtSeq(2); + txt.setBatchId(15L); + txt.setOvtTxtTypeCd("N"); + txt.setOvtValueTxt("RECOMMENDED IN SUCH INSTANCES."); + + writer.persistText(List.of(txt)); + + // Update text data + txt.setBatchId(16L); + txt.setOvtTxtTypeCd("J"); + txt.setOvtValueTxt("UPDATED VALUE"); + + writer.persistText(List.of(txt)); + + // Verify data is as expected + Integer count = + client + .sql( + "SELECT COUNT(*) FROM nrt_observation_txt WHERE observation_uid = 15 AND ovt_seq = 2") + .query(Integer.class) + .single(); + + assertThat(count).isEqualTo(1); + + Map data = + client + .sql("SELECT * FROM nrt_observation_txt WHERE observation_uid = 15 AND ovt_seq = 2") + .query() + .singleRow(); + + String actual = mapper.writeValueAsString(data); + String expected = mapper.writeValueAsString(txt); + JSONAssert.assertEquals( + expected, + actual, + new CustomComparator( + JSONCompareMode.LENIENT, new Customization("refresh_datetime", (a, b) -> true))); + } +} diff --git a/reporting-pipeline-service/src/test/java/gov/cdc/nbs/report/pipeline/observation/transformer/ObservationParserTest.java b/reporting-pipeline-service/src/test/java/gov/cdc/nbs/report/pipeline/observation/transformer/ObservationParserTest.java index e94868432..8aea0d29b 100644 --- a/reporting-pipeline-service/src/test/java/gov/cdc/nbs/report/pipeline/observation/transformer/ObservationParserTest.java +++ b/reporting-pipeline-service/src/test/java/gov/cdc/nbs/report/pipeline/observation/transformer/ObservationParserTest.java @@ -214,7 +214,6 @@ void testObservationEdxTransformation() { edx.setEdxDocumentUid(10101L); edx.setEdxActUid(observation.getActUid()); edx.setEdxAddTime("2024-09-30T21:08:19.017"); - edx.setBatchId(BATCH_ID); ParsedObservation parsedObservation = ObservationParser.parse(observation, BATCH_ID); diff --git a/reporting-pipeline-service/src/test/resources/application-test.yaml b/reporting-pipeline-service/src/test/resources/application-test.yaml index 5b42dabbc..2eaa3409e 100644 --- a/reporting-pipeline-service/src/test/resources/application-test.yaml +++ b/reporting-pipeline-service/src/test/resources/application-test.yaml @@ -27,4 +27,4 @@ spring: service: fixed-delay: cached-ids: 1000 - datamart: 3000 \ No newline at end of file + datamart: 3000 From 49962c68b526d9b1f35cca8a3580dde2247f5688 Mon Sep 17 00:00:00 2001 From: Michael Peels Date: Mon, 20 Apr 2026 10:19:11 -0400 Subject: [PATCH 07/16] Remove unused key objects --- .../dto/observation/ObservationCodedKey.java | 7 --- .../dto/observation/ObservationEdxKey.java | 14 ------ .../observation/ObservationMaterialKey.java | 6 --- .../dto/observation/ObservationReasonKey.java | 14 ------ .../dto/observation/ObservationTxtKey.java | 14 ------ .../observation/ObservationProcessor.java | 12 ++--- .../transformer/ObservationParser.java | 48 +++++++++---------- 7 files changed, 29 insertions(+), 86 deletions(-) delete mode 100644 reporting-pipeline-service/src/main/java/gov/cdc/nbs/report/pipeline/observation/model/dto/observation/ObservationCodedKey.java delete mode 100644 reporting-pipeline-service/src/main/java/gov/cdc/nbs/report/pipeline/observation/model/dto/observation/ObservationEdxKey.java delete mode 100644 reporting-pipeline-service/src/main/java/gov/cdc/nbs/report/pipeline/observation/model/dto/observation/ObservationMaterialKey.java delete mode 100644 reporting-pipeline-service/src/main/java/gov/cdc/nbs/report/pipeline/observation/model/dto/observation/ObservationReasonKey.java delete mode 100644 reporting-pipeline-service/src/main/java/gov/cdc/nbs/report/pipeline/observation/model/dto/observation/ObservationTxtKey.java diff --git a/reporting-pipeline-service/src/main/java/gov/cdc/nbs/report/pipeline/observation/model/dto/observation/ObservationCodedKey.java b/reporting-pipeline-service/src/main/java/gov/cdc/nbs/report/pipeline/observation/model/dto/observation/ObservationCodedKey.java deleted file mode 100644 index 48a894993..000000000 --- a/reporting-pipeline-service/src/main/java/gov/cdc/nbs/report/pipeline/observation/model/dto/observation/ObservationCodedKey.java +++ /dev/null @@ -1,7 +0,0 @@ -package gov.cdc.nbs.report.pipeline.observation.model.dto.observation; - -import com.fasterxml.jackson.databind.PropertyNamingStrategies; -import com.fasterxml.jackson.databind.annotation.JsonNaming; - -@JsonNaming(PropertyNamingStrategies.SnakeCaseStrategy.class) -public record ObservationCodedKey(Long observationUid, String ovcCode) {} diff --git a/reporting-pipeline-service/src/main/java/gov/cdc/nbs/report/pipeline/observation/model/dto/observation/ObservationEdxKey.java b/reporting-pipeline-service/src/main/java/gov/cdc/nbs/report/pipeline/observation/model/dto/observation/ObservationEdxKey.java deleted file mode 100644 index de559e4ca..000000000 --- a/reporting-pipeline-service/src/main/java/gov/cdc/nbs/report/pipeline/observation/model/dto/observation/ObservationEdxKey.java +++ /dev/null @@ -1,14 +0,0 @@ -package gov.cdc.nbs.report.pipeline.observation.model.dto.observation; - -import com.fasterxml.jackson.databind.PropertyNamingStrategies; -import com.fasterxml.jackson.databind.annotation.JsonNaming; -import lombok.Data; -import lombok.NoArgsConstructor; -import lombok.NonNull; - -@Data -@NoArgsConstructor -@JsonNaming(PropertyNamingStrategies.SnakeCaseStrategy.class) -public class ObservationEdxKey { - @NonNull private Long edxDocumentUid; -} diff --git a/reporting-pipeline-service/src/main/java/gov/cdc/nbs/report/pipeline/observation/model/dto/observation/ObservationMaterialKey.java b/reporting-pipeline-service/src/main/java/gov/cdc/nbs/report/pipeline/observation/model/dto/observation/ObservationMaterialKey.java deleted file mode 100644 index 391773d7b..000000000 --- a/reporting-pipeline-service/src/main/java/gov/cdc/nbs/report/pipeline/observation/model/dto/observation/ObservationMaterialKey.java +++ /dev/null @@ -1,6 +0,0 @@ -package gov.cdc.nbs.report.pipeline.observation.model.dto.observation; - -import com.fasterxml.jackson.annotation.JsonProperty; -import lombok.NonNull; - -public record ObservationMaterialKey(@NonNull @JsonProperty("material_id") Long materialId) {} diff --git a/reporting-pipeline-service/src/main/java/gov/cdc/nbs/report/pipeline/observation/model/dto/observation/ObservationReasonKey.java b/reporting-pipeline-service/src/main/java/gov/cdc/nbs/report/pipeline/observation/model/dto/observation/ObservationReasonKey.java deleted file mode 100644 index da8f43c23..000000000 --- a/reporting-pipeline-service/src/main/java/gov/cdc/nbs/report/pipeline/observation/model/dto/observation/ObservationReasonKey.java +++ /dev/null @@ -1,14 +0,0 @@ -package gov.cdc.nbs.report.pipeline.observation.model.dto.observation; - -import com.fasterxml.jackson.databind.PropertyNamingStrategies; -import com.fasterxml.jackson.databind.annotation.JsonNaming; -import lombok.Data; -import lombok.NoArgsConstructor; - -@Data -@NoArgsConstructor -@JsonNaming(PropertyNamingStrategies.SnakeCaseStrategy.class) -public class ObservationReasonKey { - private Long observationUid; - private String reasonCd; -} diff --git a/reporting-pipeline-service/src/main/java/gov/cdc/nbs/report/pipeline/observation/model/dto/observation/ObservationTxtKey.java b/reporting-pipeline-service/src/main/java/gov/cdc/nbs/report/pipeline/observation/model/dto/observation/ObservationTxtKey.java deleted file mode 100644 index 911f51cc7..000000000 --- a/reporting-pipeline-service/src/main/java/gov/cdc/nbs/report/pipeline/observation/model/dto/observation/ObservationTxtKey.java +++ /dev/null @@ -1,14 +0,0 @@ -package gov.cdc.nbs.report.pipeline.observation.model.dto.observation; - -import com.fasterxml.jackson.databind.PropertyNamingStrategies; -import com.fasterxml.jackson.databind.annotation.JsonNaming; -import lombok.Data; -import lombok.NoArgsConstructor; - -@Data -@NoArgsConstructor -@JsonNaming(PropertyNamingStrategies.SnakeCaseStrategy.class) -public class ObservationTxtKey { - private Long observationUid; - private Integer ovtSeq; -} diff --git a/reporting-pipeline-service/src/main/java/gov/cdc/nbs/report/pipeline/observation/service/observation/ObservationProcessor.java b/reporting-pipeline-service/src/main/java/gov/cdc/nbs/report/pipeline/observation/service/observation/ObservationProcessor.java index 7c0d80852..b3f7b966b 100644 --- a/reporting-pipeline-service/src/main/java/gov/cdc/nbs/report/pipeline/observation/service/observation/ObservationProcessor.java +++ b/reporting-pipeline-service/src/main/java/gov/cdc/nbs/report/pipeline/observation/service/observation/ObservationProcessor.java @@ -27,13 +27,6 @@ @Component public class ObservationProcessor { private static final Logger logger = LoggerFactory.getLogger(ObservationProcessor.class); - - private final ObservationRepository observationRepository; - private final KafkaTemplate kafkaTemplate; - private final NrtObservationWriter nrtWriter; - - private final String nrtObservationTopic; - private final ModelMapper modelMapper = new ModelMapper(); private final CustomJsonGeneratorImpl jsonGenerator = new CustomJsonGeneratorImpl(); @@ -43,6 +36,11 @@ public class ObservationProcessor { private Counter msgSuccess; private Counter msgFailure; + private final ObservationRepository observationRepository; + private final KafkaTemplate kafkaTemplate; + private final NrtObservationWriter nrtWriter; + private final String nrtObservationTopic; + public ObservationProcessor( final CustomMetrics metrics, final ObservationRepository observationRepository, diff --git a/reporting-pipeline-service/src/main/java/gov/cdc/nbs/report/pipeline/observation/transformer/ObservationParser.java b/reporting-pipeline-service/src/main/java/gov/cdc/nbs/report/pipeline/observation/transformer/ObservationParser.java index 1009d9d23..507ed4c05 100644 --- a/reporting-pipeline-service/src/main/java/gov/cdc/nbs/report/pipeline/observation/transformer/ObservationParser.java +++ b/reporting-pipeline-service/src/main/java/gov/cdc/nbs/report/pipeline/observation/transformer/ObservationParser.java @@ -47,57 +47,57 @@ public static ParsedObservation parse(final Observation observation, final long observationTransformed.setBatchId(batchId); // Person Participations - setPersonParticipations( + parsePersonParticipations( observation.getPersonParticipations(), observation.getObsDomainCdSt1(), observationTransformed); // Organization Participations - setOrganizationParticipations( + parseOrganizationParticipations( observation.getOrganizationParticipations(), observation.getObsDomainCdSt1(), observationTransformed); // Material Participations - setMaterialParticipations( + parseMaterialParticipations( observation.getMaterialParticipations(), observation.getObsDomainCdSt1(), parsedObservation); // Follow up Observations - setFollowupObservations( + parseFollowupObservations( observation.getFollowupObservations(), observation.getObsDomainCdSt1(), observationTransformed); // Parent Observations - setParentObservations(observation.getParentObservations(), observationTransformed); + parseParentObservations(observation.getParentObservations(), observationTransformed); // Act Ids - setActIds(observation.getActIds(), observationTransformed); + parseActIds(observation.getActIds(), observationTransformed); // Observation Coded data - setObservationCoded(observation.getObsCode(), parsedObservation); + parseObservationCoded(observation.getObsCode(), parsedObservation); // Observation Date data - setObservationDate(observation.getObsDate(), parsedObservation); + parseObservationDate(observation.getObsDate(), parsedObservation); // Observation Edx data - setObservationEdx(observation.getEdxIds(), parsedObservation); + parseObservationEdx(observation.getEdxIds(), parsedObservation); // Observation Numeric data - setObservationNumeric(observation.getObsNum(), parsedObservation); + parseObservationNumeric(observation.getObsNum(), parsedObservation); // Observation Reason data - setObservationReasons(observation.getObsReason(), parsedObservation); + parseObservationReasons(observation.getObsReason(), parsedObservation); // Observation Text data - setObservationTxt(observation.getObsTxt(), parsedObservation); + parseObservationTxt(observation.getObsTxt(), parsedObservation); return parsedObservation; } - private static void setPersonParticipations( + private static void parsePersonParticipations( String personParticipations, String obsDomainCdSt1, ObservationTransformed transformed) { try { JsonNode personParticipationsJsonArray = parseJsonArray(personParticipations); @@ -178,7 +178,7 @@ private static void setPersonParticipations( } } - private static void setOrganizationParticipations( + private static void parseOrganizationParticipations( String organizationParticipations, String obsDomainCdSt1, ObservationTransformed transformed) { @@ -228,7 +228,7 @@ private static void setOrganizationParticipations( } } - private static void setMaterialParticipations( + private static void parseMaterialParticipations( String materialParticipations, String obsDomainCdSt1, ParsedObservation parsedObservation) { try { JsonNode materialParticipationsJsonArray = parseJsonArray(materialParticipations); @@ -275,7 +275,7 @@ private static void setPersonParticipationRoles( } } - private static void setFollowupObservations( + private static void parseFollowupObservations( String followupObservations, String obsDomainCdSt1, ObservationTransformed transformed) { try { JsonNode followupObservationsJsonArray = parseJsonArray(followupObservations); @@ -310,7 +310,7 @@ private static void setFollowupObservations( } } - private static void setParentObservations( + private static void parseParentObservations( String parentObservations, ObservationTransformed transformed) { try { JsonNode parentObservationsJsonArray = parseJsonArray(parentObservations); @@ -339,7 +339,7 @@ private static void setParentObservations( } } - private static void setActIds(String actIds, ObservationTransformed observationTransformed) { + private static void parseActIds(String actIds, ObservationTransformed observationTransformed) { try { JsonNode actIdsJsonArray = parseJsonArray(actIds); @@ -366,7 +366,7 @@ private static void setActIds(String actIds, ObservationTransformed observationT } } - private static void setObservationCoded( + private static void parseObservationCoded( String observationCoded, ParsedObservation parsedObservation) { try { JsonNode observationCodedJsonArray = parseJsonArray(observationCoded); @@ -386,7 +386,7 @@ private static void setObservationCoded( } } - private static void setObservationDate( + private static void parseObservationDate( String observationDate, ParsedObservation parsedObservation) { try { JsonNode observationDateJsonArray = parseJsonArray(observationDate); @@ -405,7 +405,7 @@ private static void setObservationDate( } } - private static void setObservationEdx( + private static void parseObservationEdx( String observationEdx, ParsedObservation parsedObservation) { try { JsonNode observationEdxJsonArray = parseJsonArray(observationEdx); @@ -422,7 +422,7 @@ private static void setObservationEdx( } } - private static void setObservationNumeric( + private static void parseObservationNumeric( String observationNumeric, ParsedObservation parsedObservation) { try { JsonNode observationNumericJsonArray = parseJsonArray(observationNumeric); @@ -441,7 +441,7 @@ private static void setObservationNumeric( } } - private static void setObservationReasons( + private static void parseObservationReasons( String observationReasons, ParsedObservation parsedObservation) { try { JsonNode observationReasonsJsonArray = parseJsonArray(observationReasons); @@ -460,7 +460,7 @@ private static void setObservationReasons( } } - private static void setObservationTxt( + private static void parseObservationTxt( String observationTxt, ParsedObservation parsedObservation) { try { JsonNode observationTxtJsonArray = parseJsonArray(observationTxt); From f9a357a167ffbe4ddcc5de77b44d019343521a50 Mon Sep 17 00:00:00 2001 From: Michael Peels Date: Mon, 20 Apr 2026 10:41:50 -0400 Subject: [PATCH 08/16] Update comments --- .../pipeline/observation/service/ObservationService.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/reporting-pipeline-service/src/main/java/gov/cdc/nbs/report/pipeline/observation/service/ObservationService.java b/reporting-pipeline-service/src/main/java/gov/cdc/nbs/report/pipeline/observation/service/ObservationService.java index c0b3262c4..556745308 100644 --- a/reporting-pipeline-service/src/main/java/gov/cdc/nbs/report/pipeline/observation/service/ObservationService.java +++ b/reporting-pipeline-service/src/main/java/gov/cdc/nbs/report/pipeline/observation/service/ObservationService.java @@ -40,7 +40,8 @@ *
  • Fetching enriched data from the database using stored procedures. *
  • Transforming raw data into reporting-optimized formats for Observation and its related * entities (Coded, Date, EDX, Material, Numeric, Reason, Txt). - *
  • Pushing transformed data to corresponding output topics in Kafka. + *
  • Persisting transformed data to corresponding nrt_observation_* tables + *
  • Pushing transformed data to corresponding output topic in Kafka. *
  • Handling retries and dead-letter topics (DLT) for resilient processing. *
*/ @@ -141,8 +142,7 @@ private void handleActRelationship(String value, long batchId) { // For LabReport values, we only need to trigger if the relationship is deleted (not covered // in updates to Observation) // PHC targets are excluded from the LabReport association updates, as the LabReport will - // receive - // an update in Observation + // receive an update in Observation if (typeCd.equals("LabReport") && targetClassCd.equals("OBS")) { observationProcessor.process(batchId, sourceActUid); } From f1e8a0a390199cb9e0ccbd5553e315e6a5e38631 Mon Sep 17 00:00:00 2001 From: Michael Peels Date: Mon, 20 Apr 2026 11:41:27 -0400 Subject: [PATCH 09/16] Move files --- .../observation => repository}/NrtObservationWriter.java | 2 +- .../service/{observation => }/ObservationProcessor.java | 3 ++- .../pipeline/observation/service/ObservationService.java | 2 +- .../observation => repository}/NrtObservationWriterTest.java | 3 ++- .../service/{observation => }/ObservationProcessorTest.java | 3 ++- .../pipeline/observation/service/ObservationServiceTest.java | 1 - 6 files changed, 8 insertions(+), 6 deletions(-) rename reporting-pipeline-service/src/main/java/gov/cdc/nbs/report/pipeline/observation/{service/observation => repository}/NrtObservationWriter.java (99%) rename reporting-pipeline-service/src/main/java/gov/cdc/nbs/report/pipeline/observation/service/{observation => }/ObservationProcessor.java (97%) rename reporting-pipeline-service/src/test/java/gov/cdc/nbs/report/pipeline/observation/{service/observation => repository}/NrtObservationWriterTest.java (99%) rename reporting-pipeline-service/src/test/java/gov/cdc/nbs/report/pipeline/observation/service/{observation => }/ObservationProcessorTest.java (98%) diff --git a/reporting-pipeline-service/src/main/java/gov/cdc/nbs/report/pipeline/observation/service/observation/NrtObservationWriter.java b/reporting-pipeline-service/src/main/java/gov/cdc/nbs/report/pipeline/observation/repository/NrtObservationWriter.java similarity index 99% rename from reporting-pipeline-service/src/main/java/gov/cdc/nbs/report/pipeline/observation/service/observation/NrtObservationWriter.java rename to reporting-pipeline-service/src/main/java/gov/cdc/nbs/report/pipeline/observation/repository/NrtObservationWriter.java index be7234004..2d8644c57 100644 --- a/reporting-pipeline-service/src/main/java/gov/cdc/nbs/report/pipeline/observation/service/observation/NrtObservationWriter.java +++ b/reporting-pipeline-service/src/main/java/gov/cdc/nbs/report/pipeline/observation/repository/NrtObservationWriter.java @@ -1,4 +1,4 @@ -package gov.cdc.nbs.report.pipeline.observation.service.observation; +package gov.cdc.nbs.report.pipeline.observation.repository; import gov.cdc.nbs.report.pipeline.observation.model.dto.observation.ObservationCoded; import gov.cdc.nbs.report.pipeline.observation.model.dto.observation.ObservationDate; diff --git a/reporting-pipeline-service/src/main/java/gov/cdc/nbs/report/pipeline/observation/service/observation/ObservationProcessor.java b/reporting-pipeline-service/src/main/java/gov/cdc/nbs/report/pipeline/observation/service/ObservationProcessor.java similarity index 97% rename from reporting-pipeline-service/src/main/java/gov/cdc/nbs/report/pipeline/observation/service/observation/ObservationProcessor.java rename to reporting-pipeline-service/src/main/java/gov/cdc/nbs/report/pipeline/observation/service/ObservationProcessor.java index b3f7b966b..1fd440dd4 100644 --- a/reporting-pipeline-service/src/main/java/gov/cdc/nbs/report/pipeline/observation/service/observation/ObservationProcessor.java +++ b/reporting-pipeline-service/src/main/java/gov/cdc/nbs/report/pipeline/observation/service/ObservationProcessor.java @@ -1,4 +1,4 @@ -package gov.cdc.nbs.report.pipeline.observation.service.observation; +package gov.cdc.nbs.report.pipeline.observation.service; import static gov.cdc.etldatapipeline.commonutil.UtilHelper.errorMessage; @@ -10,6 +10,7 @@ import gov.cdc.nbs.report.pipeline.observation.model.dto.observation.ObservationKey; import gov.cdc.nbs.report.pipeline.observation.model.dto.observation.ObservationReporting; import gov.cdc.nbs.report.pipeline.observation.model.dto.observation.ParsedObservation; +import gov.cdc.nbs.report.pipeline.observation.repository.NrtObservationWriter; import gov.cdc.nbs.report.pipeline.observation.repository.ObservationRepository; import gov.cdc.nbs.report.pipeline.observation.transformer.ObservationParser; import io.micrometer.core.instrument.Counter; diff --git a/reporting-pipeline-service/src/main/java/gov/cdc/nbs/report/pipeline/observation/service/ObservationService.java b/reporting-pipeline-service/src/main/java/gov/cdc/nbs/report/pipeline/observation/service/ObservationService.java index 556745308..2a8e1cb3b 100644 --- a/reporting-pipeline-service/src/main/java/gov/cdc/nbs/report/pipeline/observation/service/ObservationService.java +++ b/reporting-pipeline-service/src/main/java/gov/cdc/nbs/report/pipeline/observation/service/ObservationService.java @@ -8,7 +8,7 @@ import com.fasterxml.jackson.core.JsonProcessingException; import gov.cdc.etldatapipeline.commonutil.DataProcessingException; import gov.cdc.etldatapipeline.commonutil.NoDataException; -import gov.cdc.nbs.report.pipeline.observation.service.observation.ObservationProcessor; + import java.util.NoSuchElementException; import java.util.concurrent.CompletableFuture; import java.util.concurrent.ExecutorService; diff --git a/reporting-pipeline-service/src/test/java/gov/cdc/nbs/report/pipeline/observation/service/observation/NrtObservationWriterTest.java b/reporting-pipeline-service/src/test/java/gov/cdc/nbs/report/pipeline/observation/repository/NrtObservationWriterTest.java similarity index 99% rename from reporting-pipeline-service/src/test/java/gov/cdc/nbs/report/pipeline/observation/service/observation/NrtObservationWriterTest.java rename to reporting-pipeline-service/src/test/java/gov/cdc/nbs/report/pipeline/observation/repository/NrtObservationWriterTest.java index a8e823dea..6015b018c 100644 --- a/reporting-pipeline-service/src/test/java/gov/cdc/nbs/report/pipeline/observation/service/observation/NrtObservationWriterTest.java +++ b/reporting-pipeline-service/src/test/java/gov/cdc/nbs/report/pipeline/observation/repository/NrtObservationWriterTest.java @@ -1,4 +1,4 @@ -package gov.cdc.nbs.report.pipeline.observation.service.observation; +package gov.cdc.nbs.report.pipeline.observation.repository; import static org.assertj.core.api.Assertions.assertThat; @@ -13,6 +13,7 @@ import gov.cdc.nbs.report.pipeline.observation.model.dto.observation.ObservationNumeric; import gov.cdc.nbs.report.pipeline.observation.model.dto.observation.ObservationReason; import gov.cdc.nbs.report.pipeline.observation.model.dto.observation.ObservationTxt; + import java.text.SimpleDateFormat; import java.util.List; import java.util.Map; diff --git a/reporting-pipeline-service/src/test/java/gov/cdc/nbs/report/pipeline/observation/service/observation/ObservationProcessorTest.java b/reporting-pipeline-service/src/test/java/gov/cdc/nbs/report/pipeline/observation/service/ObservationProcessorTest.java similarity index 98% rename from reporting-pipeline-service/src/test/java/gov/cdc/nbs/report/pipeline/observation/service/observation/ObservationProcessorTest.java rename to reporting-pipeline-service/src/test/java/gov/cdc/nbs/report/pipeline/observation/service/ObservationProcessorTest.java index 78ac06243..a5cb952a1 100644 --- a/reporting-pipeline-service/src/test/java/gov/cdc/nbs/report/pipeline/observation/service/observation/ObservationProcessorTest.java +++ b/reporting-pipeline-service/src/test/java/gov/cdc/nbs/report/pipeline/observation/service/ObservationProcessorTest.java @@ -1,4 +1,4 @@ -package gov.cdc.nbs.report.pipeline.observation.service.observation; +package gov.cdc.nbs.report.pipeline.observation.service; import static gov.cdc.etldatapipeline.commonutil.TestUtils.readFileData; import static org.assertj.core.api.Assertions.assertThat; @@ -15,6 +15,7 @@ import gov.cdc.nbs.report.pipeline.observation.model.dto.observation.Observation; import gov.cdc.nbs.report.pipeline.observation.model.dto.observation.ObservationKey; import gov.cdc.nbs.report.pipeline.observation.model.dto.observation.ObservationReporting; +import gov.cdc.nbs.report.pipeline.observation.repository.NrtObservationWriter; import gov.cdc.nbs.report.pipeline.observation.repository.ObservationRepository; import io.micrometer.core.instrument.simple.SimpleMeterRegistry; import java.util.Optional; diff --git a/reporting-pipeline-service/src/test/java/gov/cdc/nbs/report/pipeline/observation/service/ObservationServiceTest.java b/reporting-pipeline-service/src/test/java/gov/cdc/nbs/report/pipeline/observation/service/ObservationServiceTest.java index 3ef69c478..eb910c97d 100644 --- a/reporting-pipeline-service/src/test/java/gov/cdc/nbs/report/pipeline/observation/service/ObservationServiceTest.java +++ b/reporting-pipeline-service/src/test/java/gov/cdc/nbs/report/pipeline/observation/service/ObservationServiceTest.java @@ -4,7 +4,6 @@ import static org.junit.jupiter.api.Assertions.assertThrows; import static org.mockito.Mockito.*; -import gov.cdc.nbs.report.pipeline.observation.service.observation.ObservationProcessor; import java.util.concurrent.CompletableFuture; import java.util.concurrent.CompletionException; import org.apache.kafka.clients.consumer.ConsumerRecord; From 82afb82889d9af614b4eed714967c8562bdd32da Mon Sep 17 00:00:00 2001 From: Michael Peels Date: Mon, 20 Apr 2026 11:44:23 -0400 Subject: [PATCH 10/16] Spotless --- .../report/pipeline/observation/service/ObservationService.java | 1 - .../observation/repository/NrtObservationWriterTest.java | 1 - 2 files changed, 2 deletions(-) diff --git a/reporting-pipeline-service/src/main/java/gov/cdc/nbs/report/pipeline/observation/service/ObservationService.java b/reporting-pipeline-service/src/main/java/gov/cdc/nbs/report/pipeline/observation/service/ObservationService.java index 2a8e1cb3b..0f1b85b91 100644 --- a/reporting-pipeline-service/src/main/java/gov/cdc/nbs/report/pipeline/observation/service/ObservationService.java +++ b/reporting-pipeline-service/src/main/java/gov/cdc/nbs/report/pipeline/observation/service/ObservationService.java @@ -8,7 +8,6 @@ import com.fasterxml.jackson.core.JsonProcessingException; import gov.cdc.etldatapipeline.commonutil.DataProcessingException; import gov.cdc.etldatapipeline.commonutil.NoDataException; - import java.util.NoSuchElementException; import java.util.concurrent.CompletableFuture; import java.util.concurrent.ExecutorService; diff --git a/reporting-pipeline-service/src/test/java/gov/cdc/nbs/report/pipeline/observation/repository/NrtObservationWriterTest.java b/reporting-pipeline-service/src/test/java/gov/cdc/nbs/report/pipeline/observation/repository/NrtObservationWriterTest.java index 6015b018c..fe52cc054 100644 --- a/reporting-pipeline-service/src/test/java/gov/cdc/nbs/report/pipeline/observation/repository/NrtObservationWriterTest.java +++ b/reporting-pipeline-service/src/test/java/gov/cdc/nbs/report/pipeline/observation/repository/NrtObservationWriterTest.java @@ -13,7 +13,6 @@ import gov.cdc.nbs.report.pipeline.observation.model.dto.observation.ObservationNumeric; import gov.cdc.nbs.report.pipeline.observation.model.dto.observation.ObservationReason; import gov.cdc.nbs.report.pipeline.observation.model.dto.observation.ObservationTxt; - import java.text.SimpleDateFormat; import java.util.List; import java.util.Map; From 8c4a44e03e2ead3b0f7cdc4e1bfd0db8519dee24 Mon Sep 17 00:00:00 2001 From: Michael Peels Date: Mon, 20 Apr 2026 17:27:36 -0400 Subject: [PATCH 11/16] Update comment --- .../pipeline/observation/service/ObservationService.java | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/reporting-pipeline-service/src/main/java/gov/cdc/nbs/report/pipeline/observation/service/ObservationService.java b/reporting-pipeline-service/src/main/java/gov/cdc/nbs/report/pipeline/observation/service/ObservationService.java index 0f1b85b91..73ac8e037 100644 --- a/reporting-pipeline-service/src/main/java/gov/cdc/nbs/report/pipeline/observation/service/ObservationService.java +++ b/reporting-pipeline-service/src/main/java/gov/cdc/nbs/report/pipeline/observation/service/ObservationService.java @@ -30,7 +30,11 @@ /** * Service class for processing Observation-related change events in the Real Time Reporting (RTR) * pipeline. This service handles the "hydration" of data for Observations by consuming Kafka events - * from transactional source topics, transforming them, and producing them to reporting topics. + * from transactional source topics, transforming them, and producing them to reporting topics. This + * service operates differently than other RTR services in the fact that it also inserts directly + * into the nrt_ database tables to eliminate a race condition between the PostProcessingService and + * the Kafka Sink Connector. More info can be found here: + * https://cdc-nbs.atlassian.net/browse/APP-519 * *

Key responsibilities include: * From 4fde83c6033c4d323ef5118667dd4fa99f707181 Mon Sep 17 00:00:00 2001 From: Michael Peels Date: Mon, 20 Apr 2026 17:31:00 -0400 Subject: [PATCH 12/16] Move consumer --- ...nService.java => ObservationConsumer.java} | 9 ++++---- .../service/ObservationProcessor.java | 4 ++-- ...Test.java => ObservationConsumerTest.java} | 21 ++++++++++--------- 3 files changed, 18 insertions(+), 16 deletions(-) rename reporting-pipeline-service/src/main/java/gov/cdc/nbs/report/pipeline/observation/{service/ObservationService.java => ObservationConsumer.java} (97%) rename reporting-pipeline-service/src/test/java/gov/cdc/nbs/report/pipeline/observation/{service/ObservationServiceTest.java => ObservationConsumerTest.java} (85%) diff --git a/reporting-pipeline-service/src/main/java/gov/cdc/nbs/report/pipeline/observation/service/ObservationService.java b/reporting-pipeline-service/src/main/java/gov/cdc/nbs/report/pipeline/observation/ObservationConsumer.java similarity index 97% rename from reporting-pipeline-service/src/main/java/gov/cdc/nbs/report/pipeline/observation/service/ObservationService.java rename to reporting-pipeline-service/src/main/java/gov/cdc/nbs/report/pipeline/observation/ObservationConsumer.java index 73ac8e037..1a7f86be8 100644 --- a/reporting-pipeline-service/src/main/java/gov/cdc/nbs/report/pipeline/observation/service/ObservationService.java +++ b/reporting-pipeline-service/src/main/java/gov/cdc/nbs/report/pipeline/observation/ObservationConsumer.java @@ -1,4 +1,4 @@ -package gov.cdc.nbs.report.pipeline.observation.service; +package gov.cdc.nbs.report.pipeline.observation; import static gov.cdc.etldatapipeline.commonutil.UtilHelper.errorMessage; import static gov.cdc.etldatapipeline.commonutil.UtilHelper.extractChangeDataCaptureOperation; @@ -8,6 +8,7 @@ import com.fasterxml.jackson.core.JsonProcessingException; import gov.cdc.etldatapipeline.commonutil.DataProcessingException; import gov.cdc.etldatapipeline.commonutil.NoDataException; +import gov.cdc.nbs.report.pipeline.observation.service.ObservationProcessor; import java.util.NoSuchElementException; import java.util.concurrent.CompletableFuture; import java.util.concurrent.ExecutorService; @@ -49,8 +50,8 @@ * */ @Service -public class ObservationService { - private static final Logger logger = LoggerFactory.getLogger(ObservationService.class); +public class ObservationConsumer { + private static final Logger logger = LoggerFactory.getLogger(ObservationConsumer.class); private static final String BEFORE_PATH = "before"; private static final String TOPIC_DEBUG_LOG = "Received Observation with id: {} from topic: {}"; @@ -59,7 +60,7 @@ public class ObservationService { private final ObservationProcessor observationProcessor; private final ExecutorService obsExecutor; - public ObservationService( + public ObservationConsumer( final ObservationProcessor observationProcessor, @Value("${spring.kafka.topics.nbs.observation}") final String observationTopic, @Value("${spring.kafka.topics.nbs.act-relationship}") final String actRelationshipTopic, diff --git a/reporting-pipeline-service/src/main/java/gov/cdc/nbs/report/pipeline/observation/service/ObservationProcessor.java b/reporting-pipeline-service/src/main/java/gov/cdc/nbs/report/pipeline/observation/service/ObservationProcessor.java index 1fd440dd4..fead27561 100644 --- a/reporting-pipeline-service/src/main/java/gov/cdc/nbs/report/pipeline/observation/service/ObservationProcessor.java +++ b/reporting-pipeline-service/src/main/java/gov/cdc/nbs/report/pipeline/observation/service/ObservationProcessor.java @@ -22,10 +22,10 @@ import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.beans.factory.annotation.Value; import org.springframework.kafka.core.KafkaTemplate; -import org.springframework.stereotype.Component; +import org.springframework.stereotype.Service; /** Handles the processing of Observation data */ -@Component +@Service public class ObservationProcessor { private static final Logger logger = LoggerFactory.getLogger(ObservationProcessor.class); private final ModelMapper modelMapper = new ModelMapper(); diff --git a/reporting-pipeline-service/src/test/java/gov/cdc/nbs/report/pipeline/observation/service/ObservationServiceTest.java b/reporting-pipeline-service/src/test/java/gov/cdc/nbs/report/pipeline/observation/ObservationConsumerTest.java similarity index 85% rename from reporting-pipeline-service/src/test/java/gov/cdc/nbs/report/pipeline/observation/service/ObservationServiceTest.java rename to reporting-pipeline-service/src/test/java/gov/cdc/nbs/report/pipeline/observation/ObservationConsumerTest.java index eb910c97d..a72d77417 100644 --- a/reporting-pipeline-service/src/test/java/gov/cdc/nbs/report/pipeline/observation/service/ObservationServiceTest.java +++ b/reporting-pipeline-service/src/test/java/gov/cdc/nbs/report/pipeline/observation/ObservationConsumerTest.java @@ -1,23 +1,24 @@ -package gov.cdc.nbs.report.pipeline.observation.service; +package gov.cdc.nbs.report.pipeline.observation; import static org.assertj.core.api.Assertions.assertThat; import static org.junit.jupiter.api.Assertions.assertThrows; import static org.mockito.Mockito.*; +import gov.cdc.nbs.report.pipeline.observation.service.ObservationProcessor; import java.util.concurrent.CompletableFuture; import java.util.concurrent.CompletionException; import org.apache.kafka.clients.consumer.ConsumerRecord; import org.junit.jupiter.api.Test; import org.mockito.*; -class ObservationServiceTest { +class ObservationConsumerTest { private final String observationTopic = "Observation"; private final String actRelationshipTopic = "Act_relationship"; @Mock ObservationProcessor processor = Mockito.mock(ObservationProcessor.class); - private ObservationService service = - new ObservationService(processor, observationTopic, actRelationshipTopic, 1); + private ObservationConsumer consumer = + new ObservationConsumer(processor, observationTopic, actRelationshipTopic, 1); @Test void processesObservationMessage() { // observation_uid @@ -35,7 +36,7 @@ void processesObservationMessage() { // observation_uid ConsumerRecord consumerRecord = new ConsumerRecord<>(observationTopic, 0, 1l, null, message); // receives valid observation message - service.processMessage(consumerRecord).join(); + consumer.processMessage(consumerRecord).join(); // sends to ObservationProcessor verify(processor, times(1)).process(0, "123"); @@ -59,7 +60,7 @@ void processesActRelationshipMessage() { ConsumerRecord consumerRecord = new ConsumerRecord<>(actRelationshipTopic, 0, 1l, null, message); // receives valid act_relationship message - service.processMessage(consumerRecord).join(); + consumer.processMessage(consumerRecord).join(); // sends to ObservationProcessor verify(processor, times(1)).process(0, "1"); @@ -83,7 +84,7 @@ void doesNotProcessActRelationshipMessageBadOp() { ConsumerRecord consumerRecord = new ConsumerRecord<>(actRelationshipTopic, 0, 1l, null, message); // receives non 'delete' act_relationship message - service.processMessage(consumerRecord).join(); + consumer.processMessage(consumerRecord).join(); // does not send to ObservationProcessor verifyNoInteractions(processor); @@ -107,7 +108,7 @@ void doesNotProcessActRelationshipMessageBadTypeCd() { ConsumerRecord consumerRecord = new ConsumerRecord<>(actRelationshipTopic, 0, 1l, null, message); // receives act_relationship message with a type_cd other than 'LabReport' - service.processMessage(consumerRecord).join(); + consumer.processMessage(consumerRecord).join(); // does not send to ObservationProcessor verifyNoInteractions(processor); @@ -117,7 +118,7 @@ void doesNotProcessActRelationshipMessageBadTypeCd() { void throwsExceptionForBadTopic() { ConsumerRecord consumerRecord = new ConsumerRecord<>("bad_topic", 0, 1l, null, ""); - CompletableFuture future = service.processMessage(consumerRecord); + CompletableFuture future = consumer.processMessage(consumerRecord); CompletionException ex = assertThrows(CompletionException.class, future::join); assertThat(ex.getCause().getMessage()) @@ -138,7 +139,7 @@ void throwsExceptionForBadActRelationshipMessage() { """; ConsumerRecord consumerRecord = new ConsumerRecord<>(actRelationshipTopic, 0, 1l, null, message); - CompletableFuture future = service.processMessage(consumerRecord); + CompletableFuture future = consumer.processMessage(consumerRecord); CompletionException ex = assertThrows(CompletionException.class, future::join); assertThat(ex.getCause().getMessage()) From e12f6634f1abfb0c048dc6e7e5dab1d1a24d464f Mon Sep 17 00:00:00 2001 From: Michael Peels Date: Tue, 21 Apr 2026 10:18:30 -0400 Subject: [PATCH 13/16] Remove unused properties --- .../src/main/resources/application.yaml | 7 ------- 1 file changed, 7 deletions(-) diff --git a/reporting-pipeline-service/src/main/resources/application.yaml b/reporting-pipeline-service/src/main/resources/application.yaml index aa40a2576..074204694 100644 --- a/reporting-pipeline-service/src/main/resources/application.yaml +++ b/reporting-pipeline-service/src/main/resources/application.yaml @@ -45,13 +45,6 @@ spring: ldf-data: ${TOPIC_NRT_LDF_DATA:nrt_ldf_data} metadata-columns: ${TOPIC_NRT_METADATA_COLUMNS:nrt_metadata_columns} observation: ${TOPIC_NRT_OBSERVATION:nrt_observation} - observation-coded: ${TOPIC_NRT_OBSERVATION_CODED:nrt_observation_coded} - observation-date: ${TOPIC_NRT_OBSERVATION_DATE:nrt_observation_date} - observation-edx: ${TOPIC_NRT_OBSERVATION_EDX:nrt_observation_edx} - observation-material: ${TOPIC_NRT_OBSERVATION_MATERIAL:nrt_observation_material} - observation-numeric: ${TOPIC_NRT_OBSERVATION_NUMERIC:nrt_observation_numeric} - observation-reason: ${TOPIC_NRT_OBSERVATION_REASON:nrt_observation_reason} - observation-txt: ${TOPIC_NRT_OBSERVATION_TXT:nrt_observation_txt} odse-nbs-page: ${TOPIC_NRT_ODSE_NBS_PAGE:nrt_odse_NBS_page} odse-state-defined-field-metadata: ${TOPIC_NRT_ODSE_STATE_DEFINED_FIELD_METADATA:nrt_odse_state_defined_field_metadata} organization: ${TOPIC_NRT_ORGANIZATION:nrt_organization} From 0bd72027f0098156c924b545a268ebf949920fb7 Mon Sep 17 00:00:00 2001 From: Michael Peels Date: Fri, 24 Apr 2026 16:17:21 -0400 Subject: [PATCH 14/16] Update SQL --- .../observation/repository/NrtObservationWriter.java | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/reporting-pipeline-service/src/main/java/gov/cdc/nbs/report/pipeline/observation/repository/NrtObservationWriter.java b/reporting-pipeline-service/src/main/java/gov/cdc/nbs/report/pipeline/observation/repository/NrtObservationWriter.java index 2d8644c57..843a6faac 100644 --- a/reporting-pipeline-service/src/main/java/gov/cdc/nbs/report/pipeline/observation/repository/NrtObservationWriter.java +++ b/reporting-pipeline-service/src/main/java/gov/cdc/nbs/report/pipeline/observation/repository/NrtObservationWriter.java @@ -161,8 +161,6 @@ void persistMaterials(List materials) { ON nrt_observation_coded.observation_uid = source.observation_uid AND nrt_observation_coded.ovc_code = source.ovc_code WHEN MATCHED THEN UPDATE SET - observation_uid = source.observation_uid, - ovc_code = source.ovc_code, ovc_code_system_cd = source.ovc_code_system_cd, ovc_code_system_desc_txt = source.ovc_code_system_desc_txt, ovc_display_name = source.ovc_display_name, @@ -229,7 +227,6 @@ void persistCoded(List codedEntries) { ON nrt_observation_date.observation_uid = source.observation_uid WHEN MATCHED THEN UPDATE SET - observation_uid = source.observation_uid, ovd_from_date = source.ovd_from_date, ovd_to_date = source.ovd_to_date, ovd_seq = source.ovd_seq, @@ -275,8 +272,6 @@ void persistDate(List dateEntries) { ON nrt_observation_edx.edx_document_uid = source.edx_document_uid AND nrt_observation_edx.edx_act_uid = source.edx_act_uid WHEN MATCHED THEN UPDATE SET - edx_document_uid = source.edx_document_uid, - edx_act_uid = source.edx_act_uid, edx_add_time = source.edx_add_time WHEN NOT MATCHED THEN INSERT ( @@ -320,7 +315,6 @@ void persistEdx(List edxEntries) { ON nrt_observation_numeric.observation_uid = source.observation_uid AND nrt_observation_numeric.ovn_seq = source.ovn_seq WHEN MATCHED THEN UPDATE SET - observation_uid = source.observation_uid, ovn_high_range = source.ovn_high_range, ovn_low_range = source.ovn_low_range, ovn_comparator_cd_1 = source.ovn_comparator_cd_1, @@ -328,7 +322,6 @@ void persistEdx(List edxEntries) { ovn_numeric_value_2 = source.ovn_numeric_value_2, ovn_numeric_unit_cd = source.ovn_numeric_unit_cd, ovn_separator_cd = source.ovn_separator_cd, - ovn_seq = source.ovn_seq, batch_id = source.batch_id WHEN NOT MATCHED THEN INSERT ( @@ -387,8 +380,6 @@ void persistNumeric(List numericEntries) { ON nrt_observation_reason.observation_uid = source.observation_uid AND nrt_observation_reason.reason_cd = source.reason_cd WHEN MATCHED THEN UPDATE SET - observation_uid = source.observation_uid, - reason_cd = source.reason_cd, reason_desc_txt = source.reason_desc_txt, batch_id = source.batch_id WHEN NOT MATCHED THEN @@ -431,8 +422,6 @@ void persistReason(List reasonEntries) { ON nrt_observation_txt.observation_uid = source.observation_uid AND nrt_observation_txt.ovt_seq = source.ovt_seq WHEN MATCHED THEN UPDATE SET - observation_uid = source.observation_uid, - ovt_seq = source.ovt_seq, ovt_txt_type_cd = source.ovt_txt_type_cd, ovt_value_txt = source.ovt_value_txt, batch_id = source.batch_id From 21a086de12877bada202032834a9fdf4bbe1153f Mon Sep 17 00:00:00 2001 From: Michael Peels Date: Fri, 24 Apr 2026 16:33:01 -0400 Subject: [PATCH 15/16] slow datamart processing to help interview test --- .../src/test/resources/application-test.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/reporting-pipeline-service/src/test/resources/application-test.yaml b/reporting-pipeline-service/src/test/resources/application-test.yaml index 2eaa3409e..2a8f910f1 100644 --- a/reporting-pipeline-service/src/test/resources/application-test.yaml +++ b/reporting-pipeline-service/src/test/resources/application-test.yaml @@ -27,4 +27,4 @@ spring: service: fixed-delay: cached-ids: 1000 - datamart: 3000 + datamart: 20000 From 63001a0088254d7d3dae697725f2771bafce2799 Mon Sep 17 00:00:00 2001 From: Michael Peels Date: Fri, 24 Apr 2026 16:46:16 -0400 Subject: [PATCH 16/16] Slower processing for Interview race condition --- .../src/test/resources/application-test.yaml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/reporting-pipeline-service/src/test/resources/application-test.yaml b/reporting-pipeline-service/src/test/resources/application-test.yaml index 2a8f910f1..6594ffa17 100644 --- a/reporting-pipeline-service/src/test/resources/application-test.yaml +++ b/reporting-pipeline-service/src/test/resources/application-test.yaml @@ -24,7 +24,3 @@ spring: value-deserializer: org.springframework.kafka.support.serializer.JsonDeserializer maxPollIntervalMs: 3000 metadata.max.age.ms: 6000 -service: - fixed-delay: - cached-ids: 1000 - datamart: 20000