diff --git a/dhis-2/dhis-test-integration/src/test/java/org/hisp/dhis/tracker/export/singleevent/SingleEventChangeLogServiceTest.java b/dhis-2/dhis-test-integration/src/test/java/org/hisp/dhis/tracker/export/singleevent/SingleEventChangeLogServiceTest.java index 2a95551eea89..233172e79737 100644 --- a/dhis-2/dhis-test-integration/src/test/java/org/hisp/dhis/tracker/export/singleevent/SingleEventChangeLogServiceTest.java +++ b/dhis-2/dhis-test-integration/src/test/java/org/hisp/dhis/tracker/export/singleevent/SingleEventChangeLogServiceTest.java @@ -438,6 +438,7 @@ private void updateDataValue(String event, String dataElementUid, String newValu TrackerImportParams.builder().build(), TrackerObjects.builder().events(List.of(e)).build())); }); + manager.clear(); } private void updateEventDates(UID event, Instant newDate) throws IOException { diff --git a/dhis-2/dhis-test-integration/src/test/java/org/hisp/dhis/tracker/export/trackerevent/TrackerEventChangeLogServiceTest.java b/dhis-2/dhis-test-integration/src/test/java/org/hisp/dhis/tracker/export/trackerevent/TrackerEventChangeLogServiceTest.java index 3b8025005aba..dc343f0de87d 100644 --- a/dhis-2/dhis-test-integration/src/test/java/org/hisp/dhis/tracker/export/trackerevent/TrackerEventChangeLogServiceTest.java +++ b/dhis-2/dhis-test-integration/src/test/java/org/hisp/dhis/tracker/export/trackerevent/TrackerEventChangeLogServiceTest.java @@ -470,6 +470,7 @@ private void updateDataValue(String event, String dataElementUid, String newValu TrackerImportParams.builder().build(), TrackerObjects.builder().events(List.of(e)).build())); }); + manager.clear(); } private void updateEventDates(UID event, Instant newDate) throws IOException { diff --git a/dhis-2/dhis-test-integration/src/test/java/org/hisp/dhis/tracker/imports/bundle/EnrollmentImportTest.java b/dhis-2/dhis-test-integration/src/test/java/org/hisp/dhis/tracker/imports/bundle/EnrollmentImportTest.java index 80683d649780..b1b508cadf98 100644 --- a/dhis-2/dhis-test-integration/src/test/java/org/hisp/dhis/tracker/imports/bundle/EnrollmentImportTest.java +++ b/dhis-2/dhis-test-integration/src/test/java/org/hisp/dhis/tracker/imports/bundle/EnrollmentImportTest.java @@ -42,6 +42,7 @@ import java.util.List; import java.util.Map; import java.util.stream.Stream; +import org.hisp.dhis.dbms.DbmsManager; import org.hisp.dhis.feedback.ForbiddenException; import org.hisp.dhis.feedback.NotFoundException; import org.hisp.dhis.program.EnrollmentStatus; @@ -76,6 +77,7 @@ class EnrollmentImportTest extends PostgresIntegrationTestBase { @Autowired private NamedParameterJdbcTemplate jdbcTemplate; private User importUser; + @Autowired private DbmsManager dbmsManager; @BeforeAll void setUp() throws IOException { @@ -88,13 +90,15 @@ void setUp() throws IOException { @ParameterizedTest @MethodSource("statuses") void shouldCorrectlyPopulateCompletedDataWhenCreatingAnEnrollment(EnrollmentStatus status) - throws IOException, ForbiddenException, NotFoundException { + throws IOException, NotFoundException { TrackerImportParams params = TrackerImportParams.builder().build(); TrackerObjects trackerObjects = testSetup.fromJson("tracker/te_enrollment_event.json"); trackerObjects.getEnrollments().get(0).setStatus(status); ImportReport importReport = trackerImportService.importTracker(params, trackerObjects); + dbmsManager.clearSession(); + assertNoErrors(importReport); Enrollment enrollment = diff --git a/dhis-2/dhis-test-integration/src/test/java/org/hisp/dhis/tracker/imports/bundle/EventDataValueTest.java b/dhis-2/dhis-test-integration/src/test/java/org/hisp/dhis/tracker/imports/bundle/EventDataValueTest.java index 8a817f27dde5..f96abe6f98cf 100644 --- a/dhis-2/dhis-test-integration/src/test/java/org/hisp/dhis/tracker/imports/bundle/EventDataValueTest.java +++ b/dhis-2/dhis-test-integration/src/test/java/org/hisp/dhis/tracker/imports/bundle/EventDataValueTest.java @@ -109,6 +109,8 @@ void testEventDataValueUpdate() throws IOException { params.setImportStrategy(TrackerImportStrategy.CREATE_AND_UPDATE); testSetup.importTrackerData("tracker/event_with_updated_data_values.json", params); + manager.clear(); + List updatedEvents = manager.getAll(TrackerEvent.class); assertEquals(1, updatedEvents.size()); TrackerEvent updatedEvent = manager.get(TrackerEvent.class, updatedEvents.get(0).getUid()); diff --git a/dhis-2/dhis-test-integration/src/test/java/org/hisp/dhis/tracker/imports/bundle/LastUpdateImportTest.java b/dhis-2/dhis-test-integration/src/test/java/org/hisp/dhis/tracker/imports/bundle/LastUpdateImportTest.java index e107cc8a5822..a7e473e4f7c2 100644 --- a/dhis-2/dhis-test-integration/src/test/java/org/hisp/dhis/tracker/imports/bundle/LastUpdateImportTest.java +++ b/dhis-2/dhis-test-integration/src/test/java/org/hisp/dhis/tracker/imports/bundle/LastUpdateImportTest.java @@ -130,7 +130,7 @@ void shouldUpdateTrackedEntityWhenTrackedEntityIsUpdated() throws IOException { TrackerImportParams params = TrackerImportParams.builder().importStrategy(TrackerImportStrategy.UPDATE).build(); testSetup.importTrackerData("tracker/one_te.json", params); - + clearSession(); Date lastUpdateAfter = getTrackedEntity().getLastUpdated(); assertTrue( @@ -753,5 +753,6 @@ private void updateAttributeValue(String attribute, String attributeValue) throw ImportReport report = trackerImportService.importTracker(params, trackerObjects); assertNoErrors(report); + dbmsManager.clearSession(); } } diff --git a/dhis-2/dhis-test-integration/src/test/java/org/hisp/dhis/tracker/imports/bundle/OwnershipTest.java b/dhis-2/dhis-test-integration/src/test/java/org/hisp/dhis/tracker/imports/bundle/OwnershipTest.java index 22ddbb46ca0e..b3a40d102655 100644 --- a/dhis-2/dhis-test-integration/src/test/java/org/hisp/dhis/tracker/imports/bundle/OwnershipTest.java +++ b/dhis-2/dhis-test-integration/src/test/java/org/hisp/dhis/tracker/imports/bundle/OwnershipTest.java @@ -194,6 +194,7 @@ void testUpdateEnrollment() throws IOException { params.setImportStrategy(TrackerImportStrategy.CREATE_AND_UPDATE); ImportReport updatedReport = trackerImportService.importTracker(params, trackerObjects); manager.flush(); + manager.clear(); assertNoErrors(updatedReport); assertEquals(1, updatedReport.getStats().getUpdated()); enrollments = manager.getAll(Enrollment.class); diff --git a/dhis-2/dhis-test-integration/src/test/java/org/hisp/dhis/tracker/imports/programrule/ProgramRuleAssignActionTest.java b/dhis-2/dhis-test-integration/src/test/java/org/hisp/dhis/tracker/imports/programrule/ProgramRuleAssignActionTest.java index e8490e185953..7433533c756c 100644 --- a/dhis-2/dhis-test-integration/src/test/java/org/hisp/dhis/tracker/imports/programrule/ProgramRuleAssignActionTest.java +++ b/dhis-2/dhis-test-integration/src/test/java/org/hisp/dhis/tracker/imports/programrule/ProgramRuleAssignActionTest.java @@ -235,6 +235,8 @@ void shouldImportEventAndCorrectlyAssignPreviousEventDataValue( ImportReport importReport = trackerImportService.importTracker(params, trackerObjects); + manager.clear(); + List firstEventDataValues = getValueForAssignedDataElement(firstEventUid); List secondEventDataValues = getValueForAssignedDataElement(secondEventUid); List thirdEventDataValues = getValueForAssignedDataElement(thirdEventUid); diff --git a/dhis-2/dhis-test-integration/src/test/java/org/hisp/dhis/tracker/imports/validation/EventImportValidationTest.java b/dhis-2/dhis-test-integration/src/test/java/org/hisp/dhis/tracker/imports/validation/EventImportValidationTest.java index 005a459fb9a2..25f5bf4ccc68 100644 --- a/dhis-2/dhis-test-integration/src/test/java/org/hisp/dhis/tracker/imports/validation/EventImportValidationTest.java +++ b/dhis-2/dhis-test-integration/src/test/java/org/hisp/dhis/tracker/imports/validation/EventImportValidationTest.java @@ -368,6 +368,7 @@ void testValidateAndAddNotesToUpdatedEvent() throws IOException { // When -> Update the event and adds 3 more notes ImportReport importReport = createEvent("tracker/validations/events-with-notes-update-data.json"); + manager.clear(); // Then final TrackerEvent event = getEventFromReport(importReport); assertThat(event.getNotes(), hasSize(6)); diff --git a/dhis-2/dhis-test-web-api/src/test/java/org/hisp/dhis/webapi/controller/tracker/export/event/EventsExportChangeLogsControllerTest.java b/dhis-2/dhis-test-web-api/src/test/java/org/hisp/dhis/webapi/controller/tracker/export/event/EventsExportChangeLogsControllerTest.java index c55485e7c3ad..30f4a175d2e5 100644 --- a/dhis-2/dhis-test-web-api/src/test/java/org/hisp/dhis/webapi/controller/tracker/export/event/EventsExportChangeLogsControllerTest.java +++ b/dhis-2/dhis-test-web-api/src/test/java/org/hisp/dhis/webapi/controller/tracker/export/event/EventsExportChangeLogsControllerTest.java @@ -383,23 +383,41 @@ void shouldGetEventChangeLogsWithSimpleFieldsFilter() { } private void updateDataValue(String value) { + String body = createDataValueJson(event, value); + startNewRequestSession(); JsonWebMessage importResponse = - POST("/tracker?async=false&importStrategy=UPDATE", createDataValueJson(event, value)) + POST("/tracker?async=false&importStrategy=UPDATE", body) .content(HttpStatus.OK) .as(JsonWebMessage.class); assertEquals(HttpStatus.OK.toString(), importResponse.getStatus()); } private void updateScheduledAtEventField(String value) { + String body = createScheduledAtEventFieldJson(event, value); + startNewRequestSession(); JsonWebMessage importResponse = - POST( - "/tracker?async=false&importStrategy=UPDATE", - createScheduledAtEventFieldJson(event, value)) + POST("/tracker?async=false&importStrategy=UPDATE", body) .content(HttpStatus.OK) .as(JsonWebMessage.class); assertEquals(HttpStatus.OK.toString(), importResponse.getStatus()); } + /** + * Simulates a fresh per-request {@code EntityManager}. {@code /api/tracker/**} is excluded from + * the {@code ConditionalOpenEntityManagerInViewFilter} (OSIV is deliberately off for tracker), so + * its Hibernate session is transaction-scoped: it is opened for the import's + * {@code @Transactional} boundary and closed when that transaction completes, so each {@code + * /tracker} request starts with a fresh persistence context. These MockMvc tests otherwise share + * one thread-bound session across imports; since the tracker importer writes via JDBC (bypassing + * Hibernate), a previous import's now-stale managed entity would linger in the L1 cache and be + * returned to the next import's preheat. Flushing then clearing the session between imports + * reproduces the production per-request isolation. + */ + private void startNewRequestSession() { + dbmsManager.flushSession(); + dbmsManager.clearSession(); + } + private TrackedEntity trackedEntity() { TrackedEntity te = trackedEntity(orgUnit); manager.save(te, false); diff --git a/dhis-2/dhis-test-web-api/src/test/java/org/hisp/dhis/webapi/controller/tracker/export/singleevent/SingleEventsExportChangeLogsControllerTest.java b/dhis-2/dhis-test-web-api/src/test/java/org/hisp/dhis/webapi/controller/tracker/export/singleevent/SingleEventsExportChangeLogsControllerTest.java index bc2a98a463fb..c73cb6959336 100644 --- a/dhis-2/dhis-test-web-api/src/test/java/org/hisp/dhis/webapi/controller/tracker/export/singleevent/SingleEventsExportChangeLogsControllerTest.java +++ b/dhis-2/dhis-test-web-api/src/test/java/org/hisp/dhis/webapi/controller/tracker/export/singleevent/SingleEventsExportChangeLogsControllerTest.java @@ -364,23 +364,38 @@ void shouldGetEventChangeLogsWithSimpleFieldsFilter() { } private void updateDataValue(String value) { + String body = createDataValueJson(event, value); + startNewRequestSession(); JsonWebMessage importResponse = - POST("/tracker?async=false&importStrategy=UPDATE", createDataValueJson(event, value)) + POST("/tracker?async=false&importStrategy=UPDATE", body) .content(HttpStatus.OK) .as(JsonWebMessage.class); assertEquals(HttpStatus.OK.toString(), importResponse.getStatus()); } private void updateScheduledAtEventField(String value) { + String body = createScheduledAtEventFieldJson(event, value); + startNewRequestSession(); JsonWebMessage importResponse = - POST( - "/tracker?async=false&importStrategy=UPDATE", - createScheduledAtEventFieldJson(event, value)) + POST("/tracker?async=false&importStrategy=UPDATE", body) .content(HttpStatus.OK) .as(JsonWebMessage.class); assertEquals(HttpStatus.OK.toString(), importResponse.getStatus()); } + /** + * Simulates a fresh per-request {@code EntityManager}. Production registers an {@code + * OpenEntityManagerInViewFilter}, so every {@code /tracker} request gets its own Hibernate + * session. These MockMvc tests otherwise share one thread-bound session across imports; since the + * tracker importer writes via JDBC (bypassing Hibernate), a previous import's now-stale managed + * entity would linger in the L1 cache and be returned to the next import's preheat. Flushing then + * clearing the session between imports reproduces the production per-request isolation. + */ + private void startNewRequestSession() { + dbmsManager.flushSession(); + dbmsManager.clearSession(); + } + private SingleEvent event() { SingleEvent eventA = new SingleEvent(); eventA.setProgramStage(programStage); diff --git a/dhis-2/dhis-test-web-api/src/test/java/org/hisp/dhis/webapi/controller/tracker/export/trackerevent/TrackerEventsExportChangeLogsControllerTest.java b/dhis-2/dhis-test-web-api/src/test/java/org/hisp/dhis/webapi/controller/tracker/export/trackerevent/TrackerEventsExportChangeLogsControllerTest.java index a3b6e4eb95e5..dabb43d320be 100644 --- a/dhis-2/dhis-test-web-api/src/test/java/org/hisp/dhis/webapi/controller/tracker/export/trackerevent/TrackerEventsExportChangeLogsControllerTest.java +++ b/dhis-2/dhis-test-web-api/src/test/java/org/hisp/dhis/webapi/controller/tracker/export/trackerevent/TrackerEventsExportChangeLogsControllerTest.java @@ -389,23 +389,41 @@ void shouldGetEventChangeLogsWithSimpleFieldsFilter() { } private void updateDataValue(String value) { + String body = createDataValueJson(event, value); + startNewRequestSession(); JsonWebMessage importResponse = - POST("/tracker?async=false&importStrategy=UPDATE", createDataValueJson(event, value)) + POST("/tracker?async=false&importStrategy=UPDATE", body) .content(HttpStatus.OK) .as(JsonWebMessage.class); assertEquals(HttpStatus.OK.toString(), importResponse.getStatus()); } private void updateScheduledAtEventField(String value) { + String body = createScheduledAtEventFieldJson(event, value); + startNewRequestSession(); JsonWebMessage importResponse = - POST( - "/tracker?async=false&importStrategy=UPDATE", - createScheduledAtEventFieldJson(event, value)) + POST("/tracker?async=false&importStrategy=UPDATE", body) .content(HttpStatus.OK) .as(JsonWebMessage.class); assertEquals(HttpStatus.OK.toString(), importResponse.getStatus()); } + /** + * Simulates a fresh per-request {@code EntityManager}. {@code /api/tracker/**} is excluded from + * the {@code ConditionalOpenEntityManagerInViewFilter} (OSIV is deliberately off for tracker), so + * its Hibernate session is transaction-scoped: it is opened for the import's + * {@code @Transactional} boundary and closed when that transaction completes, so each {@code + * /tracker} request starts with a fresh persistence context. These MockMvc tests otherwise share + * one thread-bound session across imports; since the tracker importer writes via JDBC (bypassing + * Hibernate), a previous import's now-stale managed entity would linger in the L1 cache and be + * returned to the next import's preheat. Flushing then clearing the session between imports + * reproduces the production per-request isolation. + */ + private void startNewRequestSession() { + dbmsManager.flushSession(); + dbmsManager.clearSession(); + } + private TrackerEvent event() { TrackerEvent eventA = new TrackerEvent(); eventA.setProgramStage(programStage); diff --git a/dhis-2/dhis-test-web-api/src/test/java/org/hisp/dhis/webapi/controller/tracker/imports/TrackerEventSMSTest.java b/dhis-2/dhis-test-web-api/src/test/java/org/hisp/dhis/webapi/controller/tracker/imports/TrackerEventSMSTest.java index 8ea0bfd68ad4..da50450b5b6d 100644 --- a/dhis-2/dhis-test-web-api/src/test/java/org/hisp/dhis/webapi/controller/tracker/imports/TrackerEventSMSTest.java +++ b/dhis-2/dhis-test-web-api/src/test/java/org/hisp/dhis/webapi/controller/tracker/imports/TrackerEventSMSTest.java @@ -866,7 +866,9 @@ private TrackerEvent event(Enrollment enrollment) { event.setProgramStage(trackerProgramStage); event.setOrganisationUnit(enrollment.getOrganisationUnit()); event.setAttributeOptionCombo(coc); - event.setOccurredDate(new Date()); + // SMS submissions encode dates with second precision, so the occurred date must not carry + // milliseconds or it won't survive the SMS encode/decode round-trip in shouldUpdateEvent(). + event.setOccurredDate(DateUtils.getDate(2024, 9, 2, 10, 15)); event.setAutoFields(); manager.save(event); return event; diff --git a/dhis-2/dhis-tracker/src/main/java/org/hisp/dhis/tracker/imports/bundle/persister/AbstractTrackerPersister.java b/dhis-2/dhis-tracker/src/main/java/org/hisp/dhis/tracker/imports/bundle/persister/AbstractTrackerPersister.java index eef95dacb0c3..4daa1295ab4e 100644 --- a/dhis-2/dhis-tracker/src/main/java/org/hisp/dhis/tracker/imports/bundle/persister/AbstractTrackerPersister.java +++ b/dhis-2/dhis-tracker/src/main/java/org/hisp/dhis/tracker/imports/bundle/persister/AbstractTrackerPersister.java @@ -35,8 +35,6 @@ import static org.hisp.dhis.changelog.ChangeLogType.UPDATE; import com.fasterxml.jackson.databind.ObjectMapper; -import jakarta.persistence.EntityManager; -import jakarta.persistence.PersistenceContext; import java.sql.Connection; import java.sql.PreparedStatement; import java.sql.ResultSet; @@ -53,7 +51,6 @@ import java.util.Objects; import java.util.Optional; import java.util.Set; -import java.util.function.Function; import java.util.stream.Collectors; import javax.sql.DataSource; import lombok.AccessLevel; @@ -103,7 +100,10 @@ public abstract class AbstractTrackerPersister payloadAttributes, TrackedEntity trackedEntity, @@ -514,37 +536,27 @@ protected void handleTrackedEntityAttributeValues( } TrackerIdSchemeParams idSchemes = preheat.getIdSchemes(); - // TODO [Phase 6 / DHIS2-21378]: drop this JPQL query and the batch overlay below once - // EntityWriteBatch is shared across persisters and writes go through JDBC instead of the - // EntityManager. At that point the batch itself is the source of truth for "what has already - // been staged for this TE", and DB rows can be loaded eagerly by the preheat -- no per-call - // round-trip needed here. Until then: build the lookup from the EntityManager rather than - // the lazy TE collection. The collection is a Hibernate PersistentSet whose in-memory state - // diverges from the persistence context once a write to a TE attribute has been deferred - // (staged in EntityWriteBatch) and flushed by a prior persister -- the same logical TEAV - // (composite key trackedentityid + trackedentityattributeid) can appear on a TrackedEntity - // payload and on an Enrollment payload, and querying the EM here sees both DB rows and TEAVs - // already attached to the session, preventing a duplicate em.persist that would throw - // EntityExistsException. The batch overlay catches the within-persister case (e.g. two - // enrollments under the same TE both carrying the same attribute) where the second occurrence - // would otherwise produce a fresh instance with the same composite key. A TE staged for - // insert in this batch has no DB row yet -- whether or not its id is set (Phase 4a pre- - // allocates the id from the sequence) -- so the JPQL would always return empty and the lookup - // can be skipped. + // TODO [Phase 6 / DHIS2-21378]: drop this lookup and the batch overlay below once + // EntityWriteBatch is shared across persisters (it is currently created per persist() call). + // At that point the batch itself is the source of truth for "what has already been staged for + // this TE", and DB rows can be loaded eagerly by the preheat -- no per-call round-trip needed + // here. Until then: read the existing values straight from the DB via JDBC on the + // transaction-bound connection. TEAV writes also go through JDBC on that same connection (see + // EntityWriteBatch), so a TEAV inserted/updated by a prior persister in this same import is an + // uncommitted row that is nonetheless visible to this read because both run in the same + // transaction -- the same logical TEAV (composite key + // trackedentityid + trackedentityattributeid) can appear on a TrackedEntity payload and on an + // Enrollment payload, and finding the existing row here routes the second occurrence to an + // UPDATE instead of a duplicate INSERT (which would violate the composite PK). The batch + // overlay catches the within-persister case (e.g. two enrollments under the same TE both + // carrying the same attribute) where the second occurrence would otherwise produce a fresh + // instance with the same composite key. A TE staged for insert in this batch has no DB row yet + // -- whether or not its id is set (Phase 4a pre-allocates the id from the sequence) -- so the + // read would always return empty and can be skipped. Map attributeValueById = batch.isStagedAsInsert(trackedEntity) ? new HashMap<>() - : entityManager - .createQuery( - "select v from TrackedEntityAttributeValue v where v.trackedEntity = :te", - TrackedEntityAttributeValue.class) - .setParameter("te", trackedEntity) - .getResultList() - .stream() - .collect( - Collectors.toMap( - teav -> idSchemes.toMetadataIdentifier(teav.getAttribute()), - Function.identity())); + : loadExistingAttributeValues(connection, preheat, idSchemes, trackedEntity); batch .stagedFor(trackedEntity) .forEach( @@ -579,6 +591,53 @@ protected void handleTrackedEntityAttributeValues( }); } + /** + * Reads the tracked entity's existing attribute values straight from the DB via JDBC on the + * transaction-bound connection held by {@link #persist}, keyed by the attribute's {@link + * MetadataIdentifier}. Replaces the former JPQL lookup so the persister no longer needs an {@code + * EntityManager}; the values are plain {@link TrackedEntityAttributeValue} instances (not + * Hibernate-managed), matching what the {@code value} property mapping ({@code + * access="property"}) produces on load. Attributes are resolved from the preheat by primary key, + * which is independent of the import's configured idScheme -- the preheat lookup maps are keyed + * by the idScheme identifier, so resolving by the UID read from the DB would silently miss every + * existing value under idScheme CODE/NAME/ATTRIBUTE, turning updates into duplicate INSERTs. + * Existing values for attributes not in the preheat are skipped -- the caller only probes + * attributes that appear in the payload, and those are always preheated. + */ + private Map loadExistingAttributeValues( + Connection connection, + TrackerPreheat preheat, + TrackerIdSchemeParams idSchemes, + TrackedEntity trackedEntity) { + Map attributesById = + preheat.getAll(TrackedEntityAttribute.class).stream() + .collect(Collectors.toMap(TrackedEntityAttribute::getId, a -> a, (a, b) -> a)); + Map attributeValueById = new HashMap<>(); + try (PreparedStatement ps = connection.prepareStatement(EXISTING_ATTRIBUTE_VALUES_SQL)) { + ps.setLong(1, trackedEntity.getId()); + try (ResultSet rs = ps.executeQuery()) { + while (rs.next()) { + TrackedEntityAttribute attribute = + attributesById.get(rs.getLong("trackedentityattributeid")); + if (attribute == null) { + continue; + } + TrackedEntityAttributeValue value = + new TrackedEntityAttributeValue() + .setAttribute(attribute) + .setTrackedEntity(trackedEntity) + .setValue(rs.getString("value")); + attributeValueById.put(idSchemes.toMetadataIdentifier(attribute), value); + } + } + } catch (SQLException e) { + throw new PersistenceException( + "Failed to load existing attribute values for tracked entity " + trackedEntity.getUid(), + e); + } + return attributeValueById; + } + private void saveOrUpdateAttributeValue( TrackerPreheat preheat, TrackedEntity trackedEntity, diff --git a/dhis-2/dhis-tracker/src/main/java/org/hisp/dhis/tracker/imports/bundle/persister/EnrollmentPersister.java b/dhis-2/dhis-tracker/src/main/java/org/hisp/dhis/tracker/imports/bundle/persister/EnrollmentPersister.java index 9de02fb60ef2..b0a4a35b9bfe 100644 --- a/dhis-2/dhis-tracker/src/main/java/org/hisp/dhis/tracker/imports/bundle/persister/EnrollmentPersister.java +++ b/dhis-2/dhis-tracker/src/main/java/org/hisp/dhis/tracker/imports/bundle/persister/EnrollmentPersister.java @@ -30,6 +30,7 @@ package org.hisp.dhis.tracker.imports.bundle.persister; import com.fasterxml.jackson.databind.ObjectMapper; +import java.sql.Connection; import java.util.EnumSet; import java.util.List; import java.util.Set; @@ -76,6 +77,7 @@ protected String sequenceName() { @Override protected void updateAttributes( + Connection connection, TrackerPreheat preheat, org.hisp.dhis.tracker.imports.domain.Enrollment enrollment, Enrollment enrollmentToPersist, @@ -83,6 +85,7 @@ protected void updateAttributes( ChangeLogAccumulator changeLogs, EntityWriteBatch batch) { handleTrackedEntityAttributeValues( + connection, preheat, enrollment.getAttributes(), enrollmentToPersist.getTrackedEntity(), diff --git a/dhis-2/dhis-tracker/src/main/java/org/hisp/dhis/tracker/imports/bundle/persister/EntityWriteBatch.java b/dhis-2/dhis-tracker/src/main/java/org/hisp/dhis/tracker/imports/bundle/persister/EntityWriteBatch.java index 9f865587ff7f..126f20f63481 100644 --- a/dhis-2/dhis-tracker/src/main/java/org/hisp/dhis/tracker/imports/bundle/persister/EntityWriteBatch.java +++ b/dhis-2/dhis-tracker/src/main/java/org/hisp/dhis/tracker/imports/bundle/persister/EntityWriteBatch.java @@ -33,7 +33,6 @@ import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.ObjectWriter; -import jakarta.persistence.EntityManager; import java.io.IOException; import java.io.StringWriter; import java.sql.Connection; @@ -99,14 +98,20 @@ * the two tables, which is not deferrable in the live schema. Both tables use the shared * {@code hibernate_sequence}. Relationship is INSERT-only -- the persister rejects UPDATE * strategy upstream. - *
  • TEAVs continue to delegate to {@link EntityManager} until their Phase 6 turn. + *
  • TrackedEntityAttributeValues are flushed via JDBC against {@code + * trackedentityattributevalue} -- a multi-row INSERT, a single unnest UPDATE keyed on the + * composite ({@code trackedentityid}, {@code trackedentityattributeid}) PK, and a single + * unnest DELETE on the same key. Confidential attributes and the {@code encryptedvalue} + * column were removed in DHIS2-21518, so the value is written as plain text in {@code value}. * * - *

    Top-level entities are flushed before TEAVs so that ids are set (and rows present in the DB or - * in the Hibernate persistence context) before any TEAV that references them. Within top-level - * entities the order (TE inserts, TE updates, Enrollment, TrackerEvent, SingleEvent, Relationship) - * matches the persister call order enforced by {@code DefaultTrackerBundleService.commit()} and is - * FK-safe. + *

    All reads and writes in the import path are JDBC, so no Hibernate-managed entities of these + * types are loaded during commit and there is no L1 cache to reconcile after the writes. + * + *

    Top-level entities are flushed before TEAVs so that ids are set and rows are present in the DB + * before any TEAV that references them. Within top-level entities the order (TE inserts, TE + * updates, Enrollment, TrackerEvent, SingleEvent, Relationship) matches the persister call order + * enforced by {@code DefaultTrackerBundleService.commit()} and is FK-safe. * *

    Each {@link AbstractTrackerPersister#persist} call creates its own batch, so in practice only * one top-level entity type list is populated per batch (the persister's own type) alongside any @@ -398,6 +403,40 @@ class EntityWriteBatch { + " unnest(?::bigint[]) as to_id" + " ) v where r.relationshipid = v.relationshipid"; + // TrackedEntityAttributeValue has a composite PK (trackedentityid, trackedentityattributeid) and + // no sequence, so no id pre-allocation is needed. Confidential attributes (and the encrypted + // value column) were removed in DHIS2-21518, so the value is stored as plain text in `value`. + private static final String TEAV_INSERT_PREFIX = + "insert into trackedentityattributevalue (" + + "trackedentityid, trackedentityattributeid, created, lastupdated," + + " value, updatedby) values "; + private static final String TEAV_INSERT_ROW = "(?, ?, ?, ?, ?, ?)"; + + // `created` is insert-only and deliberately excluded from the UPDATE column set. + private static final String TEAV_UPDATE_SQL = + "update trackedentityattributevalue teav set" + + " lastupdated = v.lastupdated," + + " value = v.value," + + " updatedby = v.updatedby" + + " from ( select" + + " unnest(?::bigint[]) as trackedentityid," + + " unnest(?::bigint[]) as trackedentityattributeid," + + " unnest(?::timestamptz[]) as lastupdated," + + " unnest(?::text[]) as value," + + " unnest(?::text[]) as updatedby" + + " ) v where teav.trackedentityid = v.trackedentityid" + + " and teav.trackedentityattributeid = v.trackedentityattributeid"; + + // Unnest DELETE on the composite key, mirroring the unnest UPDATE shape rather than the plan's + // IN (VALUES ...) form, to stay consistent with the other batch statements and avoid dynamic SQL. + private static final String TEAV_DELETE_SQL = + "delete from trackedentityattributevalue teav" + + " using ( select" + + " unnest(?::bigint[]) as trackedentityid," + + " unnest(?::bigint[]) as trackedentityattributeid" + + " ) v where teav.trackedentityid = v.trackedentityid" + + " and teav.trackedentityattributeid = v.trackedentityattributeid"; + private final ObjectMapper objectMapper; private final List teInserts = new ArrayList<>(); @@ -521,43 +560,35 @@ void rollbackTo(Mark mark) { } /** - * Applies all staged writes. TrackedEntity and Enrollment writes go through JDBC on {@code conn} - * (including cascade INSERTs into {@code note} / {@code enrollment_notes}); TrackerEvent, - * SingleEvent, Relationship and TEAV writes still delegate to {@code entityManager}. The - * connection must be the one bound to the current Spring-managed transaction so that the JDBC - * statements and any subsequent Hibernate flush execute under the same commit. + * Applies all staged writes via JDBC on {@code conn} (including cascade INSERTs into {@code note} + * and the per-entity notes join tables). The connection must be the one bound to the current + * Spring-managed transaction so the JDBC statements execute under the same commit. All reads and + * writes in the import path are JDBC, so no Hibernate-managed entities of these types are loaded + * during commit and there is no L1 cache to reconcile. */ - void flush(EntityManager entityManager, Connection conn) throws SQLException { + void flush(Connection conn) throws SQLException { if (isEmpty()) { return; } insertTrackedEntities(conn); - updateTrackedEntities(entityManager, conn); + updateTrackedEntities(conn); insertEnrollments(conn); - updateEnrollments(entityManager, conn); + updateEnrollments(conn); insertEnrollmentNotes(conn); insertTrackerEvents(conn); updateTrackerEvents(conn); insertTrackerEventNotes(conn); - refreshUpdatedTrackerEvents(entityManager); insertSingleEvents(conn); updateSingleEvents(conn); insertSingleEventNotes(conn); - refreshUpdatedSingleEvents(entityManager); insertRelationships(conn); insertRelationshipItems(conn); updateRelationshipFromTo(conn); - for (TrackedEntityAttributeValue v : teavInserts) { - entityManager.persist(v); - } - for (TrackedEntityAttributeValue v : teavUpdates) { - entityManager.merge(v); - } - for (TrackedEntityAttributeValue v : teavDeletes) { - entityManager.remove(entityManager.contains(v) ? v : entityManager.merge(v)); - } + insertTeavs(conn); + updateTeavs(conn); + deleteTeavs(conn); teInserts.clear(); teUpdates.clear(); @@ -573,8 +604,7 @@ void flush(EntityManager entityManager, Connection conn) throws SQLException { teavDeletes.clear(); } - private void updateTrackedEntities(EntityManager entityManager, Connection conn) - throws SQLException { + private void updateTrackedEntities(Connection conn) throws SQLException { if (teUpdates.isEmpty()) { return; } @@ -622,17 +652,6 @@ private void updateTrackedEntities(EntityManager entityManager, Connection conn) ps.executeUpdate(); } } - // The TrackedEntity entities passed in here are detached copies from the preheat. The - // Hibernate persistence context separately holds the entities loaded by the preheat's queries - // (M_managed), and they still carry their pre-update state. Detach them so any subsequent - // JPQL read in this session reloads the fresh DB row instead of returning the stale L1 - // instance. - for (TrackedEntity te : teUpdates) { - TrackedEntity managed = entityManager.find(TrackedEntity.class, te.getId()); - if (managed != null) { - entityManager.detach(managed); - } - } } private void insertTrackedEntities(Connection conn) throws SQLException { @@ -748,7 +767,7 @@ private int bindEnrollmentRow(PreparedStatement ps, int p, Enrollment e) throws return p; } - private void updateEnrollments(EntityManager entityManager, Connection conn) throws SQLException { + private void updateEnrollments(Connection conn) throws SQLException { if (enrollmentUpdates.isEmpty()) { return; } @@ -814,15 +833,6 @@ private void updateEnrollments(EntityManager entityManager, Connection conn) thr ps.executeUpdate(); } } - // Same L1 staleness fix as updateTrackedEntities -- the JDBC UPDATE bypasses Hibernate, so the - // preheat-loaded managed Enrollment in the persistence context still carries the pre-update - // state. Detach it so any subsequent JPQL read reloads the fresh DB row. - for (Enrollment e : enrollmentUpdates) { - Enrollment managed = entityManager.find(Enrollment.class, e.getId()); - if (managed != null) { - entityManager.detach(managed); - } - } } private void insertTrackerEvents(Connection conn) throws SQLException { @@ -959,23 +969,6 @@ private void updateTrackerEvents(Connection conn) throws SQLException { ps.executeUpdate(); } } - // L1 staleness is fixed by refreshUpdatedTrackerEvents() called from flush() AFTER - // insertTrackerEventNotes so the refreshed entity sees the just-written notes join rows. - } - - /** - * Reload the L1-cached TrackerEvent entities affected by {@link #updateTrackerEvents} from DB. - * Must run AFTER {@link #insertTrackerEventNotes} so the refreshed entity's eager-fetched notes - * collection picks up the newly-appended notes. Mirrors the side effect of the pre-migration - * {@code em.merge(detachedDto)} path that updated the L1-managed instance in place. - */ - private void refreshUpdatedTrackerEvents(EntityManager entityManager) { - for (TrackerEvent e : trackerEventUpdates) { - TrackerEvent managed = entityManager.find(TrackerEvent.class, e.getId()); - if (managed != null) { - entityManager.refresh(managed); - } - } } /** Shared shape between {@code enrollment_notes} and {@code trackerevent_notes} cascade rows. */ @@ -1276,17 +1269,6 @@ private void updateSingleEvents(Connection conn) throws SQLException { ps.executeUpdate(); } } - // L1 staleness is fixed by refreshUpdatedSingleEvents() called from flush() AFTER - // insertSingleEventNotes -- same reasoning as refreshUpdatedTrackerEvents. - } - - private void refreshUpdatedSingleEvents(EntityManager entityManager) { - for (SingleEvent e : singleEventUpdates) { - SingleEvent managed = entityManager.find(SingleEvent.class, e.getId()); - if (managed != null) { - entityManager.refresh(managed); - } - } } /** Parallel of {@link EnrollmentNoteToInsert} for the {@code singleevent_notes} join table. */ @@ -1515,6 +1497,89 @@ private void updateRelationshipFromTo(Connection conn) throws SQLException { } } + private void insertTeavs(Connection conn) throws SQLException { + if (teavInserts.isEmpty()) { + return; + } + for (int from = 0; from < teavInserts.size(); from += BATCH_SIZE) { + int to = Math.min(from + BATCH_SIZE, teavInserts.size()); + List chunk = teavInserts.subList(from, to); + String sql = buildMultiRowInsertSql(TEAV_INSERT_PREFIX, TEAV_INSERT_ROW, chunk.size()); + try (PreparedStatement ps = conn.prepareStatement(sql)) { + int p = 1; + for (TrackedEntityAttributeValue v : chunk) { + ps.setLong(p++, v.getTrackedEntity().getId()); + ps.setLong(p++, v.getAttribute().getId()); + ps.setTimestamp(p++, toTimestamp(v.getCreated())); + ps.setTimestamp(p++, toTimestamp(v.getLastUpdated())); + setNullableString(ps, p++, v.getValue()); + setNullableString(ps, p++, v.getUpdatedBy()); + } + ps.executeUpdate(); + } + } + } + + private void updateTeavs(Connection conn) throws SQLException { + if (teavUpdates.isEmpty()) { + return; + } + for (int from = 0; from < teavUpdates.size(); from += BATCH_SIZE) { + int to = Math.min(from + BATCH_SIZE, teavUpdates.size()); + List chunk = teavUpdates.subList(from, to); + int n = chunk.size(); + + Long[] trackedEntityIds = new Long[n]; + Long[] attributeIds = new Long[n]; + Timestamp[] lastUpdated = new Timestamp[n]; + String[] value = new String[n]; + String[] updatedBy = new String[n]; + + for (int i = 0; i < n; i++) { + TrackedEntityAttributeValue v = chunk.get(i); + trackedEntityIds[i] = v.getTrackedEntity().getId(); + attributeIds[i] = v.getAttribute().getId(); + lastUpdated[i] = toTimestamp(v.getLastUpdated()); + value[i] = v.getValue(); + updatedBy[i] = v.getUpdatedBy(); + } + + try (PreparedStatement ps = conn.prepareStatement(TEAV_UPDATE_SQL)) { + ps.setArray(1, conn.createArrayOf("bigint", trackedEntityIds)); + ps.setArray(2, conn.createArrayOf("bigint", attributeIds)); + ps.setArray(3, conn.createArrayOf("timestamptz", lastUpdated)); + ps.setArray(4, conn.createArrayOf("text", value)); + ps.setArray(5, conn.createArrayOf("text", updatedBy)); + ps.executeUpdate(); + } + } + } + + private void deleteTeavs(Connection conn) throws SQLException { + if (teavDeletes.isEmpty()) { + return; + } + for (int from = 0; from < teavDeletes.size(); from += BATCH_SIZE) { + int to = Math.min(from + BATCH_SIZE, teavDeletes.size()); + List chunk = teavDeletes.subList(from, to); + int n = chunk.size(); + + Long[] trackedEntityIds = new Long[n]; + Long[] attributeIds = new Long[n]; + for (int i = 0; i < n; i++) { + TrackedEntityAttributeValue v = chunk.get(i); + trackedEntityIds[i] = v.getTrackedEntity().getId(); + attributeIds[i] = v.getAttribute().getId(); + } + + try (PreparedStatement ps = conn.prepareStatement(TEAV_DELETE_SQL)) { + ps.setArray(1, conn.createArrayOf("bigint", trackedEntityIds)); + ps.setArray(2, conn.createArrayOf("bigint", attributeIds)); + ps.executeUpdate(); + } + } + } + /** * Fetches {@code count} ids from {@code sequenceName} in a single round-trip. The sequence name * is interpolated into the SQL (not a bind parameter) because PostgreSQL's {@code nextval} takes @@ -1568,6 +1633,15 @@ private static void setNullableTimestamp(PreparedStatement ps, int index, Date d } } + private static void setNullableString(PreparedStatement ps, int index, String value) + throws SQLException { + if (value != null) { + ps.setString(index, value); + } else { + ps.setNull(index, Types.VARCHAR); + } + } + private String toJson(UserInfoSnapshot info) throws SQLException { if (info == null) { return null; diff --git a/dhis-2/dhis-tracker/src/main/java/org/hisp/dhis/tracker/imports/bundle/persister/RelationshipPersister.java b/dhis-2/dhis-tracker/src/main/java/org/hisp/dhis/tracker/imports/bundle/persister/RelationshipPersister.java index f2675922c885..7121b6d719cb 100644 --- a/dhis-2/dhis-tracker/src/main/java/org/hisp/dhis/tracker/imports/bundle/persister/RelationshipPersister.java +++ b/dhis-2/dhis-tracker/src/main/java/org/hisp/dhis/tracker/imports/bundle/persister/RelationshipPersister.java @@ -30,6 +30,7 @@ package org.hisp.dhis.tracker.imports.bundle.persister; import com.fasterxml.jackson.databind.ObjectMapper; +import java.sql.Connection; import java.util.List; import java.util.Set; import javax.sql.DataSource; @@ -82,6 +83,7 @@ protected org.hisp.dhis.tracker.model.Relationship convert( @Override protected void updateAttributes( + Connection connection, TrackerPreheat preheat, Relationship trackerDto, org.hisp.dhis.tracker.model.Relationship hibernateEntity, diff --git a/dhis-2/dhis-tracker/src/main/java/org/hisp/dhis/tracker/imports/bundle/persister/SingleEventPersister.java b/dhis-2/dhis-tracker/src/main/java/org/hisp/dhis/tracker/imports/bundle/persister/SingleEventPersister.java index 2b78b3b4be86..bb1accde9619 100644 --- a/dhis-2/dhis-tracker/src/main/java/org/hisp/dhis/tracker/imports/bundle/persister/SingleEventPersister.java +++ b/dhis-2/dhis-tracker/src/main/java/org/hisp/dhis/tracker/imports/bundle/persister/SingleEventPersister.java @@ -34,6 +34,7 @@ import static org.hisp.dhis.changelog.ChangeLogType.UPDATE; import com.fasterxml.jackson.databind.ObjectMapper; +import java.sql.Connection; import java.util.Collections; import java.util.Date; import java.util.EnumSet; @@ -160,6 +161,7 @@ protected List getByType(Track @Override protected void updateAttributes( + Connection connection, TrackerPreheat preheat, org.hisp.dhis.tracker.imports.domain.SingleEvent event, SingleEvent hibernateEntity, diff --git a/dhis-2/dhis-tracker/src/main/java/org/hisp/dhis/tracker/imports/bundle/persister/TrackedEntityPersister.java b/dhis-2/dhis-tracker/src/main/java/org/hisp/dhis/tracker/imports/bundle/persister/TrackedEntityPersister.java index eb17687c0b0f..dcfbde241156 100644 --- a/dhis-2/dhis-tracker/src/main/java/org/hisp/dhis/tracker/imports/bundle/persister/TrackedEntityPersister.java +++ b/dhis-2/dhis-tracker/src/main/java/org/hisp/dhis/tracker/imports/bundle/persister/TrackedEntityPersister.java @@ -30,6 +30,7 @@ package org.hisp.dhis.tracker.imports.bundle.persister; import com.fasterxml.jackson.databind.ObjectMapper; +import java.sql.Connection; import java.util.Collections; import java.util.List; import java.util.Set; @@ -68,6 +69,7 @@ protected String sequenceName() { @Override protected void updateAttributes( + Connection connection, TrackerPreheat preheat, org.hisp.dhis.tracker.imports.domain.TrackedEntity trackerDto, TrackedEntity te, @@ -75,7 +77,7 @@ protected void updateAttributes( ChangeLogAccumulator changeLogs, EntityWriteBatch batch) { handleTrackedEntityAttributeValues( - preheat, trackerDto.getAttributes(), te, user, changeLogs, batch); + connection, preheat, trackerDto.getAttributes(), te, user, changeLogs, batch); } @Override diff --git a/dhis-2/dhis-tracker/src/main/java/org/hisp/dhis/tracker/imports/bundle/persister/TrackerEventPersister.java b/dhis-2/dhis-tracker/src/main/java/org/hisp/dhis/tracker/imports/bundle/persister/TrackerEventPersister.java index f19620b0dfc7..a30127ea191d 100644 --- a/dhis-2/dhis-tracker/src/main/java/org/hisp/dhis/tracker/imports/bundle/persister/TrackerEventPersister.java +++ b/dhis-2/dhis-tracker/src/main/java/org/hisp/dhis/tracker/imports/bundle/persister/TrackerEventPersister.java @@ -34,6 +34,7 @@ import static org.hisp.dhis.changelog.ChangeLogType.UPDATE; import com.fasterxml.jackson.databind.ObjectMapper; +import java.sql.Connection; import java.util.Collections; import java.util.Date; import java.util.EnumSet; @@ -163,6 +164,7 @@ protected List getByType( @Override protected void updateAttributes( + Connection connection, TrackerPreheat preheat, org.hisp.dhis.tracker.imports.domain.TrackerEvent event, TrackerEvent hibernateEntity,