Skip to content

Commit 21979d7

Browse files
Megh-Shah-08Megh-Shah-08
committed
perf: optimize test case hard-delete relationship cleanup (#27418)
1 parent 285eb8a commit 21979d7

7 files changed

Lines changed: 127 additions & 23 deletions

File tree

‎openmetadata-integration-tests/src/test/java/org/openmetadata/it/tests/TestCaseResourceIT.java‎

Lines changed: 72 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -36,6 +36,7 @@
3636
import org.openmetadata.schema.type.Column;
3737
import org.openmetadata.schema.type.ColumnDataType;
3838
import org.openmetadata.schema.type.EntityHistory;
39+
import org.openmetadata.schema.type.Relationship;
3940
import org.openmetadata.schema.type.TagLabel;
4041
import org.openmetadata.schema.type.csv.CsvImportResult;
4142
import org.openmetadata.schema.utils.JsonUtils;
@@ -45,6 +46,8 @@
4546
import org.openmetadata.sdk.models.ListResponse;
4647
import org.openmetadata.sdk.network.HttpMethod;
4748
import org.openmetadata.sdk.network.RequestOptions;
49+
import org.openmetadata.service.Entity;
50+
import org.openmetadata.service.jdbi3.CollectionDAO;
4851
import org.openmetadata.service.resources.dqtests.TestCaseResource;
4952

5053
/**
@@ -4420,4 +4423,73 @@ void test_patchDisplayName_replaceOnMissingField_succeeds(TestNamespace ns) thro
44204423
TestCase updated = getEntity(created.getId().toString());
44214424
assertEquals("Updated Display Name", updated.getDisplayName());
44224425
}
4426+
4427+
@Test
4428+
void test_testCaseDeleteCleanup(TestNamespace ns) {
4429+
OpenMetadataClient client = SdkClients.adminClient();
4430+
Table table = createTable(ns);
4431+
4432+
// 1. Create a test case
4433+
TestCase testCase =
4434+
TestCaseBuilder.create(client)
4435+
.name(ns.prefix("delete_cleanup"))
4436+
.forTable(table)
4437+
.testDefinition("tableRowCountToEqual")
4438+
.parameter("value", "100")
4439+
.create();
4440+
4441+
// 2. Create multiple resolution statuses (incidents)
4442+
for (int i = 0; i < 3; i++) {
4443+
org.openmetadata.schema.api.tests.CreateTestCaseResolutionStatus status =
4444+
new org.openmetadata.schema.api.tests.CreateTestCaseResolutionStatus();
4445+
status.setTestCaseResolutionStatusType(
4446+
org.openmetadata.schema.tests.type.TestCaseResolutionStatusTypes.Ack);
4447+
status.setTestCaseReference(testCase.getFullyQualifiedName());
4448+
client.testCaseResolutionStatuses().create(status);
4449+
}
4450+
4451+
// 3. Verify relationships exist in DB using JDBI
4452+
// We expect 3 relationships from TestCase TO TestCaseResolutionStatus
4453+
List<CollectionDAO.EntityRelationshipRecord> relationships =
4454+
Entity.getCollectionDAO()
4455+
.relationshipDAO()
4456+
.findTo(testCase.getId(), Entity.TEST_CASE, Relationship.RELATED_TO.ordinal());
4457+
4458+
assertNotNull(relationships);
4459+
long statusCount =
4460+
relationships.stream()
4461+
.filter(r -> r.getType().equals(Entity.TEST_CASE_RESOLUTION_STATUS))
4462+
.count();
4463+
assertTrue(
4464+
statusCount >= 3,
4465+
"There should be at least 3 relationships to resolution statuses before delete");
4466+
4467+
// 4. Hard delete the test case
4468+
java.util.Map<String, String> params = new java.util.HashMap<>();
4469+
params.put("hardDelete", "true");
4470+
client.testCases().delete(testCase.getId().toString(), params);
4471+
4472+
// 5. Verify relationships are cleaned up
4473+
List<CollectionDAO.EntityRelationshipRecord> relationshipsAfter =
4474+
Entity.getCollectionDAO()
4475+
.relationshipDAO()
4476+
.findTo(testCase.getId(), Entity.TEST_CASE, Relationship.RELATED_TO.ordinal());
4477+
4478+
long statusCountAfter =
4479+
relationshipsAfter.stream()
4480+
.filter(r -> r.getType().equals(Entity.TEST_CASE_RESOLUTION_STATUS))
4481+
.count();
4482+
assertEquals(
4483+
0,
4484+
statusCountAfter,
4485+
"All relationships to resolution statuses should be cleaned up after hard delete");
4486+
4487+
// 6. Verify time series records still exist (only relationships are deleted)
4488+
List<String> statusesAfter =
4489+
Entity.getCollectionDAO()
4490+
.testCaseResolutionStatusTimeSeriesDao()
4491+
.listTestCaseResolutionForEntityFQNHash(testCase.getFullyQualifiedName());
4492+
assertNotNull(statusesAfter);
4493+
assertEquals(3, statusesAfter.size(), "Time series records should still exist in the database");
4494+
}
44234495
}

‎openmetadata-service/src/main/java/org/openmetadata/service/Entity.java‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -771,6 +771,15 @@ public static List<TagLabel> getEntityTags(String entityType, EntityInterface en
771771

772772
public static void deleteEntity(
773773
String updatedBy, String entityType, UUID entityId, boolean recursive, boolean hardDelete) {
774+
if (entityType.equalsIgnoreCase(Entity.TEST_CASE_RESOLUTION_STATUS)
775+
|| entityType.equalsIgnoreCase(Entity.TEST_CASE_RESULT)) {
776+
// TimeSeries entities don't have a standard repository delete flow.
777+
// We only want to delete their relationships if they are orphaned.
778+
EntityTimeSeriesRepository<?> repository = getEntityTimeSeriesRepository(entityType);
779+
// We don't call repository.delete(entityId) here because we want to preserve history.
780+
// The relationships will be cleaned up by the caller's relationshipDAO().deleteAll().
781+
return;
782+
}
774783
EntityRepository<?> dao = getEntityRepository(entityType);
775784
try {
776785
dao.find(entityId, Include.ALL);

‎openmetadata-service/src/main/java/org/openmetadata/service/apps/bundles/dataRetention/DataRetention.java‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -239,6 +239,9 @@ private void cleanOrphanedRelationshipsAndHierarchies() {
239239
result.getRelationshipResult().getRelationshipsDeleted(),
240240
result.getHierarchyResult().getTotalBrokenDeleted());
241241

242+
// Clean up orphaned TestCaseResolutionStatus relationships specifically
243+
cleanOrphanedTestCaseResolutionStatusRelationships();
244+
242245
} catch (Exception ex) {
243246
LOG.error("Failed to clean orphaned relationships and hierarchies", ex);
244247
internalStatus = AppRunRecord.Status.ACTIVE_ERROR;
@@ -251,6 +254,19 @@ private void cleanOrphanedRelationshipsAndHierarchies() {
251254
}
252255
}
253256

257+
@Transaction
258+
private void cleanOrphanedTestCaseResolutionStatusRelationships() {
259+
LOG.info("Initiating cleanup for orphaned testCaseResolutionStatus relationships.");
260+
executeWithStatsTracking(
261+
"orphaned_test_case_resolution_status_relationships",
262+
() ->
263+
collectionDAO
264+
.relationshipDAO()
265+
.deleteOrphanedRelationships(
266+
Entity.TEST_CASE, Entity.TEST_CASE_RESOLUTION_STATUS, "test_case"));
267+
LOG.info("TestCaseResolutionStatus orphaned relationships cleanup complete.");
268+
}
269+
254270
private void cleanOrphanedTagUsages() {
255271
LOG.info("Initiating orphaned tag usages cleanup.");
256272

‎openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/CollectionDAO.java‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2396,6 +2396,14 @@ default void deleteAll(UUID id, String entity) {
23962396
@SqlUpdate("DELETE FROM entity_relationship WHERE toId = :id AND toEntity = :entity")
23972397
void deleteAllTo(@BindUUID("id") UUID id, @Bind("entity") String entity);
23982398

2399+
@SqlUpdate(
2400+
"DELETE FROM entity_relationship WHERE toEntity = :toEntity AND fromEntity = :fromEntity "
2401+
+ "AND fromId NOT IN (SELECT id FROM <table>)")
2402+
int deleteOrphanedRelationships(
2403+
@Bind("fromEntity") String fromEntity,
2404+
@Bind("toEntity") String toEntity,
2405+
@Define("table") String table);
2406+
23992407
// Batch deletion methods for improved performance
24002408
@Transaction
24012409
default void batchDeleteRelationships(List<UUID> entityIds, String entityType) {

‎openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/EntityTimeSeriesRepository.java‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -391,6 +391,7 @@ public void deleteById(UUID id, boolean hardDelete) {
391391
return;
392392
}
393393
timeSeriesDao.deleteById(id);
394+
daoCollection.relationshipDAO().deleteAll(id, entityType);
394395
}
395396

396397
private Map<String, List<?>> getEntityList(List<String> jsons, boolean skipErrors) {

‎openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/TestCaseRepository.java‎

Lines changed: 5 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -891,30 +891,14 @@ private void updateLogicalTestSuite(UUID testSuiteId) {
891891
testSuiteRepository.postUpdate(original, testSuite);
892892
}
893893

894-
@Transaction
895-
@Override
896-
protected void deleteChildren(
897-
List<CollectionDAO.EntityRelationshipRecord> children, boolean hardDelete, String updatedBy) {
898-
if (hardDelete) {
899-
for (CollectionDAO.EntityRelationshipRecord entityRelationshipRecord : children) {
900-
LOG.info(
901-
"Recursively {} deleting {} {}",
902-
hardDelete ? "hard" : "soft",
903-
entityRelationshipRecord.getType(),
904-
entityRelationshipRecord.getId());
905-
TestCaseResolutionStatusRepository testCaseResolutionStatusRepository =
906-
(TestCaseResolutionStatusRepository)
907-
Entity.getEntityTimeSeriesRepository(Entity.TEST_CASE_RESOLUTION_STATUS);
908-
for (CollectionDAO.EntityRelationshipRecord child : children) {
909-
testCaseResolutionStatusRepository.deleteById(child.getId(), hardDelete);
910-
}
911-
}
912-
}
913-
}
914-
915894
@Override
916895
protected void entitySpecificCleanup(TestCase entityInterface) {
917896
deleteAllTestCaseResults(entityInterface.getFullyQualifiedName());
897+
TestCaseResolutionStatusRepository testCaseResolutionStatusRepository =
898+
(TestCaseResolutionStatusRepository)
899+
Entity.getEntityTimeSeriesRepository(Entity.TEST_CASE_RESOLUTION_STATUS);
900+
testCaseResolutionStatusRepository.deleteAllRelationshipsByTestCase(
901+
entityInterface.getFullyQualifiedName());
918902
}
919903

920904
private void deleteAllTestCaseResults(String fqn) {

‎openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/TestCaseResolutionStatusRepository.java‎

Lines changed: 16 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -236,22 +236,36 @@ public void storeInternal(
236236
recordEntity.withTestCaseReference(testCaseReference);
237237
}
238238

239+
public void deleteAllRelationshipsByTestCase(String testCaseFQN) {
240+
List<String> jsons =
241+
((CollectionDAO.TestCaseResolutionStatusTimeSeriesDAO) timeSeriesDao)
242+
.listTestCaseResolutionForEntityFQNHash(testCaseFQN);
243+
if (jsons.isEmpty()) {
244+
return;
245+
}
246+
List<UUID> statusIds =
247+
jsons.stream()
248+
.map(json -> JsonUtils.readValue(json, TestCaseResolutionStatus.class).getId())
249+
.toList();
250+
daoCollection.relationshipDAO().batchDeleteRelationships(statusIds, entityType);
251+
}
252+
239253
@Override
240254
protected void storeRelationship(TestCaseResolutionStatus recordEntity) {
241255
addRelationship(
242256
recordEntity.getTestCaseReference().getId(),
243257
recordEntity.getId(),
244258
Entity.TEST_CASE,
245259
Entity.TEST_CASE_RESOLUTION_STATUS,
246-
Relationship.PARENT_OF,
260+
Relationship.RELATED_TO,
247261
null,
248262
false);
249263
}
250264

251265
@Override
252266
protected void setInheritedFields(TestCaseResolutionStatus recordEntity) {
253267
recordEntity.setTestCaseReference(
254-
getFromEntityRef(recordEntity.getId(), Relationship.PARENT_OF, Entity.TEST_CASE, true));
268+
getFromEntityRef(recordEntity.getId(), Relationship.RELATED_TO, Entity.TEST_CASE, true));
255269
}
256270

257271
private void openOrAssignTask(TestCaseResolutionStatus incidentStatus) {

0 commit comments

Comments
 (0)