diff --git a/bootstrap/sql/migrations/native/1.13.0/mysql/schemaChanges.sql b/bootstrap/sql/migrations/native/1.13.0/mysql/schemaChanges.sql index ee41e5655fab..1d7d13817319 100644 --- a/bootstrap/sql/migrations/native/1.13.0/mysql/schemaChanges.sql +++ b/bootstrap/sql/migrations/native/1.13.0/mysql/schemaChanges.sql @@ -129,3 +129,13 @@ SELECT ue.id, re.id, 'user', 'role', 10 FROM user_entity ue, role_entity re WHERE ue.name = 'mcpapplicationbot' AND re.name = 'ApplicationBotImpersonationRole'; + +-- Add FQN hash columns to entity_relationship to enable fast prefix-based bulk deletion. +-- This allows deleting all relationships for an entire entity subtree in a single indexed query +-- instead of walking the tree entity-by-entity. +ALTER TABLE entity_relationship + ADD COLUMN fromFQNHash VARCHAR(768) DEFAULT NULL, + ADD COLUMN toFQNHash VARCHAR(768) DEFAULT NULL; + +CREATE INDEX idx_er_from_fqn_hash ON entity_relationship (fromFQNHash(768)); +CREATE INDEX idx_er_to_fqn_hash ON entity_relationship (toFQNHash(768)); diff --git a/bootstrap/sql/migrations/native/1.13.0/postgres/schemaChanges.sql b/bootstrap/sql/migrations/native/1.13.0/postgres/schemaChanges.sql index 87918fb2a7d1..5aa37757b52a 100644 --- a/bootstrap/sql/migrations/native/1.13.0/postgres/schemaChanges.sql +++ b/bootstrap/sql/migrations/native/1.13.0/postgres/schemaChanges.sql @@ -150,3 +150,20 @@ FROM user_entity ue, role_entity re WHERE ue.name = 'mcpapplicationbot' AND re.name = 'ApplicationBotImpersonationRole' ON CONFLICT DO NOTHING; + +-- Add FQN hash columns to entity_relationship to enable fast prefix-based bulk deletion. +-- This allows deleting all relationships for an entire entity subtree in a single indexed query +-- instead of walking the tree entity-by-entity. +ALTER TABLE entity_relationship + ADD COLUMN IF NOT EXISTS fromFQNHash VARCHAR(768) DEFAULT NULL, + ADD COLUMN IF NOT EXISTS toFQNHash VARCHAR(768) DEFAULT NULL; + +CREATE INDEX IF NOT EXISTS idx_er_from_fqn_hash ON entity_relationship (fromFQNHash); +CREATE INDEX IF NOT EXISTS idx_er_to_fqn_hash ON entity_relationship (toFQNHash); + +-- Fix entity_deletion_lock column types: id and entityId were created as native UUID +-- in 1.9.0 but the rest of the codebase uses VARCHAR(36) for UUID columns so that +-- BindUUID (which binds via UUID.toString()) can compare them without an explicit cast. +ALTER TABLE entity_deletion_lock + ALTER COLUMN id TYPE VARCHAR(36) USING id::VARCHAR, + ALTER COLUMN entityId TYPE VARCHAR(36) USING entityId::VARCHAR; diff --git a/bootstrap/sql/migrations/native/1.14.1/mysql/schemaChanges.sql b/bootstrap/sql/migrations/native/1.14.1/mysql/schemaChanges.sql new file mode 100644 index 000000000000..3afbbac80252 --- /dev/null +++ b/bootstrap/sql/migrations/native/1.14.1/mysql/schemaChanges.sql @@ -0,0 +1 @@ +-- No schema changes in this version. diff --git a/bootstrap/sql/migrations/native/1.14.1/postgres/schemaChanges.sql b/bootstrap/sql/migrations/native/1.14.1/postgres/schemaChanges.sql new file mode 100644 index 000000000000..3afbbac80252 --- /dev/null +++ b/bootstrap/sql/migrations/native/1.14.1/postgres/schemaChanges.sql @@ -0,0 +1 @@ +-- No schema changes in this version. diff --git a/bootstrap/sql/schema/mysql.sql b/bootstrap/sql/schema/mysql.sql index c76d44ffb301..b809fcff2c13 100644 --- a/bootstrap/sql/schema/mysql.sql +++ b/bootstrap/sql/schema/mysql.sql @@ -352,9 +352,13 @@ CREATE TABLE `entity_relationship` ( `jsonSchema` varchar(256) DEFAULT NULL, `json` json DEFAULT NULL, `deleted` tinyint(1) NOT NULL DEFAULT '0', + `fromFQNHash` varchar(768) DEFAULT NULL, + `toFQNHash` varchar(768) DEFAULT NULL, PRIMARY KEY (`fromId`,`toId`,`relation`), KEY `from_index` (`fromId`,`relation`), - KEY `to_index` (`toId`,`relation`) + KEY `to_index` (`toId`,`relation`), + KEY `idx_er_from_fqn_hash` (`fromFQNHash`(768)), + KEY `idx_er_to_fqn_hash` (`toFQNHash`(768)) ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci; /*!40101 SET character_set_client = @saved_cs_client */; diff --git a/bootstrap/sql/schema/postgres.sql b/bootstrap/sql/schema/postgres.sql index 0c6b64b116ed..7336a90ba23a 100644 --- a/bootstrap/sql/schema/postgres.sql +++ b/bootstrap/sql/schema/postgres.sql @@ -325,7 +325,9 @@ CREATE TABLE public.entity_relationship ( relation smallint NOT NULL, jsonschema character varying(256), json jsonb, - deleted boolean DEFAULT false NOT NULL + deleted boolean DEFAULT false NOT NULL, + fromfqnhash character varying(768), + tofqnhash character varying(768) ); @@ -1921,6 +1923,20 @@ CREATE INDEX entity_relationship_from_index ON public.entity_relationship USING CREATE INDEX entity_relationship_to_index ON public.entity_relationship USING btree (toid, relation); +-- +-- Name: idx_er_from_fqn_hash; Type: INDEX; Schema: public; Owner: openmetadata_user +-- + +CREATE INDEX idx_er_from_fqn_hash ON public.entity_relationship USING btree (fromfqnhash); + + +-- +-- Name: idx_er_to_fqn_hash; Type: INDEX; Schema: public; Owner: openmetadata_user +-- + +CREATE INDEX idx_er_to_fqn_hash ON public.entity_relationship USING btree (tofqnhash); + + -- -- Name: field_relationship_from_index; Type: INDEX; Schema: public; Owner: openmetadata_user -- diff --git a/openmetadata-integration-tests/src/test/java/org/openmetadata/it/factories/DatabaseSchemaTestFactory.java b/openmetadata-integration-tests/src/test/java/org/openmetadata/it/factories/DatabaseSchemaTestFactory.java index c3910fb9d9fe..b767160728aa 100644 --- a/openmetadata-integration-tests/src/test/java/org/openmetadata/it/factories/DatabaseSchemaTestFactory.java +++ b/openmetadata-integration-tests/src/test/java/org/openmetadata/it/factories/DatabaseSchemaTestFactory.java @@ -49,6 +49,14 @@ public static DatabaseSchema createSimple(TestNamespace ns) { return createSimple(ns, service); } + /** + * Create a schema with a namespaced base name using fluent API. + */ + public static DatabaseSchema createWithName( + TestNamespace ns, String databaseFqn, String baseName) { + return DatabaseSchemas.create().name(ns.prefix(baseName)).in(databaseFqn).execute(); + } + /** * Create a schema with a custom name using fluent API. * Useful for tests that need short names to avoid FQN length limits. diff --git a/openmetadata-integration-tests/src/test/java/org/openmetadata/it/factories/DatabaseTestFactory.java b/openmetadata-integration-tests/src/test/java/org/openmetadata/it/factories/DatabaseTestFactory.java index fb7e4427f789..ac7c8fdc9aab 100644 --- a/openmetadata-integration-tests/src/test/java/org/openmetadata/it/factories/DatabaseTestFactory.java +++ b/openmetadata-integration-tests/src/test/java/org/openmetadata/it/factories/DatabaseTestFactory.java @@ -29,4 +29,15 @@ public static Database create(TestNamespace ns, String serviceFqn) { public static Database createWithName(String serviceFqn, String name) { return Databases.create().name(name).in(serviceFqn).execute(); } + + /** + * Create database with a namespaced base name using fluent API. + */ + public static Database createWithName(TestNamespace ns, String serviceFqn, String baseName) { + return Databases.create() + .name(ns.prefix(baseName)) + .in(serviceFqn) + .withDescription("Test database created by integration test") + .execute(); + } } diff --git a/openmetadata-integration-tests/src/test/java/org/openmetadata/it/tests/PrefixDeletionBenchmarkIT.java b/openmetadata-integration-tests/src/test/java/org/openmetadata/it/tests/PrefixDeletionBenchmarkIT.java new file mode 100644 index 000000000000..cb02ea0d376d --- /dev/null +++ b/openmetadata-integration-tests/src/test/java/org/openmetadata/it/tests/PrefixDeletionBenchmarkIT.java @@ -0,0 +1,257 @@ +/* + * Copyright 2021 Collate + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.openmetadata.it.tests; + +import java.net.URI; +import java.net.http.HttpClient; +import java.net.http.HttpRequest; +import java.net.http.HttpResponse; +import java.time.Duration; +import java.util.ArrayList; +import java.util.List; +import java.util.UUID; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; +import org.awaitility.Awaitility; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Disabled; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.openmetadata.it.factories.DatabaseSchemaTestFactory; +import org.openmetadata.it.factories.DatabaseServiceTestFactory; +import org.openmetadata.it.factories.DatabaseTestFactory; +import org.openmetadata.it.factories.TableTestFactory; +import org.openmetadata.it.util.SdkClients; +import org.openmetadata.it.util.TestNamespace; +import org.openmetadata.it.util.TestNamespaceExtension; +import org.openmetadata.schema.entity.data.Database; +import org.openmetadata.schema.entity.data.DatabaseSchema; +import org.openmetadata.schema.entity.services.DatabaseService; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Benchmark comparing old recursive hard delete vs new FQN prefix hard delete. + * + *

Default topology: 5 databases × 5 schemas × 400 tables = 10,000 tables per service + * (~10,031 total entities including service, databases, schemas). + * + *

Run manually against a local stack: + * + * 

+ *   mvn verify -pl openmetadata-integration-tests \
+ *     -Dgroups=benchmark \
+ *     -Dit.test=PrefixDeletionBenchmarkIT \
+ *     -Dtest.databases=5      # databases per service (default: 5)
+ *     -Dtest.schemas=5        # schemas per database (default: 5)
+ *     -Dtest.tables=400       # tables per schema    (default: 400)
+ *     -Dtest.seedThreads=32   # parallel seed threads (default: 32)
+ *
+ * + *

NOTE: Setup creates entities in parallel (default 32 threads, tunable via + * -Dtest.seedThreads). At ~50ms/call and 32 threads, 10k tables seed in ~20 s. + * + *

Both deletions are timed end-to-end: the old delete is synchronous; the new prefix + * delete is async (202), so we poll until the service is gone before recording elapsed time. + */ +@Tag("benchmark") +@Disabled("Manual benchmark — run explicitly against a local mysql/postgres stack") +@ExtendWith(TestNamespaceExtension.class) +class PrefixDeletionBenchmarkIT { + + private static final Logger LOG = LoggerFactory.getLogger(PrefixDeletionBenchmarkIT.class); + + private static final int DATABASES_PER_SERVICE = Integer.getInteger("test.databases", 5); + private static final int SCHEMAS_PER_DATABASE = Integer.getInteger("test.schemas", 5); + private static final int TABLES_PER_SCHEMA = Integer.getInteger("test.tables", 400); + private static final int SEED_THREADS = Integer.getInteger("test.seedThreads", 32); + + private static final Duration DELETE_POLL_TIMEOUT = Duration.ofMinutes(10); + private static final Duration DELETE_POLL_INTERVAL = Duration.ofSeconds(2); + + @BeforeAll + static void setup() { + SdkClients.adminClient(); + } + + @Test + void benchmark_oldRecursiveHardDelete_vs_newPrefixDelete(TestNamespace ns) throws Exception { + int totalTables = DATABASES_PER_SERVICE * SCHEMAS_PER_DATABASE * TABLES_PER_SCHEMA; + int totalEntities = + 1 + DATABASES_PER_SERVICE + DATABASES_PER_SERVICE * SCHEMAS_PER_DATABASE + totalTables; + LOG.info( + "Benchmark topology: {} databases × {} schemas × {} tables = {} tables, {} total entities per service", + DATABASES_PER_SERVICE, + SCHEMAS_PER_DATABASE, + TABLES_PER_SCHEMA, + totalTables, + totalEntities); + + DatabaseService oldService = buildHierarchy(ns, "old"); + long oldMs = timeOldDelete(oldService); + + DatabaseService newService = buildHierarchy(ns, "new"); + long newMs = timeNewDelete(newService); + + double speedup = (double) oldMs / Math.max(newMs, 1); + LOG.info("=== Deletion Benchmark Results ({} entities per service) ===", totalEntities); + LOG.info(" Old recursive hard delete : {} ms", oldMs); + LOG.info(" New FQN prefix hard delete: {} ms", newMs); + LOG.info(" Speedup : {}x", String.format("%.2f", speedup)); + } + + private DatabaseService buildHierarchy(TestNamespace ns, String tag) throws Exception { + DatabaseService service = DatabaseServiceTestFactory.createPostgres(ns); + int totalEntities = + 1 + + DATABASES_PER_SERVICE + + DATABASES_PER_SERVICE * SCHEMAS_PER_DATABASE + + DATABASES_PER_SERVICE * SCHEMAS_PER_DATABASE * TABLES_PER_SCHEMA; + LOG.info( + "[{}] Seeding {} entities under service {} using {} threads ...", + tag, + totalEntities, + service.getName(), + SEED_THREADS); + long seedStart = System.currentTimeMillis(); + + ExecutorService pool = Executors.newFixedThreadPool(SEED_THREADS); + try { + List> dbFutures = new ArrayList<>(); + for (int d = 0; d < DATABASES_PER_SERVICE; d++) { + final int dIdx = d; + dbFutures.add( + pool.submit( + () -> + DatabaseTestFactory.createWithName( + ns, service.getFullyQualifiedName(), tag + "db" + dIdx))); + } + List databases = new ArrayList<>(); + for (Future f : dbFutures) { + databases.add(f.get()); + } + + List> schemaFutures = new ArrayList<>(); + for (int d = 0; d < databases.size(); d++) { + final Database database = databases.get(d); + final int dIdx = d; + for (int s = 0; s < SCHEMAS_PER_DATABASE; s++) { + final int sIdx = s; + schemaFutures.add( + pool.submit( + () -> + DatabaseSchemaTestFactory.createWithName( + ns, database.getFullyQualifiedName(), tag + "sc" + dIdx + "x" + sIdx))); + } + } + List schemas = new ArrayList<>(); + for (Future f : schemaFutures) { + schemas.add(f.get()); + } + + List> tableFutures = new ArrayList<>(); + for (int s = 0; s < schemas.size(); s++) { + final DatabaseSchema schema = schemas.get(s); + final int sIdx = s; + for (int t = 0; t < TABLES_PER_SCHEMA; t++) { + final int tIdx = t; + tableFutures.add( + pool.submit( + () -> { + TableTestFactory.createWithName( + ns, schema.getFullyQualifiedName(), tag + "tbl" + sIdx + "x" + tIdx); + return null; + })); + } + } + for (Future f : tableFutures) { + f.get(); + } + } finally { + pool.shutdown(); + pool.awaitTermination(30, TimeUnit.MINUTES); + } + + long seedMs = System.currentTimeMillis() - seedStart; + LOG.info( + "[{}] Hierarchy seeded in {} ms ({} ms/entity avg)", + tag, + seedMs, + seedMs / Math.max(totalEntities, 1)); + return service; + } + + private long timeOldDelete(DatabaseService service) throws Exception { + LOG.info("Timing OLD recursive hard delete for service {} ...", service.getName()); + long start = System.currentTimeMillis(); + + String url = + SdkClients.getServerUrl() + + "/v1/services/databaseServices/" + + service.getId() + + "?hardDelete=true&recursive=true"; + sendDelete(url); + + long elapsed = System.currentTimeMillis() - start; + LOG.info("OLD recursive hard delete completed in {} ms", elapsed); + return elapsed; + } + + private long timeNewDelete(DatabaseService service) throws Exception { + LOG.info("Timing NEW FQN prefix hard delete for service {} ...", service.getName()); + long start = System.currentTimeMillis(); + + String url = + SdkClients.getServerUrl() + "/v1/services/databaseServices/prefix/" + service.getId(); + sendDelete(url); + + // Prefix delete is async — poll until the service is actually gone so we measure + // real deletion time, not just the time to hand off the job to the executor. + UUID serviceId = service.getId(); + Awaitility.await("Wait for prefix deletion of " + service.getName() + " to complete") + .atMost(DELETE_POLL_TIMEOUT) + .pollInterval(DELETE_POLL_INTERVAL) + .until( + () -> { + try { + SdkClients.adminClient().databaseServices().get(serviceId.toString()); + return false; + } catch (Exception e) { + return true; + } + }); + + long elapsed = System.currentTimeMillis() - start; + LOG.info("NEW FQN prefix hard delete completed in {} ms", elapsed); + return elapsed; + } + + private void sendDelete(String url) throws Exception { + HttpRequest request = + HttpRequest.newBuilder() + .uri(URI.create(url)) + .header("Authorization", "Bearer " + SdkClients.getAdminToken()) + .DELETE() + .build(); + HttpResponse response = + HttpClient.newHttpClient().send(request, HttpResponse.BodyHandlers.ofString()); + if (response.statusCode() >= 300) { + throw new RuntimeException( + "Delete failed with status " + response.statusCode() + ": " + response.body()); + } + } +} diff --git a/openmetadata-integration-tests/src/test/java/org/openmetadata/it/tests/PrefixDeletionIT.java b/openmetadata-integration-tests/src/test/java/org/openmetadata/it/tests/PrefixDeletionIT.java new file mode 100644 index 000000000000..7d2b51d59086 --- /dev/null +++ b/openmetadata-integration-tests/src/test/java/org/openmetadata/it/tests/PrefixDeletionIT.java @@ -0,0 +1,280 @@ +/* + * Copyright 2021 Collate + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.openmetadata.it.tests; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; + +import java.net.URI; +import java.net.http.HttpClient; +import java.net.http.HttpRequest; +import java.net.http.HttpResponse; +import java.time.Duration; +import java.util.ArrayList; +import java.util.List; +import java.util.UUID; +import org.awaitility.Awaitility; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.openmetadata.it.factories.DatabaseSchemaTestFactory; +import org.openmetadata.it.factories.DatabaseServiceTestFactory; +import org.openmetadata.it.factories.DatabaseTestFactory; +import org.openmetadata.it.factories.TableTestFactory; +import org.openmetadata.it.util.SdkClients; +import org.openmetadata.it.util.TestNamespace; +import org.openmetadata.it.util.TestNamespaceExtension; +import org.openmetadata.schema.entity.data.Database; +import org.openmetadata.schema.entity.data.DatabaseSchema; +import org.openmetadata.schema.entity.data.Table; +import org.openmetadata.schema.entity.services.DatabaseService; + +/** + * Integration tests for the FQN prefix-based hard deletion endpoints at each hierarchy level: + * + *

+ * + *

The endpoint is async (returns 202 with a jobId). All assertions use Awaitility to poll until + * the background deletion actually completes. + */ +@ExtendWith(TestNamespaceExtension.class) +public class PrefixDeletionIT { + + private static final Duration DELETE_TIMEOUT = Duration.ofSeconds(30); + private static final Duration POLL_INTERVAL = Duration.ofSeconds(1); + + @BeforeAll + static void setup() { + SdkClients.adminClient(); + } + + // ── Service-level ──────────────────────────────────────────────────────────── + + @Test + void prefixDelete_service_removesServiceAndAllDescendants(TestNamespace ns) throws Exception { + DatabaseService service = DatabaseServiceTestFactory.createPostgres(ns); + Database database = + DatabaseTestFactory.createWithName(ns, service.getFullyQualifiedName(), "db"); + DatabaseSchema schema = + DatabaseSchemaTestFactory.createWithName(ns, database.getFullyQualifiedName(), "sc"); + + List tableIds = new ArrayList<>(); + for (int i = 0; i < 5; i++) { + Table table = TableTestFactory.createWithName(ns, schema.getFullyQualifiedName(), "t" + i); + tableIds.add(table.getId()); + } + + prefixDelete("/v1/services/databaseServices/prefix/", service.getId()); + + awaitGone( + "service", + () -> SdkClients.adminClient().databaseServices().get(service.getId().toString())); + awaitGone( + "database", () -> SdkClients.adminClient().databases().get(database.getId().toString())); + awaitGone( + "schema", () -> SdkClients.adminClient().databaseSchemas().get(schema.getId().toString())); + for (UUID tableId : tableIds) { + awaitGone("table", () -> SdkClients.adminClient().tables().get(tableId.toString())); + } + } + + @Test + void prefixDelete_service_withMultipleDatabasesAndSchemas(TestNamespace ns) throws Exception { + DatabaseService service = DatabaseServiceTestFactory.createPostgres(ns); + + List dbIds = new ArrayList<>(); + List schemaIds = new ArrayList<>(); + List tableIds = new ArrayList<>(); + + for (int d = 0; d < 2; d++) { + Database database = + DatabaseTestFactory.createWithName(ns, service.getFullyQualifiedName(), "db" + d); + dbIds.add(database.getId()); + for (int s = 0; s < 2; s++) { + DatabaseSchema schema = + DatabaseSchemaTestFactory.createWithName( + ns, database.getFullyQualifiedName(), "sc" + d + s); + schemaIds.add(schema.getId()); + for (int t = 0; t < 3; t++) { + Table table = + TableTestFactory.createWithName(ns, schema.getFullyQualifiedName(), "t" + d + s + t); + tableIds.add(table.getId()); + } + } + } + + prefixDelete("/v1/services/databaseServices/prefix/", service.getId()); + + awaitGone( + "service", + () -> SdkClients.adminClient().databaseServices().get(service.getId().toString())); + for (UUID id : dbIds) { + awaitGone("database", () -> SdkClients.adminClient().databases().get(id.toString())); + } + for (UUID id : schemaIds) { + awaitGone("schema", () -> SdkClients.adminClient().databaseSchemas().get(id.toString())); + } + for (UUID id : tableIds) { + awaitGone("table", () -> SdkClients.adminClient().tables().get(id.toString())); + } + } + + // ── Database-level ──────────────────────────────────────────────────────────── + + @Test + void prefixDelete_database_removesDatabaseAndDescendantsLeavingSiblingDatabaseIntact( + TestNamespace ns) throws Exception { + DatabaseService service = DatabaseServiceTestFactory.createPostgres(ns); + + Database targetDb = + DatabaseTestFactory.createWithName(ns, service.getFullyQualifiedName(), "tgtdb"); + DatabaseSchema targetSchema = + DatabaseSchemaTestFactory.createWithName(ns, targetDb.getFullyQualifiedName(), "tgtsc"); + Table targetTable = + TableTestFactory.createWithName(ns, targetSchema.getFullyQualifiedName(), "tgttbl"); + + Database siblingDb = + DatabaseTestFactory.createWithName(ns, service.getFullyQualifiedName(), "sibdb"); + DatabaseSchema siblingSchema = + DatabaseSchemaTestFactory.createWithName(ns, siblingDb.getFullyQualifiedName(), "sibsc"); + Table siblingTable = + TableTestFactory.createWithName(ns, siblingSchema.getFullyQualifiedName(), "sibtbl"); + + prefixDelete("/v1/databases/prefix/", targetDb.getId()); + + awaitGone( + "target database", + () -> SdkClients.adminClient().databases().get(targetDb.getId().toString())); + awaitGone( + "target schema", + () -> SdkClients.adminClient().databaseSchemas().get(targetSchema.getId().toString())); + awaitGone( + "target table", + () -> SdkClients.adminClient().tables().get(targetTable.getId().toString())); + + assertNotNull( + SdkClients.adminClient().databases().get(siblingDb.getId().toString()), + "sibling database should survive"); + assertNotNull( + SdkClients.adminClient().databaseSchemas().get(siblingSchema.getId().toString()), + "sibling schema should survive"); + assertNotNull( + SdkClients.adminClient().tables().get(siblingTable.getId().toString()), + "sibling table should survive"); + assertNotNull( + SdkClients.adminClient().databaseServices().get(service.getId().toString()), + "service should survive"); + } + + // ── Schema-level ───────────────────────────────────────────────────────────── + + @Test + void prefixDelete_schema_removesSchemaAndTablesLeavingSiblingSchemaIntact(TestNamespace ns) + throws Exception { + DatabaseService service = DatabaseServiceTestFactory.createPostgres(ns); + Database database = + DatabaseTestFactory.createWithName(ns, service.getFullyQualifiedName(), "db"); + + DatabaseSchema targetSchema = + DatabaseSchemaTestFactory.createWithName(ns, database.getFullyQualifiedName(), "tgtsc"); + List targetTableIds = new ArrayList<>(); + for (int i = 0; i < 3; i++) { + Table table = + TableTestFactory.createWithName(ns, targetSchema.getFullyQualifiedName(), "tgt" + i); + targetTableIds.add(table.getId()); + } + + DatabaseSchema siblingSchema = + DatabaseSchemaTestFactory.createWithName(ns, database.getFullyQualifiedName(), "sibsc"); + Table siblingTable = + TableTestFactory.createWithName(ns, siblingSchema.getFullyQualifiedName(), "sibtbl"); + + prefixDelete("/v1/databaseSchemas/prefix/", targetSchema.getId()); + + awaitGone( + "target schema", + () -> SdkClients.adminClient().databaseSchemas().get(targetSchema.getId().toString())); + for (UUID tableId : targetTableIds) { + awaitGone("target table", () -> SdkClients.adminClient().tables().get(tableId.toString())); + } + + assertNotNull( + SdkClients.adminClient().databaseSchemas().get(siblingSchema.getId().toString()), + "sibling schema should survive"); + assertNotNull( + SdkClients.adminClient().tables().get(siblingTable.getId().toString()), + "sibling table should survive"); + assertNotNull( + SdkClients.adminClient().databases().get(database.getId().toString()), + "database should survive"); + assertNotNull( + SdkClients.adminClient().databaseServices().get(service.getId().toString()), + "service should survive"); + } + + // ── Helpers ─────────────────────────────────────────────────────────────────── + + private void prefixDelete(String path, UUID id) throws Exception { + String url = SdkClients.getServerUrl() + path + id; + HttpRequest request = + HttpRequest.newBuilder() + .uri(URI.create(url)) + .header("Authorization", "Bearer " + SdkClients.getAdminToken()) + .DELETE() + .build(); + + HttpResponse response = + HttpClient.newHttpClient().send(request, HttpResponse.BodyHandlers.ofString()); + assertEquals( + 202, + response.statusCode(), + "Expected 202 Accepted from prefix delete " + + path + + id + + ", got " + + response.statusCode() + + ": " + + response.body()); + } + + /** + * Polls until the given fetch throws (entity gone) or the timeout elapses. + * The prefix delete API is async — the 202 only means the job was queued. + */ + private void awaitGone(String entityType, ThrowingSupplier fetch) { + Awaitility.await("Wait for " + entityType + " to be deleted") + .atMost(DELETE_TIMEOUT) + .pollInterval(POLL_INTERVAL) + .until( + () -> { + try { + fetch.get(); + return false; + } catch (Exception e) { + return true; + } + }); + } + + @FunctionalInterface + private interface ThrowingSupplier { + T get() throws Exception; + } +} diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/OpenMetadataApplication.java b/openmetadata-service/src/main/java/org/openmetadata/service/OpenMetadataApplication.java index 4b0eab6398da..b76d094d231f 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/OpenMetadataApplication.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/OpenMetadataApplication.java @@ -102,6 +102,7 @@ import org.openmetadata.service.exception.OMErrorPageHandler; import org.openmetadata.service.fernet.Fernet; import org.openmetadata.service.governance.workflows.WorkflowHandler; +import org.openmetadata.service.initialization.LockManagerInitializer; import org.openmetadata.service.jdbi3.BulkExecutor; import org.openmetadata.service.jdbi3.CollectionDAO; import org.openmetadata.service.jdbi3.EntityRelationshipRepository; @@ -275,6 +276,8 @@ public void run(OpenMetadataApplicationConfig catalogConfig, Environment environ ResourceRegistry.addResource( Entity.AUDIT_LOG, List.of(MetadataOperation.AUDIT_LOGS), Collections.emptySet()); + LockManagerInitializer.initialize(); + // Configure the Fernet instance Fernet.getInstance().setFernetKey(catalogConfig); diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/initialization/LockManagerInitializer.java b/openmetadata-service/src/main/java/org/openmetadata/service/initialization/LockManagerInitializer.java index 93cab31a2be0..4886d7837ff9 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/initialization/LockManagerInitializer.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/initialization/LockManagerInitializer.java @@ -3,6 +3,7 @@ import lombok.extern.slf4j.Slf4j; import org.openmetadata.service.Entity; import org.openmetadata.service.jdbi3.EntityRepository; +import org.openmetadata.service.jdbi3.PrefixDeletionService; import org.openmetadata.service.lock.HierarchicalLockManager; /** @@ -44,6 +45,9 @@ public static void initialize() { // Set it on EntityRepository EntityRepository.setLockManager(lockManager); + // Initialize PrefixDeletionService with the same lock manager + PrefixDeletionService.initialize(lockManager); + initialized = true; LOG.info("Hierarchical lock manager initialized successfully"); diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/CollectionDAO.java b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/CollectionDAO.java index 561e405b7dc9..d1df298eb95b 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/CollectionDAO.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/CollectionDAO.java @@ -1419,7 +1419,14 @@ List getExtensionsWithOffset( void deleteAll(@BindUUID("id") UUID id); @SqlUpdate("DELETE FROM entity_extension WHERE id IN ()") - void deleteAllBatch(@BindList("ids") List ids); + void deleteAllBatchChunk(@BindList("ids") List ids); + + default void deleteAllBatch(List ids) { + int chunkSize = 50_000; + for (int i = 0; i < ids.size(); i += chunkSize) { + deleteAllBatchChunk(ids.subList(i, Math.min(i + chunkSize, ids.size()))); + } + } } class EntityVersionPair { @@ -1556,15 +1563,21 @@ default void bulkRemoveFromRelationship( @ConnectionAwareSqlUpdate( value = - "INSERT INTO entity_relationship(fromId, toId, fromEntity, toEntity, relation, json) " - + "VALUES (:fromId, :toId, :fromEntity, :toEntity, :relation, :json) " - + "ON DUPLICATE KEY UPDATE json = :json", + "INSERT INTO entity_relationship" + + "(fromId, toId, fromEntity, toEntity, relation, json, fromFQNHash, toFQNHash) " + + "VALUES (:fromId, :toId, :fromEntity, :toEntity, :relation, :json, :fromFQNHash, :toFQNHash) " + + "ON DUPLICATE KEY UPDATE json = :json, " + + "fromFQNHash = COALESCE(:fromFQNHash, fromFQNHash), " + + "toFQNHash = COALESCE(:toFQNHash, toFQNHash)", connectionType = MYSQL) @ConnectionAwareSqlUpdate( value = - "INSERT INTO entity_relationship(fromId, toId, fromEntity, toEntity, relation, json) VALUES " - + "(:fromId, :toId, :fromEntity, :toEntity, :relation, (:json :: jsonb)) " - + "ON CONFLICT (fromId, toId, relation) DO UPDATE SET json = EXCLUDED.json", + "INSERT INTO entity_relationship" + + "(fromId, toId, fromEntity, toEntity, relation, json, fromFQNHash, toFQNHash) " + + "VALUES (:fromId, :toId, :fromEntity, :toEntity, :relation, (:json :: jsonb), :fromFQNHash, :toFQNHash) " + + "ON CONFLICT (fromId, toId, relation) DO UPDATE SET json = EXCLUDED.json, " + + "fromFQNHash = COALESCE(EXCLUDED.fromFQNHash, entity_relationship.fromFQNHash), " + + "toFQNHash = COALESCE(EXCLUDED.toFQNHash, entity_relationship.toFQNHash)", connectionType = POSTGRES) void insert( @BindUUID("fromId") UUID fromId, @@ -1572,7 +1585,14 @@ void insert( @Bind("fromEntity") String fromEntity, @Bind("toEntity") String toEntity, @Bind("relation") int relation, - @Bind("json") String json); + @Bind("json") String json, + @Bind("fromFQNHash") String fromFQNHash, + @Bind("toFQNHash") String toFQNHash); + + default void insert( + UUID fromId, UUID toId, String fromEntity, String toEntity, int relation, String json) { + insert(fromId, toId, fromEntity, toEntity, relation, json, null, null); + } @ConnectionAwareSqlUpdate( value = @@ -2421,6 +2441,32 @@ default void batchDeleteRelationships(List entityIds, String entityType) { @SqlUpdate("DELETE from entity_relationship WHERE fromId = :id or toId = :id") void deleteAllWithId(@BindUUID("id") UUID id); + @SqlUpdate( + "UPDATE entity_relationship SET fromFQNHash = :fqnHash " + + "WHERE fromId = :id AND fromFQNHash IS NULL") + void backfillFromFqnHash(@Bind("id") String id, @Bind("fqnHash") String fqnHash); + + @SqlUpdate( + "UPDATE entity_relationship SET toFQNHash = :fqnHash " + + "WHERE toId = :id AND toFQNHash IS NULL") + void backfillToFqnHash(@Bind("id") String id, @Bind("fqnHash") String fqnHash); + + @SqlUpdate( + "DELETE FROM entity_relationship WHERE fromFQNHash = :exact OR fromFQNHash LIKE :prefix") + void deleteByFromFqnHashPrefix( + @Bind("exact") String exactHash, @Bind("prefix") String prefixPattern); + + @SqlUpdate("DELETE FROM entity_relationship WHERE toFQNHash = :exact OR toFQNHash LIKE :prefix") + void deleteByToFqnHashPrefix( + @Bind("exact") String exactHash, @Bind("prefix") String prefixPattern); + + @Transaction + default void deleteAllByFqnHashPrefix(String fqnHashPrefix) { + String prefixPattern = fqnHashPrefix + ".%"; + deleteByFromFqnHashPrefix(fqnHashPrefix, prefixPattern); + deleteByToFqnHashPrefix(fqnHashPrefix, prefixPattern); + } + @ConnectionAwareSqlUpdate( value = "DELETE FROM entity_relationship " @@ -3195,6 +3241,19 @@ List> listCountThreadsByGlossaryAndTerms( @SqlQuery("select id from thread_entity where entityId = :entityId") List findByEntityId(@Bind("entityId") String entityId); + @SqlQuery("SELECT id FROM thread_entity WHERE entityId IN ()") + List findByEntityIdsChunk(@BindList("entityIds") List entityIds); + + default List findByEntityIds(List entityIds) { + int chunkSize = 50_000; + List result = new ArrayList<>(); + for (int i = 0; i < entityIds.size(); i += chunkSize) { + result.addAll( + findByEntityIdsChunk(entityIds.subList(i, Math.min(i + chunkSize, entityIds.size())))); + } + return result; + } + @ConnectionAwareSqlUpdate( value = "UPDATE thread_entity SET json = JSON_SET(json, '$.about', :newEntityLink)\n" @@ -6168,6 +6227,16 @@ List getUsageById( @SqlUpdate("DELETE FROM entity_usage WHERE id = :id") void delete(@BindUUID("id") UUID id); + @SqlUpdate("DELETE FROM entity_usage WHERE id IN ()") + void deleteBatchChunk(@BindList("ids") List ids); + + default void deleteBatch(List ids) { + int chunkSize = 50_000; + for (int i = 0; i < ids.size(); i += chunkSize) { + deleteBatchChunk(ids.subList(i, Math.min(i + chunkSize, ids.size()))); + } + } + /** * TODO: Not sure I get what the next comment means, but tests now use mysql 8 so maybe tests can be improved here * Note not using in following percentile computation PERCENT_RANK function as unit tests use mysql5.7, and it does diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/DataContractRepository.java b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/DataContractRepository.java index cbc426e6c81a..08018062026e 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/DataContractRepository.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/DataContractRepository.java @@ -1725,6 +1725,21 @@ public FeedRepository.TaskWorkflow getTaskWorkflow(FeedRepository.ThreadContext return super.getTaskWorkflow(threadContext); } + @Override + protected void preDeleteByFqnHashPrefix(String fqnHashPrefix, String deletedBy) { + List ids = getDao().findIdsByFqnHashPrefix(fqnHashPrefix); + for (UUID id : ids) { + try { + DataContract contract = find(id, Include.ALL); + if (!nullOrEmpty(contract.getQualityExpectations())) { + deleteTestSuite(contract); + } + } catch (Exception e) { + LOG.warn("Failed cleanup for data contract {}: {}", id, e.getMessage()); + } + } + } + @Override protected void preDelete(DataContract entity, String deletedBy) { // Inherited contracts cannot be deleted - they are virtual contracts derived from Data Product diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/DatabaseSchemaRepository.java b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/DatabaseSchemaRepository.java index 205c84e54095..8106cd2169fc 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/DatabaseSchemaRepository.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/DatabaseSchemaRepository.java @@ -129,10 +129,14 @@ public void storeRelationships(DatabaseSchema schema) { EntityReference database = schema.getDatabase(); addRelationship( database.getId(), + database.getFullyQualifiedName(), schema.getId(), + schema.getFullyQualifiedName(), database.getType(), Entity.DATABASE_SCHEMA, - Relationship.CONTAINS); + Relationship.CONTAINS, + null, + false); } @Override diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/EntityDAO.java b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/EntityDAO.java index 0e8c8d69e562..8e8d918ec70a 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/EntityDAO.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/EntityDAO.java @@ -191,6 +191,19 @@ default void updateFqn(String oldPrefix, String newPrefix) { void updateFqnInternal( @Define("mySqlUpdate") String mySqlUpdate, @Define("postgresUpdate") String postgresUpdate); + @SqlQuery("SELECT id FROM WHERE LIKE :prefix") + List findIdsByFqnHashPrefixInternal( + @Define("table") String table, @Define("col") String col, @Bind("prefix") String prefix); + + default List findIdsByFqnHashPrefix(String fqnHashPrefix) { + if (!"fqnHash".equals(getNameHashColumn())) { + return List.of(); + } + return findIdsByFqnHashPrefixInternal(getTableName(), "fqnHash", fqnHashPrefix + ".%").stream() + .map(UUID::fromString) + .toList(); + } + @SqlQuery("SELECT json FROM
WHERE id = :id ") String findById( @Define("table") String table, @BindUUID("id") UUID id, @Define("cond") String cond); @@ -686,6 +699,16 @@ default void delete(UUID id) { } } + @SqlUpdate("DELETE FROM
WHERE id IN ()") + void deleteBatchInternal(@Define("table") String table, @BindList("ids") List ids); + + default void deleteBatch(List ids) { + if (ids == null || ids.isEmpty()) { + return; + } + deleteBatchInternal(getTableName(), ids); + } + record EntityNameColumnHashJsonPair(String nameColumnHash, String json) {} class EntityNameColumnHashJsonPairMapper implements RowMapper { diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/EntityRepository.java b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/EntityRepository.java index 2e1ab6552f44..539923c9d2df 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/EntityRepository.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/EntityRepository.java @@ -422,6 +422,10 @@ public static void setLockManager(HierarchicalLockManager manager) { lockManager = manager; } + public static HierarchicalLockManager getLockManager() { + return lockManager; + } + public boolean isSupportsOwners() { return supportsOwners; } @@ -1105,7 +1109,15 @@ protected void setInheritedFields(List entities, Fields fields) { protected final void addServiceRelationship(T entity, EntityReference service) { if (service != null) { addRelationship( - service.getId(), entity.getId(), service.getType(), entityType, Relationship.CONTAINS); + service.getId(), + service.getFullyQualifiedName(), + entity.getId(), + entity.getFullyQualifiedName(), + service.getType(), + entityType, + Relationship.CONTAINS, + null, + false); } } @@ -3435,6 +3447,19 @@ protected void postDelete(T entity, boolean hardDelete) { } } + /** + * Called once per entity type before prefix deletion removes entities from the DB. + * Override for bulk-efficient cleanup (e.g. calling an external system like Airflow). + * Entities matching the prefix are still in the DB when this runs. + */ + protected void preDeleteByFqnHashPrefix(String fqnHashPrefix, String deletedBy) {} + + /** + * Called once per entity type after prefix deletion has removed entities from the DB. + * Override for post-deletion metadata updates that do not require loading the deleted entities. + */ + protected void postDeleteByFqnHashPrefix(String fqnHashPrefix) {} + public final void deleteFromSearch(T entity, boolean hardDelete) { try (var ignored = phase("lifecycleDispatch")) { if (hardDelete) { @@ -3785,6 +3810,16 @@ protected final void cleanup(T entityInterface) { protected void entitySpecificCleanup(T entityInterface) {} + @SuppressWarnings("unchecked") + final void callPreDelete(EntityInterface entity, String deletedBy) { + preDelete((T) entity, deletedBy); + } + + @SuppressWarnings("unchecked") + final void invalidateEntity(EntityInterface entity) { + invalidate((T) entity); + } + private void invalidate(T entity) { CACHE_WITH_ID.invalidate(new ImmutablePair<>(entityType, entity.getId())); CACHE_WITH_NAME.invalidate(new ImmutablePair<>(entityType, entity.getFullyQualifiedName())); @@ -4917,17 +4952,34 @@ public final void addRelationship( Relationship relationship, String json, boolean bidirectional) { + addRelationship( + fromId, null, toId, null, fromEntity, toEntity, relationship, json, bidirectional); + } + + @Transaction + public final void addRelationship( + UUID fromId, + String fromFqn, + UUID toId, + String toFqn, + String fromEntity, + String toEntity, + Relationship relationship, + String json, + boolean bidirectional) { UUID from = fromId; UUID to = toId; + String fromHash = fromFqn != null ? FullyQualifiedName.buildHash(fromFqn) : null; + String toHash = toFqn != null ? FullyQualifiedName.buildHash(toFqn) : null; if (bidirectional && fromId.compareTo(toId) > 0) { - // For bidirectional relationship, instead of adding two row fromId -> toId and toId -> - // fromId, just add one row where fromId is alphabetically less than toId from = toId; to = fromId; + fromHash = toFqn != null ? FullyQualifiedName.buildHash(toFqn) : null; + toHash = fromFqn != null ? FullyQualifiedName.buildHash(fromFqn) : null; } daoCollection .relationshipDAO() - .insert(from, to, fromEntity, toEntity, relationship.ordinal(), json); + .insert(from, to, fromEntity, toEntity, relationship.ordinal(), json, fromHash, toHash); // Update RDF EntityRelationship entityRelationship = @@ -4940,7 +4992,6 @@ public final void addRelationship( RdfUpdater.addRelationship(entityRelationship); if (bidirectional) { - // Also add the reverse relationship to RDF EntityRelationship reverseRelationship = new EntityRelationship() .withFromId(toId) @@ -4952,6 +5003,10 @@ public final void addRelationship( } } + protected void deleteTimeSeriesByFqnPrefix(String fqnHashPrefix) { + // No-op default — override in repositories with associated time-series tables + } + @Transaction public final void bulkAddToRelationship( UUID fromId, List toId, String fromEntity, String toEntity, Relationship relationship) { diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/EntityTimeSeriesDAO.java b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/EntityTimeSeriesDAO.java index 96ea3e06a08f..65da83fff948 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/EntityTimeSeriesDAO.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/EntityTimeSeriesDAO.java @@ -530,6 +530,17 @@ default void delete(String entityFQNHash, String extension) { delete(getTimeSeriesTableName(), entityFQNHash, extension); } + @SqlUpdate( + "DELETE FROM
WHERE entityFQNHash = :exactHash OR entityFQNHash LIKE :prefixHash") + void deleteByFqnHashPrefixInternal( + @Define("table") String table, + @Bind("exactHash") String exactHash, + @Bind("prefixHash") String prefixHash); + + default void deleteByFqnHashPrefix(String fqnHashPrefix) { + deleteByFqnHashPrefixInternal(getTimeSeriesTableName(), fqnHashPrefix, fqnHashPrefix + ".%"); + } + @SqlUpdate( "DELETE FROM
WHERE entityFQNHash = :entityFQNHash AND extension = :extension AND timestamp = :timestamp") void deleteAtTimestamp( diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/FeedRepository.java b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/FeedRepository.java index 0364eb316305..04fa428f4609 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/FeedRepository.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/FeedRepository.java @@ -772,6 +772,22 @@ public void deleteByAbout(UUID entityId) { } } + @Transaction + public void deleteByAboutBatch(List entityIds) { + if (nullOrEmpty(entityIds)) { + return; + } + List ids = entityIds.stream().map(UUID::toString).toList(); + List threadIds = listOrEmpty(dao.feedDAO().findByEntityIds(ids)); + for (String threadId : threadIds) { + try { + deleteThreadInternal(UUID.fromString(threadId)); + } catch (Exception ex) { + // Continue deletion + } + } + } + public List getThreadsCount(String link) { List> result; EntityLink entityLink = EntityLink.parse(link); diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/IngestionPipelineRepository.java b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/IngestionPipelineRepository.java index a6ad876c7eab..c9c2339f9891 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/IngestionPipelineRepository.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/IngestionPipelineRepository.java @@ -488,6 +488,24 @@ protected void postDelete(IngestionPipeline entity, boolean hardDelete) { .delete(entity.getFullyQualifiedName(), PIPELINE_STATUS_EXTENSION); } + @Override + protected void preDeleteByFqnHashPrefix(String fqnHashPrefix, String deletedBy) { + List ids = getDao().findIdsByFqnHashPrefix(fqnHashPrefix); + for (UUID id : ids) { + try { + IngestionPipeline pipeline = find(id, Include.ALL); + if (pipelineServiceClient != null) { + pipelineServiceClient.deletePipeline(pipeline); + } + daoCollection + .entityExtensionTimeSeriesDao() + .delete(pipeline.getFullyQualifiedName(), PIPELINE_STATUS_EXTENSION); + } catch (Exception e) { + LOG.warn("Failed cleanup for ingestion pipeline {}: {}", id, e.getMessage()); + } + } + } + @Override protected EntityReference getParentReference(IngestionPipeline entity) { return entity.getService(); diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/PrefixDeletionService.java b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/PrefixDeletionService.java new file mode 100644 index 000000000000..9fff9e217e4e --- /dev/null +++ b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/PrefixDeletionService.java @@ -0,0 +1,229 @@ +/* + * Copyright 2021 Collate + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.openmetadata.service.jdbi3; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.UUID; +import lombok.extern.slf4j.Slf4j; +import org.openmetadata.schema.EntityInterface; +import org.openmetadata.service.Entity; +import org.openmetadata.service.events.lifecycle.EntityLifecycleEventDispatcher; +import org.openmetadata.service.lock.HierarchicalLockManager; +import org.openmetadata.service.search.SearchRepository; +import org.openmetadata.service.util.FullyQualifiedName; + +/** + * Orchestrates fast prefix-based hard deletion of an entity and all its descendants. + * + *

Uses 7 phases to atomically delete an entire subtree by FQN prefix rather than walking the + * entity tree one-by-one. This is orders of magnitude faster for large hierarchies and eliminates + * the race condition where concurrent ingestion creates orphaned entities during slow cascade + * deletion. + * + *

This service handles hard delete only. Soft delete still uses the existing tree walk in + * {@link EntityRepository} to preserve relationship data for restoration. + */ +@Slf4j +public final class PrefixDeletionService { + + private static volatile PrefixDeletionService instance; + + private final HierarchicalLockManager lockManager; + + private PrefixDeletionService(HierarchicalLockManager lockManager) { + this.lockManager = lockManager; + } + + public static void initialize(HierarchicalLockManager lockMgr) { + if (instance == null) { + synchronized (PrefixDeletionService.class) { + if (instance == null) { + instance = new PrefixDeletionService(lockMgr); + } + } + } + } + + public static PrefixDeletionService getInstance() { + return instance; + } + + /** + * Hard-deletes the given root entity and all descendants whose FQN starts with the root's FQN. + * Works at any hierarchy level (service, database, schema, etc.). + */ + public void deletePrefixHard(EntityInterface rootEntity, String deletedBy) { + String rootFqn = rootEntity.getFullyQualifiedName(); + String fqnHashPrefix = FullyQualifiedName.buildHash(rootFqn); + DeletionLock lock = acquireLock(rootEntity, deletedBy); + try { + Map> descendantsByType = collectDescendantIds(fqnHashPrefix); + List allIds = buildAllIds(rootEntity.getId(), descendantsByType); + deleteDependencyTables(rootFqn, fqnHashPrefix, allIds); + runEntityHooks(rootEntity, deletedBy, fqnHashPrefix); + deleteEntityTables(descendantsByType, rootEntity); + runPostEntityHooks(fqnHashPrefix); + cleanSearchIndex(rootEntity, descendantsByType.keySet()); + emitDeleteEvent(rootEntity); + } finally { + releaseLock(lock, rootEntity); + } + } + + private DeletionLock acquireLock(EntityInterface entity, String deletedBy) { + if (lockManager == null) { + return null; + } + try { + return lockManager.acquireDeletionLock(entity, deletedBy, true); + } catch (Exception e) { + LOG.warn( + "Could not acquire deletion lock for {}: {}", + entity.getFullyQualifiedName(), + e.getMessage()); + return null; + } + } + + private Map> collectDescendantIds(String fqnHashPrefix) { + Map> descendantsByType = new HashMap<>(); + for (String entityType : Entity.getEntityList()) { + try { + EntityRepository repo = Entity.getEntityRepository(entityType); + List ids = repo.getDao().findIdsByFqnHashPrefix(fqnHashPrefix); + if (!ids.isEmpty()) { + descendantsByType.put(entityType, ids); + } + } catch (Exception e) { + LOG.debug("Skipping type {} during descendant collection: {}", entityType, e.getMessage()); + } + } + return descendantsByType; + } + + private List buildAllIds(UUID rootId, Map> descendantsByType) { + List allIds = new ArrayList<>(); + allIds.add(rootId.toString()); + for (List ids : descendantsByType.values()) { + for (UUID id : ids) { + allIds.add(id.toString()); + } + } + return allIds; + } + + private void deleteDependencyTables(String rootFqn, String fqnHashPrefix, List allIds) { + CollectionDAO dao = Entity.getCollectionDAO(); + dao.relationshipDAO().deleteAllByFqnHashPrefix(fqnHashPrefix); + dao.fieldRelationshipDAO().deleteAllByPrefix(rootFqn); + dao.entityExtensionDAO().deleteAllBatch(allIds); + dao.entityExtensionTimeSeriesDao().deleteByFqnHashPrefix(fqnHashPrefix); + dao.tagUsageDAO().deleteTagLabelsByTargetPrefix(rootFqn); + dao.usageDAO().deleteBatch(allIds); + List allUuids = allIds.stream().map(UUID::fromString).toList(); + Entity.getFeedRepository().deleteByAboutBatch(allUuids); + } + + private void runEntityHooks(EntityInterface rootEntity, String deletedBy, String fqnHashPrefix) { + String rootType = rootEntity.getEntityReference().getType(); + try { + Entity.getEntityRepository(rootType).callPreDelete(rootEntity, deletedBy); + } catch (Exception e) { + LOG.warn( + "preDelete hook failed for {}: {}", rootEntity.getFullyQualifiedName(), e.getMessage()); + } + for (String entityType : Entity.getEntityList()) { + try { + Entity.getEntityRepository(entityType).deleteTimeSeriesByFqnPrefix(fqnHashPrefix); + } catch (Exception e) { + LOG.debug("deleteTimeSeriesByFqnPrefix failed for type {}: {}", entityType, e.getMessage()); + } + try { + Entity.getEntityRepository(entityType).preDeleteByFqnHashPrefix(fqnHashPrefix, deletedBy); + } catch (Exception e) { + LOG.debug("preDeleteByFqnHashPrefix failed for type {}: {}", entityType, e.getMessage()); + } + } + } + + private void runPostEntityHooks(String fqnHashPrefix) { + for (String entityType : Entity.getEntityList()) { + try { + Entity.getEntityRepository(entityType).postDeleteByFqnHashPrefix(fqnHashPrefix); + } catch (Exception e) { + LOG.debug("postDeleteByFqnHashPrefix failed for type {}: {}", entityType, e.getMessage()); + } + } + } + + private void deleteEntityTables( + Map> descendantsByType, EntityInterface rootEntity) { + for (Map.Entry> entry : descendantsByType.entrySet()) { + String entityType = entry.getKey(); + List ids = entry.getValue().stream().map(UUID::toString).toList(); + try { + Entity.getEntityRepository(entityType).getDao().deleteBatch(ids); + } catch (Exception e) { + LOG.warn( + "Failed to delete {} entities of type {}: {}", ids.size(), entityType, e.getMessage()); + } + } + String rootType = rootEntity.getEntityReference().getType(); + EntityRepository rootRepo = Entity.getEntityRepository(rootType); + rootRepo.getDao().delete(rootEntity.getId()); + rootRepo.invalidateEntity(rootEntity); + } + + private void cleanSearchIndex(EntityInterface rootEntity, Set descendantTypes) { + SearchRepository searchRepo = Entity.getSearchRepository(); + if (searchRepo == null) { + return; + } + String rootFqn = rootEntity.getFullyQualifiedName(); + String rootType = rootEntity.getEntityReference().getType(); + searchRepo.deleteByEntityTypeFqnPrefix(rootType, rootFqn); + for (String entityType : descendantTypes) { + searchRepo.deleteByEntityTypeFqnPrefix(entityType, rootFqn); + } + } + + private void emitDeleteEvent(EntityInterface rootEntity) { + try { + EntityLifecycleEventDispatcher.getInstance().onEntityDeleted(rootEntity, null); + } catch (Exception e) { + LOG.warn( + "Failed to emit delete event for {}: {}", + rootEntity.getFullyQualifiedName(), + e.getMessage()); + } + } + + private void releaseLock(DeletionLock lock, EntityInterface entity) { + if (lock == null || lockManager == null) { + return; + } + try { + lockManager.releaseDeletionLock(entity.getId(), entity.getEntityReference().getType()); + } catch (Exception e) { + LOG.warn( + "Failed to release deletion lock for {}: {}", + entity.getFullyQualifiedName(), + e.getMessage()); + } + } +} diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/TableRepository.java b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/TableRepository.java index 81f4fa8c1ab8..bea11248d43d 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/TableRepository.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/TableRepository.java @@ -1632,13 +1632,16 @@ private void collectColumnFqns(List columns, List columnFqns) { @Override public void storeRelationships(Table table) { - // Add relationship from database to table addRelationship( table.getDatabaseSchema().getId(), + table.getDatabaseSchema().getFullyQualifiedName(), table.getId(), + table.getFullyQualifiedName(), DATABASE_SCHEMA, TABLE, - Relationship.CONTAINS); + Relationship.CONTAINS, + null, + false); } @Override diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/WorkflowDefinitionRepository.java b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/WorkflowDefinitionRepository.java index 3c3178a2ab83..2e3f5e8cc437 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/WorkflowDefinitionRepository.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/WorkflowDefinitionRepository.java @@ -14,6 +14,7 @@ import org.openmetadata.schema.governance.workflows.elements.EdgeDefinition; import org.openmetadata.schema.governance.workflows.elements.WorkflowNodeDefinitionInterface; import org.openmetadata.schema.type.EntityReference; +import org.openmetadata.schema.type.Include; import org.openmetadata.schema.type.change.ChangeSource; import org.openmetadata.schema.utils.JsonUtils; import org.openmetadata.service.Entity; @@ -54,6 +55,22 @@ protected void postUpdate(WorkflowDefinition original, WorkflowDefinition update WorkflowHandler.getInstance().deploy(new Workflow(updated)); } + @Override + protected void preDeleteByFqnHashPrefix(String fqnHashPrefix, String deletedBy) { + if (!WorkflowHandler.isInitialized()) { + return; + } + List ids = getDao().findIdsByFqnHashPrefix(fqnHashPrefix); + for (UUID id : ids) { + try { + WorkflowDefinition definition = find(id, Include.ALL); + WorkflowHandler.getInstance().deleteWorkflowDefinition(definition); + } catch (Exception e) { + LOG.warn("Failed to delete workflow definition {}: {}", id, e.getMessage()); + } + } + } + @Override protected void postDelete(WorkflowDefinition entity, boolean hardDelete) { super.postDelete(entity, hardDelete); diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/WorkflowRepository.java b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/WorkflowRepository.java index d1b3ca6173d1..d6d5728a674b 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/WorkflowRepository.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/WorkflowRepository.java @@ -4,8 +4,11 @@ import java.util.ArrayList; import java.util.List; +import java.util.UUID; +import lombok.extern.slf4j.Slf4j; import org.jdbi.v3.sqlobject.transaction.Transaction; import org.openmetadata.schema.entity.automations.Workflow; +import org.openmetadata.schema.type.Include; import org.openmetadata.schema.type.change.ChangeSource; import org.openmetadata.service.Entity; import org.openmetadata.service.resources.automations.WorkflowResource; @@ -14,6 +17,7 @@ import org.openmetadata.service.util.EntityUtil; import org.openmetadata.service.util.EntityUtil.RelationIncludes; +@Slf4j public class WorkflowRepository extends EntityRepository { private static final String PATCH_FIELDS = "status,response"; @@ -80,6 +84,19 @@ public void storeEntities(List workflows) { dao.insertMany(dao.getTableName(), dao.getNameHashColumn(), fqns, jsons); } + @Override + protected void preDeleteByFqnHashPrefix(String fqnHashPrefix, String deletedBy) { + List ids = getDao().findIdsByFqnHashPrefix(fqnHashPrefix); + for (UUID id : ids) { + try { + Workflow workflow = find(id, Include.ALL); + SecretsManagerFactory.getSecretsManager().deleteSecretsFromWorkflow(workflow); + } catch (Exception e) { + LOG.warn("Failed to delete secrets for workflow {}: {}", id, e.getMessage()); + } + } + } + /** Remove the secrets from the secret manager */ @Override protected void postDelete(Workflow workflow, boolean hardDelete) { diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/migration/mysql/v1130/Migration.java b/openmetadata-service/src/main/java/org/openmetadata/service/migration/mysql/v1130/Migration.java index d146890469f6..b887c80ee789 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/migration/mysql/v1130/Migration.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/migration/mysql/v1130/Migration.java @@ -1,10 +1,12 @@ package org.openmetadata.service.migration.mysql.v1130; import lombok.SneakyThrows; +import lombok.extern.slf4j.Slf4j; import org.openmetadata.service.migration.api.MigrationProcessImpl; import org.openmetadata.service.migration.utils.MigrationFile; import org.openmetadata.service.migration.utils.v1130.MigrationUtil; +@Slf4j public class Migration extends MigrationProcessImpl { public Migration(MigrationFile migrationFile) { @@ -15,5 +17,13 @@ public Migration(MigrationFile migrationFile) { @SneakyThrows public void runDataMigration() { MigrationUtil.updateOwnerChartFormulas(); + try { + MigrationUtil.backfillRelationshipFqnHashes(handle); + } catch (Exception e) { + LOG.error( + "Failed to backfill FQN hashes in entity_relationship during v1130 migration. " + + "Fast prefix deletion may not work correctly for pre-existing relationships.", + e); + } } } diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/migration/mysql/v1141/Migration.java b/openmetadata-service/src/main/java/org/openmetadata/service/migration/mysql/v1141/Migration.java new file mode 100644 index 000000000000..7d381de87393 --- /dev/null +++ b/openmetadata-service/src/main/java/org/openmetadata/service/migration/mysql/v1141/Migration.java @@ -0,0 +1,24 @@ +/* + * Copyright 2021 Collate + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.openmetadata.service.migration.mysql.v1141; + +import org.openmetadata.service.migration.api.MigrationProcessImpl; +import org.openmetadata.service.migration.utils.MigrationFile; + +public class Migration extends MigrationProcessImpl { + + public Migration(MigrationFile migrationFile) { + super(migrationFile); + } +} diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/migration/postgres/v1130/Migration.java b/openmetadata-service/src/main/java/org/openmetadata/service/migration/postgres/v1130/Migration.java index 909ea509319d..570eda6bf4ab 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/migration/postgres/v1130/Migration.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/migration/postgres/v1130/Migration.java @@ -1,10 +1,12 @@ package org.openmetadata.service.migration.postgres.v1130; import lombok.SneakyThrows; +import lombok.extern.slf4j.Slf4j; import org.openmetadata.service.migration.api.MigrationProcessImpl; import org.openmetadata.service.migration.utils.MigrationFile; import org.openmetadata.service.migration.utils.v1130.MigrationUtil; +@Slf4j public class Migration extends MigrationProcessImpl { public Migration(MigrationFile migrationFile) { @@ -15,5 +17,13 @@ public Migration(MigrationFile migrationFile) { @SneakyThrows public void runDataMigration() { MigrationUtil.updateOwnerChartFormulas(); + try { + MigrationUtil.backfillRelationshipFqnHashes(handle); + } catch (Exception e) { + LOG.error( + "Failed to backfill FQN hashes in entity_relationship during v1130 migration. " + + "Fast prefix deletion may not work correctly for pre-existing relationships.", + e); + } } } diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/migration/postgres/v1141/Migration.java b/openmetadata-service/src/main/java/org/openmetadata/service/migration/postgres/v1141/Migration.java new file mode 100644 index 000000000000..e0c6d4f761f8 --- /dev/null +++ b/openmetadata-service/src/main/java/org/openmetadata/service/migration/postgres/v1141/Migration.java @@ -0,0 +1,24 @@ +/* + * Copyright 2021 Collate + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.openmetadata.service.migration.postgres.v1141; + +import org.openmetadata.service.migration.api.MigrationProcessImpl; +import org.openmetadata.service.migration.utils.MigrationFile; + +public class Migration extends MigrationProcessImpl { + + public Migration(MigrationFile migrationFile) { + super(migrationFile); + } +} diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/migration/utils/v1130/MigrationUtil.java b/openmetadata-service/src/main/java/org/openmetadata/service/migration/utils/v1130/MigrationUtil.java index f5005e76f4b3..d6ab1addbc73 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/migration/utils/v1130/MigrationUtil.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/migration/utils/v1130/MigrationUtil.java @@ -1,17 +1,87 @@ package org.openmetadata.service.migration.utils.v1130; import lombok.extern.slf4j.Slf4j; +import org.jdbi.v3.core.Handle; import org.openmetadata.schema.dataInsight.custom.DataInsightCustomChart; +import org.openmetadata.service.Entity; import org.openmetadata.service.jdbi3.DataInsightSystemChartRepository; +import org.openmetadata.service.jdbi3.EntityRepository; import org.openmetadata.service.util.EntityUtil; @Slf4j public class MigrationUtil { private MigrationUtil() {} + private static final String FQNHASH_COL = "fqnHash"; + private static final String NAMEHASH_COL = "nameHash"; + private static final String OLD_FIELD = "owners.name.keyword"; private static final String NEW_FIELD = "ownerName"; + /** + * Backfills fromFQNHash and toFQNHash in entity_relationship for all registered entity types. + * Uses a direct correlated-subquery UPDATE (one per direction per entity type) to avoid + * LIMIT/OFFSET pagination ordering bugs that could silently skip rows. + */ + public static void backfillRelationshipFqnHashes(Handle handle) { + for (String entityType : Entity.getEntityList()) { + try { + backfillForEntityType(handle, entityType); + } catch (Exception e) { + LOG.warn( + "Failed to backfill FQN hashes for entity type {}: {}", entityType, e.getMessage()); + } + } + } + + private static void backfillForEntityType(Handle handle, String entityType) { + EntityRepository repo = Entity.getEntityRepository(entityType); + String hashCol = repo.getDao().getNameHashColumn(); + if (!FQNHASH_COL.equals(hashCol) && !NAMEHASH_COL.equals(hashCol)) { + return; + } + String tableName = repo.getDao().getTableName(); + int fromUpdated = updateFromHashes(handle, entityType, tableName, hashCol); + int toUpdated = updateToHashes(handle, entityType, tableName, hashCol); + if (fromUpdated + toUpdated > 0) { + LOG.info( + "Backfilled FQN hashes for entity type {}: {} fromFQNHash, {} toFQNHash", + entityType, + fromUpdated, + toUpdated); + } + } + + private static int updateFromHashes( + Handle handle, String entityType, String tableName, String hashCol) { + return handle + .createUpdate( + "UPDATE entity_relationship SET fromFQNHash = (" + + "SELECT CAST(t." + + hashCol + + " AS CHAR(768)) FROM " + + tableName + + " t WHERE CAST(t.id AS CHAR(36)) = entity_relationship.fromId" + + ") WHERE fromEntity = :entityType AND fromFQNHash IS NULL") + .bind("entityType", entityType) + .execute(); + } + + private static int updateToHashes( + Handle handle, String entityType, String tableName, String hashCol) { + return handle + .createUpdate( + "UPDATE entity_relationship SET toFQNHash = (" + + "SELECT CAST(t." + + hashCol + + " AS CHAR(768)) FROM " + + tableName + + " t WHERE CAST(t.id AS CHAR(36)) = entity_relationship.toId" + + ") WHERE toEntity = :entityType AND toFQNHash IS NULL") + .bind("entityType", entityType) + .execute(); + } + public static void updateOwnerChartFormulas() { DataInsightSystemChartRepository repository = new DataInsightSystemChartRepository(); String[] chartNames = { diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/resources/EntityResource.java b/openmetadata-service/src/main/java/org/openmetadata/service/resources/EntityResource.java index bee61836e6c8..a8f3c21945d9 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/resources/EntityResource.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/resources/EntityResource.java @@ -73,6 +73,7 @@ import org.openmetadata.service.exception.CatalogExceptionMessage; import org.openmetadata.service.jdbi3.EntityRepository; import org.openmetadata.service.jdbi3.ListFilter; +import org.openmetadata.service.jdbi3.PrefixDeletionService; import org.openmetadata.service.limits.Limits; import org.openmetadata.service.mapper.EntityMapper; import org.openmetadata.service.monitoring.LatencyPhase; @@ -704,6 +705,43 @@ public Response deleteByIdAsync( return response; } + public Response deletePrefixHardById(UriInfo uriInfo, SecurityContext securityContext, UUID id) { + String jobId = UUID.randomUUID().toString(); + OperationContext operationContext = new OperationContext(entityType, MetadataOperation.DELETE); + authorizer.authorize( + securityContext, + operationContext, + getResourceContextById(id, ResourceContextInterface.Operation.DELETE)); + T entity = repository.get(uriInfo, id, repository.getFields("name"), Include.ALL, false); + String userName = securityContext.getUserPrincipal().getName(); + ExecutorService executorService = AsyncService.getInstance().getExecutorService(); + executorService.submit( + RequestLatencyContext.wrapWithContext( + () -> { + try { + PrefixDeletionService.getInstance().deletePrefixHard(entity, userName); + limits.invalidateCache(entityType); + WebsocketNotificationHandler.sendDeleteOperationCompleteNotification( + jobId, securityContext, entity); + } catch (Exception e) { + WebsocketNotificationHandler.sendDeleteOperationFailedNotification( + jobId, + securityContext, + entity, + e.getMessage() == null ? e.toString() : e.getMessage()); + } + })); + return Response.accepted() + .entity( + new DeleteEntityResponse( + jobId, + "Fast prefix deletion initiated for " + entity.getName(), + entity.getName(), + true, + true)) + .build(); + } + public Response deleteByName( UriInfo uriInfo, SecurityContext securityContext, diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/resources/ai/AIApplicationResource.java b/openmetadata-service/src/main/java/org/openmetadata/service/resources/ai/AIApplicationResource.java index 2a8d91b6ac11..6e1f364bd038 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/resources/ai/AIApplicationResource.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/resources/ai/AIApplicationResource.java @@ -510,6 +510,27 @@ public Response deleteByIdAsync( return deleteByIdAsync(uriInfo, securityContext, id, false, hardDelete); } + @DELETE + @Path("/prefix/{id}") + @Operation( + operationId = "deleteAiApplicationPrefixHard", + summary = "Hard-delete a AI application and all descendants by FQN prefix", + description = + "Bulk hard-delete this AI application and all descendants whose FQN starts with this " + + "entity's FQN. Significantly faster than recursive delete for large hierarchies.", + responses = { + @ApiResponse(responseCode = "202", description = "Deletion accepted and running"), + @ApiResponse(responseCode = "404", description = "Entity for instance {id} is not found") + }) + public Response deletePrefixHardById( + @Context UriInfo uriInfo, + @Context SecurityContext securityContext, + @Parameter(description = "Id of the AI application", schema = @Schema(type = "UUID")) + @PathParam("id") + UUID id) { + return super.deletePrefixHardById(uriInfo, securityContext, id); + } + @DELETE @Path("/name/{fqn}") @Operation( diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/resources/ai/AIGovernancePolicyResource.java b/openmetadata-service/src/main/java/org/openmetadata/service/resources/ai/AIGovernancePolicyResource.java index a469b705e839..4cbf1a456a2b 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/resources/ai/AIGovernancePolicyResource.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/resources/ai/AIGovernancePolicyResource.java @@ -497,6 +497,27 @@ public Response deleteByIdAsync( return deleteByIdAsync(uriInfo, securityContext, id, recursive, hardDelete); } + @DELETE + @Path("/prefix/{id}") + @Operation( + operationId = "deleteAiGovernancePolicyPrefixHard", + summary = "Hard-delete a AI governance policy and all descendants by FQN prefix", + description = + "Bulk hard-delete this AI governance policy and all descendants whose FQN starts with this " + + "entity's FQN. Significantly faster than recursive delete for large hierarchies.", + responses = { + @ApiResponse(responseCode = "202", description = "Deletion accepted and running"), + @ApiResponse(responseCode = "404", description = "Entity for instance {id} is not found") + }) + public Response deletePrefixHardById( + @Context UriInfo uriInfo, + @Context SecurityContext securityContext, + @Parameter(description = "Id of the AI governance policy", schema = @Schema(type = "UUID")) + @PathParam("id") + UUID id) { + return super.deletePrefixHardById(uriInfo, securityContext, id); + } + @DELETE @Path("/name/{fqn}") @Operation( diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/resources/ai/LLMModelResource.java b/openmetadata-service/src/main/java/org/openmetadata/service/resources/ai/LLMModelResource.java index 1bfbc7aeb4fe..ce357b08fb4e 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/resources/ai/LLMModelResource.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/resources/ai/LLMModelResource.java @@ -477,6 +477,27 @@ public Response delete( return delete(uriInfo, securityContext, id, false, hardDelete); } + @DELETE + @Path("/prefix/{id}") + @Operation( + operationId = "deleteLlmModelPrefixHard", + summary = "Hard-delete a LLM model and all descendants by FQN prefix", + description = + "Bulk hard-delete this LLM model and all descendants whose FQN starts with this " + + "entity's FQN. Significantly faster than recursive delete for large hierarchies.", + responses = { + @ApiResponse(responseCode = "202", description = "Deletion accepted and running"), + @ApiResponse(responseCode = "404", description = "Entity for instance {id} is not found") + }) + public Response deletePrefixHardById( + @Context UriInfo uriInfo, + @Context SecurityContext securityContext, + @Parameter(description = "Id of the LLM model", schema = @Schema(type = "UUID")) + @PathParam("id") + UUID id) { + return super.deletePrefixHardById(uriInfo, securityContext, id); + } + @DELETE @Path("/name/{fqn}") @Operation( diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/resources/ai/McpServerResource.java b/openmetadata-service/src/main/java/org/openmetadata/service/resources/ai/McpServerResource.java index 7a602ee98364..26ca0249bf47 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/resources/ai/McpServerResource.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/resources/ai/McpServerResource.java @@ -531,6 +531,27 @@ public Response deleteByIdAsync( return deleteByIdAsync(uriInfo, securityContext, id, recursive, hardDelete); } + @DELETE + @Path("/prefix/{id}") + @Operation( + operationId = "deleteMcpServerPrefixHard", + summary = "Hard-delete a MCP server and all descendants by FQN prefix", + description = + "Bulk hard-delete this MCP server and all descendants whose FQN starts with this " + + "entity's FQN. Significantly faster than recursive delete for large hierarchies.", + responses = { + @ApiResponse(responseCode = "202", description = "Deletion accepted and running"), + @ApiResponse(responseCode = "404", description = "Entity for instance {id} is not found") + }) + public Response deletePrefixHardById( + @Context UriInfo uriInfo, + @Context SecurityContext securityContext, + @Parameter(description = "Id of the MCP server", schema = @Schema(type = "UUID")) + @PathParam("id") + UUID id) { + return super.deletePrefixHardById(uriInfo, securityContext, id); + } + @DELETE @Path("/name/{fqn}") @Operation( diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/resources/ai/PromptTemplateResource.java b/openmetadata-service/src/main/java/org/openmetadata/service/resources/ai/PromptTemplateResource.java index 416ec1d43bdb..6980877e2559 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/resources/ai/PromptTemplateResource.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/resources/ai/PromptTemplateResource.java @@ -494,6 +494,27 @@ public Response deleteByIdAsync( return deleteByIdAsync(uriInfo, securityContext, id, recursive, hardDelete); } + @DELETE + @Path("/prefix/{id}") + @Operation( + operationId = "deletePromptTemplatePrefixHard", + summary = "Hard-delete a prompt template and all descendants by FQN prefix", + description = + "Bulk hard-delete this prompt template and all descendants whose FQN starts with this " + + "entity's FQN. Significantly faster than recursive delete for large hierarchies.", + responses = { + @ApiResponse(responseCode = "202", description = "Deletion accepted and running"), + @ApiResponse(responseCode = "404", description = "Entity for instance {id} is not found") + }) + public Response deletePrefixHardById( + @Context UriInfo uriInfo, + @Context SecurityContext securityContext, + @Parameter(description = "Id of the prompt template", schema = @Schema(type = "UUID")) + @PathParam("id") + UUID id) { + return super.deletePrefixHardById(uriInfo, securityContext, id); + } + @DELETE @Path("/name/{fqn}") @Operation( diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/resources/analytics/WebAnalyticEventResource.java b/openmetadata-service/src/main/java/org/openmetadata/service/resources/analytics/WebAnalyticEventResource.java index d7471083d91c..8b002e216747 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/resources/analytics/WebAnalyticEventResource.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/resources/analytics/WebAnalyticEventResource.java @@ -352,6 +352,27 @@ public Response deleteByIdAsync( return deleteByIdAsync(uriInfo, securityContext, id, false, hardDelete); } + @DELETE + @Path("/prefix/{id}") + @Operation( + operationId = "deleteWebAnalyticEventPrefixHard", + summary = "Hard-delete a web analytic event and all descendants by FQN prefix", + description = + "Bulk hard-delete this web analytic event and all descendants whose FQN starts with this " + + "entity's FQN. Significantly faster than recursive delete for large hierarchies.", + responses = { + @ApiResponse(responseCode = "202", description = "Deletion accepted and running"), + @ApiResponse(responseCode = "404", description = "Entity for instance {id} is not found") + }) + public Response deletePrefixHardById( + @Context UriInfo uriInfo, + @Context SecurityContext securityContext, + @Parameter(description = "Id of the web analytic event", schema = @Schema(type = "UUID")) + @PathParam("id") + UUID id) { + return super.deletePrefixHardById(uriInfo, securityContext, id); + } + @DELETE @Path("/name/{fqn}") @Operation( diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/resources/apis/APICollectionResource.java b/openmetadata-service/src/main/java/org/openmetadata/service/resources/apis/APICollectionResource.java index 3f575db7f5c2..82c90e3ce9d1 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/resources/apis/APICollectionResource.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/resources/apis/APICollectionResource.java @@ -545,6 +545,27 @@ public Response updateVote( .toResponse(); } + @DELETE + @Path("/prefix/{id}") + @Operation( + operationId = "deleteApiCollectionPrefixHard", + summary = "Hard-delete a API collection and all descendants by FQN prefix", + description = + "Bulk hard-delete this API collection and all descendants whose FQN starts with this " + + "entity's FQN. Significantly faster than recursive delete for large hierarchies.", + responses = { + @ApiResponse(responseCode = "202", description = "Deletion accepted and running"), + @ApiResponse(responseCode = "404", description = "Entity for instance {id} is not found") + }) + public Response deletePrefixHardById( + @Context UriInfo uriInfo, + @Context SecurityContext securityContext, + @Parameter(description = "Id of the API collection", schema = @Schema(type = "UUID")) + @PathParam("id") + UUID id) { + return super.deletePrefixHardById(uriInfo, securityContext, id); + } + @DELETE @Path("/name/{fqn}") @Operation( diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/resources/apis/APIEndpointResource.java b/openmetadata-service/src/main/java/org/openmetadata/service/resources/apis/APIEndpointResource.java index 4f76cd9e7123..aac911877929 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/resources/apis/APIEndpointResource.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/resources/apis/APIEndpointResource.java @@ -591,6 +591,27 @@ public Response deleteByIdAsync( return deleteByIdAsync(uriInfo, securityContext, id, false, hardDelete); } + @DELETE + @Path("/prefix/{id}") + @Operation( + operationId = "deleteApiEndpointPrefixHard", + summary = "Hard-delete a API endpoint and all descendants by FQN prefix", + description = + "Bulk hard-delete this API endpoint and all descendants whose FQN starts with this " + + "entity's FQN. Significantly faster than recursive delete for large hierarchies.", + responses = { + @ApiResponse(responseCode = "202", description = "Deletion accepted and running"), + @ApiResponse(responseCode = "404", description = "Entity for instance {id} is not found") + }) + public Response deletePrefixHardById( + @Context UriInfo uriInfo, + @Context SecurityContext securityContext, + @Parameter(description = "Id of the API endpoint", schema = @Schema(type = "UUID")) + @PathParam("id") + UUID id) { + return super.deletePrefixHardById(uriInfo, securityContext, id); + } + @DELETE @Path("/name/{fqn}") @Operation( diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/resources/apps/AppMarketPlaceResource.java b/openmetadata-service/src/main/java/org/openmetadata/service/resources/apps/AppMarketPlaceResource.java index 168650c7a4cf..4cb59f641ec2 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/resources/apps/AppMarketPlaceResource.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/resources/apps/AppMarketPlaceResource.java @@ -374,6 +374,29 @@ public Response createOrUpdate( return createOrUpdate(uriInfo, securityContext, app); } + @DELETE + @Path("/prefix/{id}") + @Operation( + operationId = "deleteAppMarketPlacePrefixHard", + summary = "Hard-delete a app marketplace definition and all descendants by FQN prefix", + description = + "Bulk hard-delete this app marketplace definition and all descendants whose FQN starts with this " + + "entity's FQN. Significantly faster than recursive delete for large hierarchies.", + responses = { + @ApiResponse(responseCode = "202", description = "Deletion accepted and running"), + @ApiResponse(responseCode = "404", description = "Entity for instance {id} is not found") + }) + public Response deletePrefixHardById( + @Context UriInfo uriInfo, + @Context SecurityContext securityContext, + @Parameter( + description = "Id of the app marketplace definition", + schema = @Schema(type = "UUID")) + @PathParam("id") + UUID id) { + return super.deletePrefixHardById(uriInfo, securityContext, id); + } + @DELETE @Path("/name/{name}") @Operation( diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/resources/apps/AppResource.java b/openmetadata-service/src/main/java/org/openmetadata/service/resources/apps/AppResource.java index 5ab4f8ac5e5e..b67f5b27c3ab 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/resources/apps/AppResource.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/resources/apps/AppResource.java @@ -921,6 +921,26 @@ public Response createOrUpdate( return createOrUpdate(uriInfo, securityContext, app); } + @DELETE + @Path("/prefix/{id}") + @Operation( + operationId = "deleteAppPrefixHard", + summary = "Hard-delete a app and all descendants by FQN prefix", + description = + "Bulk hard-delete this app and all descendants whose FQN starts with this " + + "entity's FQN. Significantly faster than recursive delete for large hierarchies.", + responses = { + @ApiResponse(responseCode = "202", description = "Deletion accepted and running"), + @ApiResponse(responseCode = "404", description = "Entity for instance {id} is not found") + }) + public Response deletePrefixHardById( + @Context UriInfo uriInfo, + @Context SecurityContext securityContext, + @Parameter(description = "Id of the app", schema = @Schema(type = "UUID")) @PathParam("id") + UUID id) { + return super.deletePrefixHardById(uriInfo, securityContext, id); + } + @DELETE @Path("/name/{name}") @Operation( diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/resources/automations/WorkflowResource.java b/openmetadata-service/src/main/java/org/openmetadata/service/resources/automations/WorkflowResource.java index 6e97b951fc10..180816170730 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/resources/automations/WorkflowResource.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/resources/automations/WorkflowResource.java @@ -529,6 +529,27 @@ public Response deleteByIdAsync( return deleteByIdAsync(uriInfo, securityContext, id, false, hardDelete); } + @DELETE + @Path("/prefix/{id}") + @Operation( + operationId = "deleteWorkflowPrefixHard", + summary = "Hard-delete a workflow and all descendants by FQN prefix", + description = + "Bulk hard-delete this workflow and all descendants whose FQN starts with this " + + "entity's FQN. Significantly faster than recursive delete for large hierarchies.", + responses = { + @ApiResponse(responseCode = "202", description = "Deletion accepted and running"), + @ApiResponse(responseCode = "404", description = "Entity for instance {id} is not found") + }) + public Response deletePrefixHardById( + @Context UriInfo uriInfo, + @Context SecurityContext securityContext, + @Parameter(description = "Id of the workflow", schema = @Schema(type = "UUID")) + @PathParam("id") + UUID id) { + return super.deletePrefixHardById(uriInfo, securityContext, id); + } + @DELETE @Path("/name/{name}") @Operation( diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/resources/bots/BotResource.java b/openmetadata-service/src/main/java/org/openmetadata/service/resources/bots/BotResource.java index 6175e4eac8fd..c2cad05c9c3a 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/resources/bots/BotResource.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/resources/bots/BotResource.java @@ -435,6 +435,26 @@ public Response deleteByIdAsync( return deleteByIdAsync(uriInfo, securityContext, id, true, hardDelete); } + @DELETE + @Path("/prefix/{id}") + @Operation( + operationId = "deleteBotPrefixHard", + summary = "Hard-delete a bot and all descendants by FQN prefix", + description = + "Bulk hard-delete this bot and all descendants whose FQN starts with this " + + "entity's FQN. Significantly faster than recursive delete for large hierarchies.", + responses = { + @ApiResponse(responseCode = "202", description = "Deletion accepted and running"), + @ApiResponse(responseCode = "404", description = "Entity for instance {id} is not found") + }) + public Response deletePrefixHardById( + @Context UriInfo uriInfo, + @Context SecurityContext securityContext, + @Parameter(description = "Id of the bot", schema = @Schema(type = "UUID")) @PathParam("id") + UUID id) { + return super.deletePrefixHardById(uriInfo, securityContext, id); + } + @DELETE @Path("/name/{name}") @Operation( diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/resources/charts/ChartResource.java b/openmetadata-service/src/main/java/org/openmetadata/service/resources/charts/ChartResource.java index 96ff75878e9e..ece8379534d1 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/resources/charts/ChartResource.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/resources/charts/ChartResource.java @@ -530,6 +530,26 @@ public Response deleteByIdAsync( return deleteByIdAsync(uriInfo, securityContext, id, false, hardDelete); } + @DELETE + @Path("/prefix/{id}") + @Operation( + operationId = "deleteChartPrefixHard", + summary = "Hard-delete a chart and all descendants by FQN prefix", + description = + "Bulk hard-delete this chart and all descendants whose FQN starts with this " + + "entity's FQN. Significantly faster than recursive delete for large hierarchies.", + responses = { + @ApiResponse(responseCode = "202", description = "Deletion accepted and running"), + @ApiResponse(responseCode = "404", description = "Entity for instance {id} is not found") + }) + public Response deletePrefixHardById( + @Context UriInfo uriInfo, + @Context SecurityContext securityContext, + @Parameter(description = "Id of the chart", schema = @Schema(type = "UUID")) @PathParam("id") + UUID id) { + return super.deletePrefixHardById(uriInfo, securityContext, id); + } + @DELETE @Path("/name/{fqn}") @Operation( diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/resources/dashboards/DashboardResource.java b/openmetadata-service/src/main/java/org/openmetadata/service/resources/dashboards/DashboardResource.java index 3ac05657402a..b6fd4aba0a54 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/resources/dashboards/DashboardResource.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/resources/dashboards/DashboardResource.java @@ -575,6 +575,27 @@ public Response deleteByIdAsync( return deleteByIdAsync(uriInfo, securityContext, id, false, hardDelete); } + @DELETE + @Path("/prefix/{id}") + @Operation( + operationId = "deleteDashboardPrefixHard", + summary = "Hard-delete a dashboard and all descendants by FQN prefix", + description = + "Bulk hard-delete this dashboard and all descendants whose FQN starts with this " + + "entity's FQN. Significantly faster than recursive delete for large hierarchies.", + responses = { + @ApiResponse(responseCode = "202", description = "Deletion accepted and running"), + @ApiResponse(responseCode = "404", description = "Entity for instance {id} is not found") + }) + public Response deletePrefixHardById( + @Context UriInfo uriInfo, + @Context SecurityContext securityContext, + @Parameter(description = "Id of the dashboard", schema = @Schema(type = "UUID")) + @PathParam("id") + UUID id) { + return super.deletePrefixHardById(uriInfo, securityContext, id); + } + @DELETE @Path("/name/{fqn}") @Operation( diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/resources/data/DataContractResource.java b/openmetadata-service/src/main/java/org/openmetadata/service/resources/data/DataContractResource.java index eaffd7591f8c..f7c48ca20976 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/resources/data/DataContractResource.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/resources/data/DataContractResource.java @@ -550,6 +550,27 @@ public Response delete( return delete(uriInfo, securityContext, id, recursive, hardDelete); } + @DELETE + @Path("/prefix/{id}") + @Operation( + operationId = "deleteDataContractPrefixHard", + summary = "Hard-delete a data contract and all descendants by FQN prefix", + description = + "Bulk hard-delete this data contract and all descendants whose FQN starts with this " + + "entity's FQN. Significantly faster than recursive delete for large hierarchies.", + responses = { + @ApiResponse(responseCode = "202", description = "Deletion accepted and running"), + @ApiResponse(responseCode = "404", description = "Entity for instance {id} is not found") + }) + public Response deletePrefixHardById( + @Context UriInfo uriInfo, + @Context SecurityContext securityContext, + @Parameter(description = "Id of the data contract", schema = @Schema(type = "UUID")) + @PathParam("id") + UUID id) { + return super.deletePrefixHardById(uriInfo, securityContext, id); + } + @DELETE @Path("/name/{fqn}") @Operation( diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/resources/databases/DatabaseResource.java b/openmetadata-service/src/main/java/org/openmetadata/service/resources/databases/DatabaseResource.java index 200170085ec4..6335a77b9bb4 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/resources/databases/DatabaseResource.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/resources/databases/DatabaseResource.java @@ -715,6 +715,27 @@ public Response updateVote( .toResponse(); } + @DELETE + @Path("/prefix/{id}") + @Operation( + operationId = "deleteDatabasePrefixHard", + summary = "Hard-delete a database and all descendants by FQN prefix", + description = + "Bulk hard-delete this database and all descendants whose FQN starts with this " + + "entity's FQN. Significantly faster than recursive delete for large hierarchies.", + responses = { + @ApiResponse(responseCode = "202", description = "Deletion accepted and running"), + @ApiResponse(responseCode = "404", description = "Entity for instance {id} is not found") + }) + public Response deletePrefixHardById( + @Context UriInfo uriInfo, + @Context SecurityContext securityContext, + @Parameter(description = "Id of the database", schema = @Schema(type = "UUID")) + @PathParam("id") + UUID id) { + return super.deletePrefixHardById(uriInfo, securityContext, id); + } + @DELETE @Path("/name/{fqn}") @Operation( diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/resources/databases/DatabaseSchemaResource.java b/openmetadata-service/src/main/java/org/openmetadata/service/resources/databases/DatabaseSchemaResource.java index 7be0e5fa9441..8efee5b4af2c 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/resources/databases/DatabaseSchemaResource.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/resources/databases/DatabaseSchemaResource.java @@ -778,6 +778,27 @@ public Response deleteByIdAsync( return deleteByIdAsync(uriInfo, securityContext, id, recursive, hardDelete); } + @DELETE + @Path("/prefix/{id}") + @Operation( + operationId = "deleteDatabaseSchemaPrefixHard", + summary = "Hard-delete a database schema and all descendants by FQN prefix", + description = + "Bulk hard-delete this database schema and all descendants whose FQN starts with this " + + "entity's FQN. Significantly faster than recursive delete for large hierarchies.", + responses = { + @ApiResponse(responseCode = "202", description = "Deletion accepted and running"), + @ApiResponse(responseCode = "404", description = "Entity for instance {id} is not found") + }) + public Response deletePrefixHardById( + @Context UriInfo uriInfo, + @Context SecurityContext securityContext, + @Parameter(description = "Id of the database schema", schema = @Schema(type = "UUID")) + @PathParam("id") + UUID id) { + return super.deletePrefixHardById(uriInfo, securityContext, id); + } + @DELETE @Path("/name/{fqn}") @Operation( diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/resources/databases/StoredProcedureResource.java b/openmetadata-service/src/main/java/org/openmetadata/service/resources/databases/StoredProcedureResource.java index 760da9de6b55..c1557e331983 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/resources/databases/StoredProcedureResource.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/resources/databases/StoredProcedureResource.java @@ -14,6 +14,8 @@ import jakarta.validation.constraints.Max; import jakarta.validation.constraints.Min; import jakarta.ws.rs.*; +import jakarta.ws.rs.DELETE; +import jakarta.ws.rs.PathParam; import jakarta.ws.rs.core.*; import java.util.List; import java.util.UUID; @@ -566,6 +568,27 @@ public Response deleteByIdAsync( return deleteByIdAsync(uriInfo, securityContext, id, recursive, hardDelete); } + @DELETE + @Path("/prefix/{id}") + @Operation( + operationId = "deleteStoredProcedurePrefixHard", + summary = "Hard-delete a stored procedure and all descendants by FQN prefix", + description = + "Bulk hard-delete this stored procedure and all descendants whose FQN starts with this " + + "entity's FQN. Significantly faster than recursive delete for large hierarchies.", + responses = { + @ApiResponse(responseCode = "202", description = "Deletion accepted and running"), + @ApiResponse(responseCode = "404", description = "Entity for instance {id} is not found") + }) + public Response deletePrefixHardById( + @Context UriInfo uriInfo, + @Context SecurityContext securityContext, + @Parameter(description = "Id of the stored procedure", schema = @Schema(type = "UUID")) + @PathParam("id") + UUID id) { + return super.deletePrefixHardById(uriInfo, securityContext, id); + } + @DELETE @Path("/name/{fqn}") @Operation( diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/resources/databases/TableResource.java b/openmetadata-service/src/main/java/org/openmetadata/service/resources/databases/TableResource.java index fea914d1726f..79c590bb2e0d 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/resources/databases/TableResource.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/resources/databases/TableResource.java @@ -750,6 +750,26 @@ public Response deleteByIdAsync( return deleteByIdAsync(uriInfo, securityContext, id, recursive, hardDelete); } + @DELETE + @Path("/prefix/{id}") + @Operation( + operationId = "deleteTablePrefixHard", + summary = "Hard-delete a table and all descendants by FQN prefix", + description = + "Bulk hard-delete this table and all descendants whose FQN starts with this " + + "entity's FQN. Significantly faster than recursive delete for large hierarchies.", + responses = { + @ApiResponse(responseCode = "202", description = "Deletion accepted and running"), + @ApiResponse(responseCode = "404", description = "Entity for instance {id} is not found") + }) + public Response deletePrefixHardById( + @Context UriInfo uriInfo, + @Context SecurityContext securityContext, + @Parameter(description = "Id of the table", schema = @Schema(type = "UUID")) @PathParam("id") + UUID id) { + return super.deletePrefixHardById(uriInfo, securityContext, id); + } + @DELETE @Path("/name/{fqn}") @Operation( diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/resources/datainsight/DataInsightChartResource.java b/openmetadata-service/src/main/java/org/openmetadata/service/resources/datainsight/DataInsightChartResource.java index 9a28ee188715..a29a8753576a 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/resources/datainsight/DataInsightChartResource.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/resources/datainsight/DataInsightChartResource.java @@ -454,6 +454,27 @@ public Response deleteByIdAsync( return deleteByIdAsync(uriInfo, securityContext, id, false, hardDelete); } + @DELETE + @Path("/prefix/{id}") + @Operation( + operationId = "deleteDataInsightChartPrefixHard", + summary = "Hard-delete a data insight chart and all descendants by FQN prefix", + description = + "Bulk hard-delete this data insight chart and all descendants whose FQN starts with this " + + "entity's FQN. Significantly faster than recursive delete for large hierarchies.", + responses = { + @ApiResponse(responseCode = "202", description = "Deletion accepted and running"), + @ApiResponse(responseCode = "404", description = "Entity for instance {id} is not found") + }) + public Response deletePrefixHardById( + @Context UriInfo uriInfo, + @Context SecurityContext securityContext, + @Parameter(description = "Id of the data insight chart", schema = @Schema(type = "UUID")) + @PathParam("id") + UUID id) { + return super.deletePrefixHardById(uriInfo, securityContext, id); + } + @DELETE @Path("/name/{fqn}") @Operation( diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/resources/datainsight/system/DataInsightSystemChartResource.java b/openmetadata-service/src/main/java/org/openmetadata/service/resources/datainsight/system/DataInsightSystemChartResource.java index dbb6468ee460..33f1c1b09e06 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/resources/datainsight/system/DataInsightSystemChartResource.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/resources/datainsight/system/DataInsightSystemChartResource.java @@ -24,6 +24,7 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.UUID; import org.openmetadata.schema.dataInsight.DataInsightChart; import org.openmetadata.schema.dataInsight.custom.DataInsightCustomChart; import org.openmetadata.schema.dataInsight.custom.DataInsightCustomChartResultList; @@ -305,4 +306,27 @@ public Response stopChartDataStreaming( return Response.status(Response.Status.INTERNAL_SERVER_ERROR).entity(errorResponse).build(); } } + + @DELETE + @Path("/prefix/{id}") + @Operation( + operationId = "deleteDataInsightSystemChartPrefixHard", + summary = "Hard-delete a data insight system chart and all descendants by FQN prefix", + description = + "Bulk hard-delete this data insight system chart and all descendants whose FQN starts with this " + + "entity's FQN. Significantly faster than recursive delete for large hierarchies.", + responses = { + @ApiResponse(responseCode = "202", description = "Deletion accepted and running"), + @ApiResponse(responseCode = "404", description = "Entity for instance {id} is not found") + }) + public Response deletePrefixHardById( + @Context UriInfo uriInfo, + @Context SecurityContext securityContext, + @Parameter( + description = "Id of the data insight system chart", + schema = @Schema(type = "UUID")) + @PathParam("id") + UUID id) { + return super.deletePrefixHardById(uriInfo, securityContext, id); + } } diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/resources/datamodels/DashboardDataModelResource.java b/openmetadata-service/src/main/java/org/openmetadata/service/resources/datamodels/DashboardDataModelResource.java index 8a69e00aa088..06d7629daca2 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/resources/datamodels/DashboardDataModelResource.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/resources/datamodels/DashboardDataModelResource.java @@ -588,6 +588,27 @@ public Response deleteByIdAsync( return deleteByIdAsync(uriInfo, securityContext, id, recursive, hardDelete); } + @DELETE + @Path("/prefix/{id}") + @Operation( + operationId = "deleteDashboardDataModelPrefixHard", + summary = "Hard-delete a dashboard data model and all descendants by FQN prefix", + description = + "Bulk hard-delete this dashboard data model and all descendants whose FQN starts with this " + + "entity's FQN. Significantly faster than recursive delete for large hierarchies.", + responses = { + @ApiResponse(responseCode = "202", description = "Deletion accepted and running"), + @ApiResponse(responseCode = "404", description = "Entity for instance {id} is not found") + }) + public Response deletePrefixHardById( + @Context UriInfo uriInfo, + @Context SecurityContext securityContext, + @Parameter(description = "Id of the dashboard data model", schema = @Schema(type = "UUID")) + @PathParam("id") + UUID id) { + return super.deletePrefixHardById(uriInfo, securityContext, id); + } + @DELETE @Path("/name/{fqn}") @Operation( diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/resources/docstore/DocStoreResource.java b/openmetadata-service/src/main/java/org/openmetadata/service/resources/docstore/DocStoreResource.java index cd5e19efe834..2324efe73476 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/resources/docstore/DocStoreResource.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/resources/docstore/DocStoreResource.java @@ -452,6 +452,27 @@ public Response deleteByIdAsync( return deleteByIdAsync(uriInfo, securityContext, id, false, true); } + @DELETE + @Path("/prefix/{id}") + @Operation( + operationId = "deleteDocStorePrefixHard", + summary = "Hard-delete a doc store and all descendants by FQN prefix", + description = + "Bulk hard-delete this doc store and all descendants whose FQN starts with this " + + "entity's FQN. Significantly faster than recursive delete for large hierarchies.", + responses = { + @ApiResponse(responseCode = "202", description = "Deletion accepted and running"), + @ApiResponse(responseCode = "404", description = "Entity for instance {id} is not found") + }) + public Response deletePrefixHardById( + @Context UriInfo uriInfo, + @Context SecurityContext securityContext, + @Parameter(description = "Id of the doc store", schema = @Schema(type = "UUID")) + @PathParam("id") + UUID id) { + return super.deletePrefixHardById(uriInfo, securityContext, id); + } + @DELETE @Path("/name/{name}") @Operation( diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/resources/domains/DataProductResource.java b/openmetadata-service/src/main/java/org/openmetadata/service/resources/domains/DataProductResource.java index 967d470c9ab5..405cf0040dd6 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/resources/domains/DataProductResource.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/resources/domains/DataProductResource.java @@ -921,6 +921,27 @@ public Response deleteByIdAsync( return deleteByIdAsync(uriInfo, securityContext, id, true, true); } + @DELETE + @Path("/prefix/{id}") + @Operation( + operationId = "deleteDataProductPrefixHard", + summary = "Hard-delete a data product and all descendants by FQN prefix", + description = + "Bulk hard-delete this data product and all descendants whose FQN starts with this " + + "entity's FQN. Significantly faster than recursive delete for large hierarchies.", + responses = { + @ApiResponse(responseCode = "202", description = "Deletion accepted and running"), + @ApiResponse(responseCode = "404", description = "Entity for instance {id} is not found") + }) + public Response deletePrefixHardById( + @Context UriInfo uriInfo, + @Context SecurityContext securityContext, + @Parameter(description = "Id of the data product", schema = @Schema(type = "UUID")) + @PathParam("id") + UUID id) { + return super.deletePrefixHardById(uriInfo, securityContext, id); + } + @DELETE @Path("/name/{name}") @Operation( diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/resources/domains/DomainResource.java b/openmetadata-service/src/main/java/org/openmetadata/service/resources/domains/DomainResource.java index 028df9d98a95..1142c80fc1fd 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/resources/domains/DomainResource.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/resources/domains/DomainResource.java @@ -496,6 +496,26 @@ public Response deleteByIdAsync( return deleteByIdAsync(uriInfo, securityContext, id, true, true); } + @DELETE + @Path("/prefix/{id}") + @Operation( + operationId = "deleteDomainPrefixHard", + summary = "Hard-delete a domain and all descendants by FQN prefix", + description = + "Bulk hard-delete this domain and all descendants whose FQN starts with this " + + "entity's FQN. Significantly faster than recursive delete for large hierarchies.", + responses = { + @ApiResponse(responseCode = "202", description = "Deletion accepted and running"), + @ApiResponse(responseCode = "404", description = "Entity for instance {id} is not found") + }) + public Response deletePrefixHardById( + @Context UriInfo uriInfo, + @Context SecurityContext securityContext, + @Parameter(description = "Id of the domain", schema = @Schema(type = "UUID")) @PathParam("id") + UUID id) { + return super.deletePrefixHardById(uriInfo, securityContext, id); + } + @DELETE @Path("/name/{name}") @Operation( diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/resources/dqtests/TestCaseResource.java b/openmetadata-service/src/main/java/org/openmetadata/service/resources/dqtests/TestCaseResource.java index f5140b493459..99f8d6f06f0b 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/resources/dqtests/TestCaseResource.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/resources/dqtests/TestCaseResource.java @@ -924,6 +924,27 @@ public Response deleteByIdAsync( return deleteByIdAsync(uriInfo, securityContext, id, recursive, hardDelete); } + @DELETE + @Path("/prefix/{id}") + @Operation( + operationId = "deleteTestCasePrefixHard", + summary = "Hard-delete a test case and all descendants by FQN prefix", + description = + "Bulk hard-delete this test case and all descendants whose FQN starts with this " + + "entity's FQN. Significantly faster than recursive delete for large hierarchies.", + responses = { + @ApiResponse(responseCode = "202", description = "Deletion accepted and running"), + @ApiResponse(responseCode = "404", description = "Entity for instance {id} is not found") + }) + public Response deletePrefixHardById( + @Context UriInfo uriInfo, + @Context SecurityContext securityContext, + @Parameter(description = "Id of the test case", schema = @Schema(type = "UUID")) + @PathParam("id") + UUID id) { + return super.deletePrefixHardById(uriInfo, securityContext, id); + } + @DELETE @Path("/name/{fqn}") @Operation( diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/resources/dqtests/TestDefinitionResource.java b/openmetadata-service/src/main/java/org/openmetadata/service/resources/dqtests/TestDefinitionResource.java index 923e699ee405..3c4a9fa0dbe7 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/resources/dqtests/TestDefinitionResource.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/resources/dqtests/TestDefinitionResource.java @@ -459,6 +459,27 @@ public Response deleteByIdAsync( return deleteByIdAsync(uriInfo, securityContext, id, recursive, hardDelete); } + @DELETE + @Path("/prefix/{id}") + @Operation( + operationId = "deleteTestDefinitionPrefixHard", + summary = "Hard-delete a test definition and all descendants by FQN prefix", + description = + "Bulk hard-delete this test definition and all descendants whose FQN starts with this " + + "entity's FQN. Significantly faster than recursive delete for large hierarchies.", + responses = { + @ApiResponse(responseCode = "202", description = "Deletion accepted and running"), + @ApiResponse(responseCode = "404", description = "Entity for instance {id} is not found") + }) + public Response deletePrefixHardById( + @Context UriInfo uriInfo, + @Context SecurityContext securityContext, + @Parameter(description = "Id of the test definition", schema = @Schema(type = "UUID")) + @PathParam("id") + UUID id) { + return super.deletePrefixHardById(uriInfo, securityContext, id); + } + @DELETE @Path("/name/{name}") @Operation( diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/resources/dqtests/TestSuiteResource.java b/openmetadata-service/src/main/java/org/openmetadata/service/resources/dqtests/TestSuiteResource.java index fed7540d002f..4e9ae67964ce 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/resources/dqtests/TestSuiteResource.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/resources/dqtests/TestSuiteResource.java @@ -806,6 +806,27 @@ public Response deleteAsync( return repository.deleteLogicalTestSuiteAsync(securityContext, testSuite, hardDelete); } + @DELETE + @Path("/prefix/{id}") + @Operation( + operationId = "deleteTestSuitePrefixHard", + summary = "Hard-delete a test suite and all descendants by FQN prefix", + description = + "Bulk hard-delete this test suite and all descendants whose FQN starts with this " + + "entity's FQN. Significantly faster than recursive delete for large hierarchies.", + responses = { + @ApiResponse(responseCode = "202", description = "Deletion accepted and running"), + @ApiResponse(responseCode = "404", description = "Entity for instance {id} is not found") + }) + public Response deletePrefixHardById( + @Context UriInfo uriInfo, + @Context SecurityContext securityContext, + @Parameter(description = "Id of the test suite", schema = @Schema(type = "UUID")) + @PathParam("id") + UUID id) { + return super.deletePrefixHardById(uriInfo, securityContext, id); + } + @DELETE @Path("/name/{name}") @Operation( diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/resources/drives/DirectoryResource.java b/openmetadata-service/src/main/java/org/openmetadata/service/resources/drives/DirectoryResource.java index 016c1fdef4ab..e48c0da1a958 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/resources/drives/DirectoryResource.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/resources/drives/DirectoryResource.java @@ -425,6 +425,27 @@ public Response delete( return delete(uriInfo, securityContext, id, recursive, hardDelete); } + @DELETE + @Path("/prefix/{id}") + @Operation( + operationId = "deleteDirectoryPrefixHard", + summary = "Hard-delete a directory and all descendants by FQN prefix", + description = + "Bulk hard-delete this directory and all descendants whose FQN starts with this " + + "entity's FQN. Significantly faster than recursive delete for large hierarchies.", + responses = { + @ApiResponse(responseCode = "202", description = "Deletion accepted and running"), + @ApiResponse(responseCode = "404", description = "Entity for instance {id} is not found") + }) + public Response deletePrefixHardById( + @Context UriInfo uriInfo, + @Context SecurityContext securityContext, + @Parameter(description = "Id of the directory", schema = @Schema(type = "UUID")) + @PathParam("id") + UUID id) { + return super.deletePrefixHardById(uriInfo, securityContext, id); + } + @DELETE @Path("/name/{fqn}") @Operation( diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/resources/drives/FileResource.java b/openmetadata-service/src/main/java/org/openmetadata/service/resources/drives/FileResource.java index bcbbdd6088a9..7761ddd02bf3 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/resources/drives/FileResource.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/resources/drives/FileResource.java @@ -418,6 +418,26 @@ public Response delete( return delete(uriInfo, securityContext, id, false, hardDelete); } + @DELETE + @Path("/prefix/{id}") + @Operation( + operationId = "deleteFilePrefixHard", + summary = "Hard-delete a file and all descendants by FQN prefix", + description = + "Bulk hard-delete this file and all descendants whose FQN starts with this " + + "entity's FQN. Significantly faster than recursive delete for large hierarchies.", + responses = { + @ApiResponse(responseCode = "202", description = "Deletion accepted and running"), + @ApiResponse(responseCode = "404", description = "Entity for instance {id} is not found") + }) + public Response deletePrefixHardById( + @Context UriInfo uriInfo, + @Context SecurityContext securityContext, + @Parameter(description = "Id of the file", schema = @Schema(type = "UUID")) @PathParam("id") + UUID id) { + return super.deletePrefixHardById(uriInfo, securityContext, id); + } + @DELETE @Path("/name/{fqn}") @Operation( diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/resources/drives/SpreadsheetResource.java b/openmetadata-service/src/main/java/org/openmetadata/service/resources/drives/SpreadsheetResource.java index 28335091455c..8426dd2aecbb 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/resources/drives/SpreadsheetResource.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/resources/drives/SpreadsheetResource.java @@ -429,6 +429,27 @@ public Response delete( return delete(uriInfo, securityContext, id, recursive, hardDelete); } + @DELETE + @Path("/prefix/{id}") + @Operation( + operationId = "deleteSpreadsheetPrefixHard", + summary = "Hard-delete a spreadsheet and all descendants by FQN prefix", + description = + "Bulk hard-delete this spreadsheet and all descendants whose FQN starts with this " + + "entity's FQN. Significantly faster than recursive delete for large hierarchies.", + responses = { + @ApiResponse(responseCode = "202", description = "Deletion accepted and running"), + @ApiResponse(responseCode = "404", description = "Entity for instance {id} is not found") + }) + public Response deletePrefixHardById( + @Context UriInfo uriInfo, + @Context SecurityContext securityContext, + @Parameter(description = "Id of the spreadsheet", schema = @Schema(type = "UUID")) + @PathParam("id") + UUID id) { + return super.deletePrefixHardById(uriInfo, securityContext, id); + } + @DELETE @Path("/name/{fqn}") @Operation( diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/resources/drives/WorksheetResource.java b/openmetadata-service/src/main/java/org/openmetadata/service/resources/drives/WorksheetResource.java index c094d0bbb595..602f78617b41 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/resources/drives/WorksheetResource.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/resources/drives/WorksheetResource.java @@ -412,6 +412,27 @@ public Response delete( return delete(uriInfo, securityContext, id, false, hardDelete); } + @DELETE + @Path("/prefix/{id}") + @Operation( + operationId = "deleteWorksheetPrefixHard", + summary = "Hard-delete a worksheet and all descendants by FQN prefix", + description = + "Bulk hard-delete this worksheet and all descendants whose FQN starts with this " + + "entity's FQN. Significantly faster than recursive delete for large hierarchies.", + responses = { + @ApiResponse(responseCode = "202", description = "Deletion accepted and running"), + @ApiResponse(responseCode = "404", description = "Entity for instance {id} is not found") + }) + public Response deletePrefixHardById( + @Context UriInfo uriInfo, + @Context SecurityContext securityContext, + @Parameter(description = "Id of the worksheet", schema = @Schema(type = "UUID")) + @PathParam("id") + UUID id) { + return super.deletePrefixHardById(uriInfo, securityContext, id); + } + @DELETE @Path("/name/{fqn}") @Operation( diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/resources/events/NotificationTemplateResource.java b/openmetadata-service/src/main/java/org/openmetadata/service/resources/events/NotificationTemplateResource.java index 48886f978a64..d3d557dfcea3 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/resources/events/NotificationTemplateResource.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/resources/events/NotificationTemplateResource.java @@ -635,6 +635,27 @@ public Response delete( return super.delete(uriInfo, securityContext, id, recursive, hardDelete); } + @DELETE + @Path("/prefix/{id}") + @Operation( + operationId = "deleteNotificationTemplatePrefixHard", + summary = "Hard-delete a notification template and all descendants by FQN prefix", + description = + "Bulk hard-delete this notification template and all descendants whose FQN starts with this " + + "entity's FQN. Significantly faster than recursive delete for large hierarchies.", + responses = { + @ApiResponse(responseCode = "202", description = "Deletion accepted and running"), + @ApiResponse(responseCode = "404", description = "Entity for instance {id} is not found") + }) + public Response deletePrefixHardById( + @Context UriInfo uriInfo, + @Context SecurityContext securityContext, + @Parameter(description = "Id of the notification template", schema = @Schema(type = "UUID")) + @PathParam("id") + UUID id) { + return super.deletePrefixHardById(uriInfo, securityContext, id); + } + @DELETE @Path("/name/{fqn}") @Operation( diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/resources/events/subscription/EventSubscriptionResource.java b/openmetadata-service/src/main/java/org/openmetadata/service/resources/events/subscription/EventSubscriptionResource.java index 9833213eeacd..ce4339e4b3de 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/resources/events/subscription/EventSubscriptionResource.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/resources/events/subscription/EventSubscriptionResource.java @@ -561,6 +561,27 @@ public Response deleteEventSubscriptionAsync( return deleteByIdAsync(uriInfo, securityContext, id, true, true); } + @DELETE + @Path("/prefix/{id}") + @Operation( + operationId = "deleteEventSubscriptionPrefixHard", + summary = "Hard-delete a event subscription and all descendants by FQN prefix", + description = + "Bulk hard-delete this event subscription and all descendants whose FQN starts with this " + + "entity's FQN. Significantly faster than recursive delete for large hierarchies.", + responses = { + @ApiResponse(responseCode = "202", description = "Deletion accepted and running"), + @ApiResponse(responseCode = "404", description = "Entity for instance {id} is not found") + }) + public Response deletePrefixHardById( + @Context UriInfo uriInfo, + @Context SecurityContext securityContext, + @Parameter(description = "Id of the event subscription", schema = @Schema(type = "UUID")) + @PathParam("id") + UUID id) { + return super.deletePrefixHardById(uriInfo, securityContext, id); + } + @DELETE @Path("/name/{name}") @Operation( diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/resources/glossary/GlossaryResource.java b/openmetadata-service/src/main/java/org/openmetadata/service/resources/glossary/GlossaryResource.java index 57b8c03e391f..12be16ac4fb7 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/resources/glossary/GlossaryResource.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/resources/glossary/GlossaryResource.java @@ -474,6 +474,27 @@ public Response deleteByIdAsync( return deleteByIdAsync(uriInfo, securityContext, id, recursive, hardDelete); } + @DELETE + @Path("/prefix/{id}") + @Operation( + operationId = "deleteGlossaryPrefixHard", + summary = "Hard-delete a glossary and all descendants by FQN prefix", + description = + "Bulk hard-delete this glossary and all descendants whose FQN starts with this " + + "entity's FQN. Significantly faster than recursive delete for large hierarchies.", + responses = { + @ApiResponse(responseCode = "202", description = "Deletion accepted and running"), + @ApiResponse(responseCode = "404", description = "Entity for instance {id} is not found") + }) + public Response deletePrefixHardById( + @Context UriInfo uriInfo, + @Context SecurityContext securityContext, + @Parameter(description = "Id of the glossary", schema = @Schema(type = "UUID")) + @PathParam("id") + UUID id) { + return super.deletePrefixHardById(uriInfo, securityContext, id); + } + @DELETE @Path("/name/{name}") @Operation( diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/resources/glossary/GlossaryTermResource.java b/openmetadata-service/src/main/java/org/openmetadata/service/resources/glossary/GlossaryTermResource.java index e7e2eef75782..4997fcbc4cea 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/resources/glossary/GlossaryTermResource.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/resources/glossary/GlossaryTermResource.java @@ -1046,6 +1046,27 @@ public Response deleteByIdAsync( return deleteByIdAsync(uriInfo, securityContext, id, recursive, hardDelete); } + @DELETE + @Path("/prefix/{id}") + @Operation( + operationId = "deleteGlossaryTermPrefixHard", + summary = "Hard-delete a glossary term and all descendants by FQN prefix", + description = + "Bulk hard-delete this glossary term and all descendants whose FQN starts with this " + + "entity's FQN. Significantly faster than recursive delete for large hierarchies.", + responses = { + @ApiResponse(responseCode = "202", description = "Deletion accepted and running"), + @ApiResponse(responseCode = "404", description = "Entity for instance {id} is not found") + }) + public Response deletePrefixHardById( + @Context UriInfo uriInfo, + @Context SecurityContext securityContext, + @Parameter(description = "Id of the glossary term", schema = @Schema(type = "UUID")) + @PathParam("id") + UUID id) { + return super.deletePrefixHardById(uriInfo, securityContext, id); + } + @DELETE @Path("/name/{fqn}") @Operation( diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/resources/governance/WorkflowDefinitionResource.java b/openmetadata-service/src/main/java/org/openmetadata/service/resources/governance/WorkflowDefinitionResource.java index e52a68bd1c84..b32c384ce0a4 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/resources/governance/WorkflowDefinitionResource.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/resources/governance/WorkflowDefinitionResource.java @@ -484,6 +484,27 @@ public Response deleteByIdAsync( return deleteByIdAsync(uriInfo, securityContext, id, recursive, hardDelete); } + @DELETE + @Path("/prefix/{id}") + @Operation( + operationId = "deleteWorkflowDefinitionPrefixHard", + summary = "Hard-delete a workflow definition and all descendants by FQN prefix", + description = + "Bulk hard-delete this workflow definition and all descendants whose FQN starts with this " + + "entity's FQN. Significantly faster than recursive delete for large hierarchies.", + responses = { + @ApiResponse(responseCode = "202", description = "Deletion accepted and running"), + @ApiResponse(responseCode = "404", description = "Entity for instance {id} is not found") + }) + public Response deletePrefixHardById( + @Context UriInfo uriInfo, + @Context SecurityContext securityContext, + @Parameter(description = "Id of the workflow definition", schema = @Schema(type = "UUID")) + @PathParam("id") + UUID id) { + return super.deletePrefixHardById(uriInfo, securityContext, id); + } + @DELETE @Path("/name/{fqn}") @Operation( diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/resources/kpi/KpiResource.java b/openmetadata-service/src/main/java/org/openmetadata/service/resources/kpi/KpiResource.java index e8d345599fb6..e0d119dd244a 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/resources/kpi/KpiResource.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/resources/kpi/KpiResource.java @@ -379,6 +379,26 @@ public Response createOrUpdate( return createOrUpdate(uriInfo, securityContext, kpi); } + @DELETE + @Path("/prefix/{id}") + @Operation( + operationId = "deleteKpiPrefixHard", + summary = "Hard-delete a KPI and all descendants by FQN prefix", + description = + "Bulk hard-delete this KPI and all descendants whose FQN starts with this " + + "entity's FQN. Significantly faster than recursive delete for large hierarchies.", + responses = { + @ApiResponse(responseCode = "202", description = "Deletion accepted and running"), + @ApiResponse(responseCode = "404", description = "Entity for instance {id} is not found") + }) + public Response deletePrefixHardById( + @Context UriInfo uriInfo, + @Context SecurityContext securityContext, + @Parameter(description = "Id of the KPI", schema = @Schema(type = "UUID")) @PathParam("id") + UUID id) { + return super.deletePrefixHardById(uriInfo, securityContext, id); + } + @DELETE @Path("/name/{name}") @Operation( diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/resources/learning/LearningResourceResource.java b/openmetadata-service/src/main/java/org/openmetadata/service/resources/learning/LearningResourceResource.java index 3874474e37cd..41deb13beef2 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/resources/learning/LearningResourceResource.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/resources/learning/LearningResourceResource.java @@ -451,4 +451,25 @@ private LearningResource toEntity(CreateLearningResource create, String updatedB : LearningResource.Status.fromValue(create.getStatus().value())); return resource; } + + @DELETE + @Path("/prefix/{id}") + @Operation( + operationId = "deleteLearningResourcePrefixHard", + summary = "Hard-delete a learning resource and all descendants by FQN prefix", + description = + "Bulk hard-delete this learning resource and all descendants whose FQN starts with this " + + "entity's FQN. Significantly faster than recursive delete for large hierarchies.", + responses = { + @ApiResponse(responseCode = "202", description = "Deletion accepted and running"), + @ApiResponse(responseCode = "404", description = "Entity for instance {id} is not found") + }) + public Response deletePrefixHardById( + @Context UriInfo uriInfo, + @Context SecurityContext securityContext, + @Parameter(description = "Id of the learning resource", schema = @Schema(type = "UUID")) + @PathParam("id") + UUID id) { + return super.deletePrefixHardById(uriInfo, securityContext, id); + } } diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/resources/metrics/MetricResource.java b/openmetadata-service/src/main/java/org/openmetadata/service/resources/metrics/MetricResource.java index 9865a2218bd3..14a4c4253e85 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/resources/metrics/MetricResource.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/resources/metrics/MetricResource.java @@ -541,6 +541,26 @@ public Response deleteByIdAsync( return deleteByIdAsync(uriInfo, securityContext, id, false, hardDelete); } + @DELETE + @Path("/prefix/{id}") + @Operation( + operationId = "deleteMetricPrefixHard", + summary = "Hard-delete a metric and all descendants by FQN prefix", + description = + "Bulk hard-delete this metric and all descendants whose FQN starts with this " + + "entity's FQN. Significantly faster than recursive delete for large hierarchies.", + responses = { + @ApiResponse(responseCode = "202", description = "Deletion accepted and running"), + @ApiResponse(responseCode = "404", description = "Entity for instance {id} is not found") + }) + public Response deletePrefixHardById( + @Context UriInfo uriInfo, + @Context SecurityContext securityContext, + @Parameter(description = "Id of the metric", schema = @Schema(type = "UUID")) @PathParam("id") + UUID id) { + return super.deletePrefixHardById(uriInfo, securityContext, id); + } + @DELETE @Path("/name/{fqn}") @Operation( diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/resources/mlmodels/MlModelResource.java b/openmetadata-service/src/main/java/org/openmetadata/service/resources/mlmodels/MlModelResource.java index 785c2358cba3..2357ee44d35c 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/resources/mlmodels/MlModelResource.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/resources/mlmodels/MlModelResource.java @@ -584,6 +584,27 @@ public Response deleteByIdAsync( return deleteByIdAsync(uriInfo, securityContext, id, false, hardDelete); } + @DELETE + @Path("/prefix/{id}") + @Operation( + operationId = "deleteMlModelPrefixHard", + summary = "Hard-delete a ML model and all descendants by FQN prefix", + description = + "Bulk hard-delete this ML model and all descendants whose FQN starts with this " + + "entity's FQN. Significantly faster than recursive delete for large hierarchies.", + responses = { + @ApiResponse(responseCode = "202", description = "Deletion accepted and running"), + @ApiResponse(responseCode = "404", description = "Entity for instance {id} is not found") + }) + public Response deletePrefixHardById( + @Context UriInfo uriInfo, + @Context SecurityContext securityContext, + @Parameter(description = "Id of the ML model", schema = @Schema(type = "UUID")) + @PathParam("id") + UUID id) { + return super.deletePrefixHardById(uriInfo, securityContext, id); + } + @DELETE @Path("/name/{fqn}") @Operation( diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/resources/pipelines/PipelineResource.java b/openmetadata-service/src/main/java/org/openmetadata/service/resources/pipelines/PipelineResource.java index c307cdd2e5e3..fcff5edac6aa 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/resources/pipelines/PipelineResource.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/resources/pipelines/PipelineResource.java @@ -1218,6 +1218,27 @@ public Response deleteByIdAsync( return deleteByIdAsync(uriInfo, securityContext, id, recursive, hardDelete); } + @DELETE + @Path("/prefix/{id}") + @Operation( + operationId = "deletePipelinePrefixHard", + summary = "Hard-delete a pipeline and all descendants by FQN prefix", + description = + "Bulk hard-delete this pipeline and all descendants whose FQN starts with this " + + "entity's FQN. Significantly faster than recursive delete for large hierarchies.", + responses = { + @ApiResponse(responseCode = "202", description = "Deletion accepted and running"), + @ApiResponse(responseCode = "404", description = "Entity for instance {id} is not found") + }) + public Response deletePrefixHardById( + @Context UriInfo uriInfo, + @Context SecurityContext securityContext, + @Parameter(description = "Id of the pipeline", schema = @Schema(type = "UUID")) + @PathParam("id") + UUID id) { + return super.deletePrefixHardById(uriInfo, securityContext, id); + } + @DELETE @Path("/name/{fqn}") @Operation( diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/resources/policies/PolicyResource.java b/openmetadata-service/src/main/java/org/openmetadata/service/resources/policies/PolicyResource.java index 80c59d85b093..906ae17d3200 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/resources/policies/PolicyResource.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/resources/policies/PolicyResource.java @@ -506,6 +506,26 @@ public Response deleteByIdAsync( return deleteByIdAsync(uriInfo, securityContext, id, false, hardDelete); } + @DELETE + @Path("/prefix/{id}") + @Operation( + operationId = "deletePolicyPrefixHard", + summary = "Hard-delete a policy and all descendants by FQN prefix", + description = + "Bulk hard-delete this policy and all descendants whose FQN starts with this " + + "entity's FQN. Significantly faster than recursive delete for large hierarchies.", + responses = { + @ApiResponse(responseCode = "202", description = "Deletion accepted and running"), + @ApiResponse(responseCode = "404", description = "Entity for instance {id} is not found") + }) + public Response deletePrefixHardById( + @Context UriInfo uriInfo, + @Context SecurityContext securityContext, + @Parameter(description = "Id of the policy", schema = @Schema(type = "UUID")) @PathParam("id") + UUID id) { + return super.deletePrefixHardById(uriInfo, securityContext, id); + } + @DELETE @Path("/name/{fqn}") @Operation( diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/resources/query/QueryResource.java b/openmetadata-service/src/main/java/org/openmetadata/service/resources/query/QueryResource.java index 695855121df7..fd93f795eca3 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/resources/query/QueryResource.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/resources/query/QueryResource.java @@ -697,6 +697,26 @@ public Response deleteByIdAsync( return deleteByIdAsync(uriInfo, securityContext, id, false, true); } + @DELETE + @Path("/prefix/{id}") + @Operation( + operationId = "deleteQueryPrefixHard", + summary = "Hard-delete a query and all descendants by FQN prefix", + description = + "Bulk hard-delete this query and all descendants whose FQN starts with this " + + "entity's FQN. Significantly faster than recursive delete for large hierarchies.", + responses = { + @ApiResponse(responseCode = "202", description = "Deletion accepted and running"), + @ApiResponse(responseCode = "404", description = "Entity for instance {id} is not found") + }) + public Response deletePrefixHardById( + @Context UriInfo uriInfo, + @Context SecurityContext securityContext, + @Parameter(description = "Id of the query", schema = @Schema(type = "UUID")) @PathParam("id") + UUID id) { + return super.deletePrefixHardById(uriInfo, securityContext, id); + } + @DELETE @Path("/name/{fqn}") @Operation( diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/resources/reports/ReportResource.java b/openmetadata-service/src/main/java/org/openmetadata/service/resources/reports/ReportResource.java index 3f0fef5416bc..d73d315eadbc 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/resources/reports/ReportResource.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/resources/reports/ReportResource.java @@ -23,6 +23,7 @@ import io.swagger.v3.oas.annotations.tags.Tag; import jakarta.validation.Valid; import jakarta.ws.rs.Consumes; +import jakarta.ws.rs.DELETE; import jakarta.ws.rs.DefaultValue; import jakarta.ws.rs.GET; import jakarta.ws.rs.POST; @@ -227,4 +228,24 @@ private void addToReport(SecurityContext securityContext, Report report) { .withUpdatedBy(securityContext.getUserPrincipal().getName()) .withUpdatedAt(System.currentTimeMillis()); } + + @DELETE + @Path("/prefix/{id}") + @Operation( + operationId = "deleteReportPrefixHard", + summary = "Hard-delete a report and all descendants by FQN prefix", + description = + "Bulk hard-delete this report and all descendants whose FQN starts with this " + + "entity's FQN. Significantly faster than recursive delete for large hierarchies.", + responses = { + @ApiResponse(responseCode = "202", description = "Deletion accepted and running"), + @ApiResponse(responseCode = "404", description = "Entity for instance {id} is not found") + }) + public Response deletePrefixHardById( + @Context UriInfo uriInfo, + @Context SecurityContext securityContext, + @Parameter(description = "Id of the report", schema = @Schema(type = "UUID")) @PathParam("id") + UUID id) { + return super.deletePrefixHardById(uriInfo, securityContext, id); + } } diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/resources/searchindex/SearchIndexResource.java b/openmetadata-service/src/main/java/org/openmetadata/service/resources/searchindex/SearchIndexResource.java index 2570ba07642a..0cc38a0e29de 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/resources/searchindex/SearchIndexResource.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/resources/searchindex/SearchIndexResource.java @@ -663,6 +663,27 @@ public Response deleteByIdAsync( return deleteByIdAsync(uriInfo, securityContext, id, recursive, hardDelete); } + @DELETE + @Path("/prefix/{id}") + @Operation( + operationId = "deleteSearchIndexPrefixHard", + summary = "Hard-delete a search index and all descendants by FQN prefix", + description = + "Bulk hard-delete this search index and all descendants whose FQN starts with this " + + "entity's FQN. Significantly faster than recursive delete for large hierarchies.", + responses = { + @ApiResponse(responseCode = "202", description = "Deletion accepted and running"), + @ApiResponse(responseCode = "404", description = "Entity for instance {id} is not found") + }) + public Response deletePrefixHardById( + @Context UriInfo uriInfo, + @Context SecurityContext securityContext, + @Parameter(description = "Id of the search index", schema = @Schema(type = "UUID")) + @PathParam("id") + UUID id) { + return super.deletePrefixHardById(uriInfo, securityContext, id); + } + @DELETE @Path("/name/{fqn}") @Operation( diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/resources/services/apiservices/APIServiceResource.java b/openmetadata-service/src/main/java/org/openmetadata/service/resources/services/apiservices/APIServiceResource.java index b8555d192487..67c8313986ec 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/resources/services/apiservices/APIServiceResource.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/resources/services/apiservices/APIServiceResource.java @@ -522,6 +522,27 @@ public Response deleteByIdAsync( return deleteByIdAsync(uriInfo, securityContext, id, recursive, hardDelete); } + @DELETE + @Path("/prefix/{id}") + @Operation( + operationId = "deleteApiServicePrefixHard", + summary = "Hard-delete a API service and all descendants by FQN prefix", + description = + "Bulk hard-delete this API service and all descendants whose FQN starts with this " + + "entity's FQN. Significantly faster than recursive delete for large hierarchies.", + responses = { + @ApiResponse(responseCode = "202", description = "Deletion accepted and running"), + @ApiResponse(responseCode = "404", description = "Entity for instance {id} is not found") + }) + public Response deletePrefixHardById( + @Context UriInfo uriInfo, + @Context SecurityContext securityContext, + @Parameter(description = "Id of the API service", schema = @Schema(type = "UUID")) + @PathParam("id") + UUID id) { + return super.deletePrefixHardById(uriInfo, securityContext, id); + } + @DELETE @Path("/name/{fqn}") @Operation( diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/resources/services/connections/TestConnectionDefinitionResource.java b/openmetadata-service/src/main/java/org/openmetadata/service/resources/services/connections/TestConnectionDefinitionResource.java index b13a03e7a461..f1e348b3e45b 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/resources/services/connections/TestConnectionDefinitionResource.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/resources/services/connections/TestConnectionDefinitionResource.java @@ -12,6 +12,7 @@ import jakarta.validation.constraints.Max; import jakarta.validation.constraints.Min; import jakarta.ws.rs.Consumes; +import jakarta.ws.rs.DELETE; import jakarta.ws.rs.DefaultValue; import jakarta.ws.rs.GET; import jakarta.ws.rs.Path; @@ -20,6 +21,7 @@ import jakarta.ws.rs.QueryParam; import jakarta.ws.rs.core.Context; import jakarta.ws.rs.core.MediaType; +import jakarta.ws.rs.core.Response; import jakarta.ws.rs.core.SecurityContext; import jakarta.ws.rs.core.UriInfo; import java.io.IOException; @@ -217,4 +219,27 @@ public TestConnectionDefinition getByName( Include include) { return getByNameInternal(uriInfo, securityContext, name, fieldsParam, include); } + + @DELETE + @Path("/prefix/{id}") + @Operation( + operationId = "deleteTestConnectionDefinitionPrefixHard", + summary = "Hard-delete a test connection definition and all descendants by FQN prefix", + description = + "Bulk hard-delete this test connection definition and all descendants whose FQN starts with this " + + "entity's FQN. Significantly faster than recursive delete for large hierarchies.", + responses = { + @ApiResponse(responseCode = "202", description = "Deletion accepted and running"), + @ApiResponse(responseCode = "404", description = "Entity for instance {id} is not found") + }) + public Response deletePrefixHardById( + @Context UriInfo uriInfo, + @Context SecurityContext securityContext, + @Parameter( + description = "Id of the test connection definition", + schema = @Schema(type = "UUID")) + @PathParam("id") + UUID id) { + return super.deletePrefixHardById(uriInfo, securityContext, id); + } } diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/resources/services/dashboard/DashboardServiceResource.java b/openmetadata-service/src/main/java/org/openmetadata/service/resources/services/dashboard/DashboardServiceResource.java index 5ebcc81c1391..978b95ec70f8 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/resources/services/dashboard/DashboardServiceResource.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/resources/services/dashboard/DashboardServiceResource.java @@ -573,6 +573,27 @@ public Response deleteByIdAsync( return deleteByIdAsync(uriInfo, securityContext, id, recursive, hardDelete); } + @DELETE + @Path("/prefix/{id}") + @Operation( + operationId = "deleteDashboardServicePrefixHard", + summary = "Hard-delete a dashboard service and all descendants by FQN prefix", + description = + "Bulk hard-delete this dashboard service and all descendants whose FQN starts with this " + + "entity's FQN. Significantly faster than recursive delete for large hierarchies.", + responses = { + @ApiResponse(responseCode = "202", description = "Deletion accepted and running"), + @ApiResponse(responseCode = "404", description = "Entity for instance {id} is not found") + }) + public Response deletePrefixHardById( + @Context UriInfo uriInfo, + @Context SecurityContext securityContext, + @Parameter(description = "Id of the dashboard service", schema = @Schema(type = "UUID")) + @PathParam("id") + UUID id) { + return super.deletePrefixHardById(uriInfo, securityContext, id); + } + @DELETE @Path("/name/{name}") @Operation( diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/resources/services/database/DatabaseServiceResource.java b/openmetadata-service/src/main/java/org/openmetadata/service/resources/services/database/DatabaseServiceResource.java index 394a145c493f..3e9fae1315cb 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/resources/services/database/DatabaseServiceResource.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/resources/services/database/DatabaseServiceResource.java @@ -732,6 +732,30 @@ public Response deleteByIdAsync( return deleteByIdAsync(uriInfo, securityContext, id, recursive, hardDelete); } + @DELETE + @Path("/prefix/{id}") + @Operation( + operationId = "deleteDatabaseServicePrefixHard", + summary = "Hard-delete a database service and all descendants by FQN prefix", + description = + "Bulk hard-delete this database service and all descendants (databases, schemas, tables) " + + "whose FQN starts with this service's FQN. Significantly faster than recursive " + + "delete for large hierarchies.", + responses = { + @ApiResponse(responseCode = "202", description = "Deletion accepted and running"), + @ApiResponse( + responseCode = "404", + description = "DatabaseService for instance {id} is not found") + }) + public Response deletePrefixHardById( + @Context UriInfo uriInfo, + @Context SecurityContext securityContext, + @Parameter(description = "Id of the database service", schema = @Schema(type = "UUID")) + @PathParam("id") + UUID id) { + return super.deletePrefixHardById(uriInfo, securityContext, id); + } + @DELETE @Path("/name/{name}") @Operation( diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/resources/services/drive/DriveServiceResource.java b/openmetadata-service/src/main/java/org/openmetadata/service/resources/services/drive/DriveServiceResource.java index fa3b7c45231c..41b0557f5385 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/resources/services/drive/DriveServiceResource.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/resources/services/drive/DriveServiceResource.java @@ -650,6 +650,27 @@ public Response deleteByIdAsync( return deleteByIdAsync(uriInfo, securityContext, id, recursive, hardDelete); } + @DELETE + @Path("/prefix/{id}") + @Operation( + operationId = "deleteDriveServicePrefixHard", + summary = "Hard-delete a drive service and all descendants by FQN prefix", + description = + "Bulk hard-delete this drive service and all descendants whose FQN starts with this " + + "entity's FQN. Significantly faster than recursive delete for large hierarchies.", + responses = { + @ApiResponse(responseCode = "202", description = "Deletion accepted and running"), + @ApiResponse(responseCode = "404", description = "Entity for instance {id} is not found") + }) + public Response deletePrefixHardById( + @Context UriInfo uriInfo, + @Context SecurityContext securityContext, + @Parameter(description = "Id of the drive service", schema = @Schema(type = "UUID")) + @PathParam("id") + UUID id) { + return super.deletePrefixHardById(uriInfo, securityContext, id); + } + @DELETE @Path("/name/{name}") @Operation( diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/resources/services/ingestionpipelines/IngestionPipelineResource.java b/openmetadata-service/src/main/java/org/openmetadata/service/resources/services/ingestionpipelines/IngestionPipelineResource.java index 9216cdd4eb2b..10c92cf58936 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/resources/services/ingestionpipelines/IngestionPipelineResource.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/resources/services/ingestionpipelines/IngestionPipelineResource.java @@ -904,6 +904,27 @@ public Response deleteByIdAsync( return deleteByIdAsync(uriInfo, securityContext, id, false, hardDelete); } + @DELETE + @Path("/prefix/{id}") + @Operation( + operationId = "deleteIngestionPipelinePrefixHard", + summary = "Hard-delete a ingestion pipeline and all descendants by FQN prefix", + description = + "Bulk hard-delete this ingestion pipeline and all descendants whose FQN starts with this " + + "entity's FQN. Significantly faster than recursive delete for large hierarchies.", + responses = { + @ApiResponse(responseCode = "202", description = "Deletion accepted and running"), + @ApiResponse(responseCode = "404", description = "Entity for instance {id} is not found") + }) + public Response deletePrefixHardById( + @Context UriInfo uriInfo, + @Context SecurityContext securityContext, + @Parameter(description = "Id of the ingestion pipeline", schema = @Schema(type = "UUID")) + @PathParam("id") + UUID id) { + return super.deletePrefixHardById(uriInfo, securityContext, id); + } + @DELETE @Path("/name/{fqn}") @Operation( diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/resources/services/llm/LLMServiceResource.java b/openmetadata-service/src/main/java/org/openmetadata/service/resources/services/llm/LLMServiceResource.java index f2fe8332044e..4a9acd9b2760 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/resources/services/llm/LLMServiceResource.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/resources/services/llm/LLMServiceResource.java @@ -575,6 +575,27 @@ public Response deleteByIdAsync( return deleteByIdAsync(uriInfo, securityContext, id, recursive, hardDelete); } + @DELETE + @Path("/prefix/{id}") + @Operation( + operationId = "deleteLlmServicePrefixHard", + summary = "Hard-delete a LLM service and all descendants by FQN prefix", + description = + "Bulk hard-delete this LLM service and all descendants whose FQN starts with this " + + "entity's FQN. Significantly faster than recursive delete for large hierarchies.", + responses = { + @ApiResponse(responseCode = "202", description = "Deletion accepted and running"), + @ApiResponse(responseCode = "404", description = "Entity for instance {id} is not found") + }) + public Response deletePrefixHardById( + @Context UriInfo uriInfo, + @Context SecurityContext securityContext, + @Parameter(description = "Id of the LLM service", schema = @Schema(type = "UUID")) + @PathParam("id") + UUID id) { + return super.deletePrefixHardById(uriInfo, securityContext, id); + } + @DELETE @Path("/name/{name}") @Operation( diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/resources/services/mcp/McpServiceResource.java b/openmetadata-service/src/main/java/org/openmetadata/service/resources/services/mcp/McpServiceResource.java index 826b5505f385..3a12dc3ccf3d 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/resources/services/mcp/McpServiceResource.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/resources/services/mcp/McpServiceResource.java @@ -568,6 +568,27 @@ public Response deleteByIdAsync( return deleteByIdAsync(uriInfo, securityContext, id, recursive, hardDelete); } + @DELETE + @Path("/prefix/{id}") + @Operation( + operationId = "deleteMcpServicePrefixHard", + summary = "Hard-delete a MCP service and all descendants by FQN prefix", + description = + "Bulk hard-delete this MCP service and all descendants whose FQN starts with this " + + "entity's FQN. Significantly faster than recursive delete for large hierarchies.", + responses = { + @ApiResponse(responseCode = "202", description = "Deletion accepted and running"), + @ApiResponse(responseCode = "404", description = "Entity for instance {id} is not found") + }) + public Response deletePrefixHardById( + @Context UriInfo uriInfo, + @Context SecurityContext securityContext, + @Parameter(description = "Id of the MCP service", schema = @Schema(type = "UUID")) + @PathParam("id") + UUID id) { + return super.deletePrefixHardById(uriInfo, securityContext, id); + } + @DELETE @Path("/name/{name}") @Operation( diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/resources/services/messaging/MessagingServiceResource.java b/openmetadata-service/src/main/java/org/openmetadata/service/resources/services/messaging/MessagingServiceResource.java index 5ac5acc91193..b2ba489fe805 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/resources/services/messaging/MessagingServiceResource.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/resources/services/messaging/MessagingServiceResource.java @@ -576,6 +576,27 @@ public Response deleteByIdAsync( return deleteByIdAsync(uriInfo, securityContext, id, recursive, hardDelete); } + @DELETE + @Path("/prefix/{id}") + @Operation( + operationId = "deleteMessagingServicePrefixHard", + summary = "Hard-delete a messaging service and all descendants by FQN prefix", + description = + "Bulk hard-delete this messaging service and all descendants whose FQN starts with this " + + "entity's FQN. Significantly faster than recursive delete for large hierarchies.", + responses = { + @ApiResponse(responseCode = "202", description = "Deletion accepted and running"), + @ApiResponse(responseCode = "404", description = "Entity for instance {id} is not found") + }) + public Response deletePrefixHardById( + @Context UriInfo uriInfo, + @Context SecurityContext securityContext, + @Parameter(description = "Id of the messaging service", schema = @Schema(type = "UUID")) + @PathParam("id") + UUID id) { + return super.deletePrefixHardById(uriInfo, securityContext, id); + } + @DELETE @Path("/name/{name}") @Operation( diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/resources/services/metadata/MetadataServiceResource.java b/openmetadata-service/src/main/java/org/openmetadata/service/resources/services/metadata/MetadataServiceResource.java index 9cab5010d6d8..fc36dd07ccac 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/resources/services/metadata/MetadataServiceResource.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/resources/services/metadata/MetadataServiceResource.java @@ -616,6 +616,27 @@ public Response deleteByIdAsync( return deleteByIdAsync(uriInfo, securityContext, id, recursive, hardDelete); } + @DELETE + @Path("/prefix/{id}") + @Operation( + operationId = "deleteMetadataServicePrefixHard", + summary = "Hard-delete a metadata service and all descendants by FQN prefix", + description = + "Bulk hard-delete this metadata service and all descendants whose FQN starts with this " + + "entity's FQN. Significantly faster than recursive delete for large hierarchies.", + responses = { + @ApiResponse(responseCode = "202", description = "Deletion accepted and running"), + @ApiResponse(responseCode = "404", description = "Entity for instance {id} is not found") + }) + public Response deletePrefixHardById( + @Context UriInfo uriInfo, + @Context SecurityContext securityContext, + @Parameter(description = "Id of the metadata service", schema = @Schema(type = "UUID")) + @PathParam("id") + UUID id) { + return super.deletePrefixHardById(uriInfo, securityContext, id); + } + @DELETE @Path("/name/{name}") @Operation( diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/resources/services/mlmodel/MlModelServiceResource.java b/openmetadata-service/src/main/java/org/openmetadata/service/resources/services/mlmodel/MlModelServiceResource.java index 1568140a2bf4..1627cb02527d 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/resources/services/mlmodel/MlModelServiceResource.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/resources/services/mlmodel/MlModelServiceResource.java @@ -589,6 +589,27 @@ public Response deleteByIdAsync( return deleteByIdAsync(uriInfo, securityContext, id, recursive, hardDelete); } + @DELETE + @Path("/prefix/{id}") + @Operation( + operationId = "deleteMlModelServicePrefixHard", + summary = "Hard-delete a ML model service and all descendants by FQN prefix", + description = + "Bulk hard-delete this ML model service and all descendants whose FQN starts with this " + + "entity's FQN. Significantly faster than recursive delete for large hierarchies.", + responses = { + @ApiResponse(responseCode = "202", description = "Deletion accepted and running"), + @ApiResponse(responseCode = "404", description = "Entity for instance {id} is not found") + }) + public Response deletePrefixHardById( + @Context UriInfo uriInfo, + @Context SecurityContext securityContext, + @Parameter(description = "Id of the ML model service", schema = @Schema(type = "UUID")) + @PathParam("id") + UUID id) { + return super.deletePrefixHardById(uriInfo, securityContext, id); + } + @DELETE @Path("/name/{name}") @Operation( diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/resources/services/pipeline/PipelineServiceResource.java b/openmetadata-service/src/main/java/org/openmetadata/service/resources/services/pipeline/PipelineServiceResource.java index c8d79e38a3b5..cabdbc86994d 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/resources/services/pipeline/PipelineServiceResource.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/resources/services/pipeline/PipelineServiceResource.java @@ -589,6 +589,27 @@ public Response deleteByIdAsync( return deleteByIdAsync(uriInfo, securityContext, id, recursive, hardDelete); } + @DELETE + @Path("/prefix/{id}") + @Operation( + operationId = "deletePipelineServicePrefixHard", + summary = "Hard-delete a pipeline service and all descendants by FQN prefix", + description = + "Bulk hard-delete this pipeline service and all descendants whose FQN starts with this " + + "entity's FQN. Significantly faster than recursive delete for large hierarchies.", + responses = { + @ApiResponse(responseCode = "202", description = "Deletion accepted and running"), + @ApiResponse(responseCode = "404", description = "Entity for instance {id} is not found") + }) + public Response deletePrefixHardById( + @Context UriInfo uriInfo, + @Context SecurityContext securityContext, + @Parameter(description = "Id of the pipeline service", schema = @Schema(type = "UUID")) + @PathParam("id") + UUID id) { + return super.deletePrefixHardById(uriInfo, securityContext, id); + } + @DELETE @Path("/name/{fqn}") @Operation( diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/resources/services/searchIndexes/SearchServiceResource.java b/openmetadata-service/src/main/java/org/openmetadata/service/resources/services/searchIndexes/SearchServiceResource.java index 7d3b9adb9b6f..7339d8b8a081 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/resources/services/searchIndexes/SearchServiceResource.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/resources/services/searchIndexes/SearchServiceResource.java @@ -575,6 +575,27 @@ public Response deleteByIdAsync( return deleteByIdAsync(uriInfo, securityContext, id, recursive, hardDelete); } + @DELETE + @Path("/prefix/{id}") + @Operation( + operationId = "deleteSearchIndexServicePrefixHard", + summary = "Hard-delete a search index service and all descendants by FQN prefix", + description = + "Bulk hard-delete this search index service and all descendants whose FQN starts with this " + + "entity's FQN. Significantly faster than recursive delete for large hierarchies.", + responses = { + @ApiResponse(responseCode = "202", description = "Deletion accepted and running"), + @ApiResponse(responseCode = "404", description = "Entity for instance {id} is not found") + }) + public Response deletePrefixHardById( + @Context UriInfo uriInfo, + @Context SecurityContext securityContext, + @Parameter(description = "Id of the search index service", schema = @Schema(type = "UUID")) + @PathParam("id") + UUID id) { + return super.deletePrefixHardById(uriInfo, securityContext, id); + } + @DELETE @Path("/name/{fqn}") @Operation( diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/resources/services/security/SecurityServiceResource.java b/openmetadata-service/src/main/java/org/openmetadata/service/resources/services/security/SecurityServiceResource.java index 1b86dd974a49..5e8c73ac768f 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/resources/services/security/SecurityServiceResource.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/resources/services/security/SecurityServiceResource.java @@ -700,6 +700,27 @@ public Response deleteByIdAsync( return deleteByIdAsync(uriInfo, securityContext, id, recursive, hardDelete); } + @DELETE + @Path("/prefix/{id}") + @Operation( + operationId = "deleteSecurityServicePrefixHard", + summary = "Hard-delete a security service and all descendants by FQN prefix", + description = + "Bulk hard-delete this security service and all descendants whose FQN starts with this " + + "entity's FQN. Significantly faster than recursive delete for large hierarchies.", + responses = { + @ApiResponse(responseCode = "202", description = "Deletion accepted and running"), + @ApiResponse(responseCode = "404", description = "Entity for instance {id} is not found") + }) + public Response deletePrefixHardById( + @Context UriInfo uriInfo, + @Context SecurityContext securityContext, + @Parameter(description = "Id of the security service", schema = @Schema(type = "UUID")) + @PathParam("id") + UUID id) { + return super.deletePrefixHardById(uriInfo, securityContext, id); + } + @DELETE @Path("/name/{name}") @Operation( diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/resources/services/storage/StorageServiceResource.java b/openmetadata-service/src/main/java/org/openmetadata/service/resources/services/storage/StorageServiceResource.java index a9e24a8bdfc5..04ffdc6c1f53 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/resources/services/storage/StorageServiceResource.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/resources/services/storage/StorageServiceResource.java @@ -573,6 +573,27 @@ public Response deleteByIdAsync( return deleteByIdAsync(uriInfo, securityContext, id, recursive, hardDelete); } + @DELETE + @Path("/prefix/{id}") + @Operation( + operationId = "deleteStorageServicePrefixHard", + summary = "Hard-delete a storage service and all descendants by FQN prefix", + description = + "Bulk hard-delete this storage service and all descendants whose FQN starts with this " + + "entity's FQN. Significantly faster than recursive delete for large hierarchies.", + responses = { + @ApiResponse(responseCode = "202", description = "Deletion accepted and running"), + @ApiResponse(responseCode = "404", description = "Entity for instance {id} is not found") + }) + public Response deletePrefixHardById( + @Context UriInfo uriInfo, + @Context SecurityContext securityContext, + @Parameter(description = "Id of the storage service", schema = @Schema(type = "UUID")) + @PathParam("id") + UUID id) { + return super.deletePrefixHardById(uriInfo, securityContext, id); + } + @DELETE @Path("/name/{fqn}") @Operation( diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/resources/storages/ContainerResource.java b/openmetadata-service/src/main/java/org/openmetadata/service/resources/storages/ContainerResource.java index a2e65b5beb10..90b3469c3402 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/resources/storages/ContainerResource.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/resources/storages/ContainerResource.java @@ -584,6 +584,27 @@ public Response updateVote( .toResponse(); } + @DELETE + @Path("/prefix/{id}") + @Operation( + operationId = "deleteContainerPrefixHard", + summary = "Hard-delete a container and all descendants by FQN prefix", + description = + "Bulk hard-delete this container and all descendants whose FQN starts with this " + + "entity's FQN. Significantly faster than recursive delete for large hierarchies.", + responses = { + @ApiResponse(responseCode = "202", description = "Deletion accepted and running"), + @ApiResponse(responseCode = "404", description = "Entity for instance {id} is not found") + }) + public Response deletePrefixHardById( + @Context UriInfo uriInfo, + @Context SecurityContext securityContext, + @Parameter(description = "Id of the container", schema = @Schema(type = "UUID")) + @PathParam("id") + UUID id) { + return super.deletePrefixHardById(uriInfo, securityContext, id); + } + @DELETE @Path("/name/{fqn}") @Operation( diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/resources/tags/ClassificationResource.java b/openmetadata-service/src/main/java/org/openmetadata/service/resources/tags/ClassificationResource.java index 89ebc8ab3f30..c41f376c60b9 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/resources/tags/ClassificationResource.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/resources/tags/ClassificationResource.java @@ -437,6 +437,27 @@ public Response deleteByIdAsync( return deleteByIdAsync(uriInfo, securityContext, id, recursive, hardDelete); } + @DELETE + @Path("/prefix/{id}") + @Operation( + operationId = "deleteClassificationPrefixHard", + summary = "Hard-delete a classification and all descendants by FQN prefix", + description = + "Bulk hard-delete this classification and all descendants whose FQN starts with this " + + "entity's FQN. Significantly faster than recursive delete for large hierarchies.", + responses = { + @ApiResponse(responseCode = "202", description = "Deletion accepted and running"), + @ApiResponse(responseCode = "404", description = "Entity for instance {id} is not found") + }) + public Response deletePrefixHardById( + @Context UriInfo uriInfo, + @Context SecurityContext securityContext, + @Parameter(description = "Id of the classification", schema = @Schema(type = "UUID")) + @PathParam("id") + UUID id) { + return super.deletePrefixHardById(uriInfo, securityContext, id); + } + @DELETE @Path("/name/{name}") @Operation( diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/resources/tags/TagResource.java b/openmetadata-service/src/main/java/org/openmetadata/service/resources/tags/TagResource.java index 00f943464369..310ef39da624 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/resources/tags/TagResource.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/resources/tags/TagResource.java @@ -517,6 +517,26 @@ public Response deleteByIdAsync( return deleteByIdAsync(uriInfo, securityContext, id, recursive, hardDelete); } + @DELETE + @Path("/prefix/{id}") + @Operation( + operationId = "deleteTagPrefixHard", + summary = "Hard-delete a tag and all descendants by FQN prefix", + description = + "Bulk hard-delete this tag and all descendants whose FQN starts with this " + + "entity's FQN. Significantly faster than recursive delete for large hierarchies.", + responses = { + @ApiResponse(responseCode = "202", description = "Deletion accepted and running"), + @ApiResponse(responseCode = "404", description = "Entity for instance {id} is not found") + }) + public Response deletePrefixHardById( + @Context UriInfo uriInfo, + @Context SecurityContext securityContext, + @Parameter(description = "Id of the tag", schema = @Schema(type = "UUID")) @PathParam("id") + UUID id) { + return super.deletePrefixHardById(uriInfo, securityContext, id); + } + @DELETE @Path("/name/{fqn}") @Operation( diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/resources/teams/PersonaResource.java b/openmetadata-service/src/main/java/org/openmetadata/service/resources/teams/PersonaResource.java index 3c9b4cd8a2c6..059dfcb5e02d 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/resources/teams/PersonaResource.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/resources/teams/PersonaResource.java @@ -30,6 +30,8 @@ import jakarta.validation.constraints.Max; import jakarta.validation.constraints.Min; import jakarta.ws.rs.*; +import jakarta.ws.rs.DELETE; +import jakarta.ws.rs.PathParam; import jakarta.ws.rs.core.*; import java.util.List; import java.util.UUID; @@ -414,6 +416,27 @@ public Response deleteByIdAsync( return deleteByIdAsync(uriInfo, securityContext, id, false, true); } + @DELETE + @Path("/prefix/{id}") + @Operation( + operationId = "deletePersonaPrefixHard", + summary = "Hard-delete a persona and all descendants by FQN prefix", + description = + "Bulk hard-delete this persona and all descendants whose FQN starts with this " + + "entity's FQN. Significantly faster than recursive delete for large hierarchies.", + responses = { + @ApiResponse(responseCode = "202", description = "Deletion accepted and running"), + @ApiResponse(responseCode = "404", description = "Entity for instance {id} is not found") + }) + public Response deletePrefixHardById( + @Context UriInfo uriInfo, + @Context SecurityContext securityContext, + @Parameter(description = "Id of the persona", schema = @Schema(type = "UUID")) + @PathParam("id") + UUID id) { + return super.deletePrefixHardById(uriInfo, securityContext, id); + } + @DELETE @Path("/name/{name}") @Operation( diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/resources/teams/RoleResource.java b/openmetadata-service/src/main/java/org/openmetadata/service/resources/teams/RoleResource.java index 5f9dde00c015..a265019257dd 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/resources/teams/RoleResource.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/resources/teams/RoleResource.java @@ -475,6 +475,26 @@ public Response deleteByIdAsync( return deleteByIdAsync(uriInfo, securityContext, id, true, hardDelete); } + @DELETE + @Path("/prefix/{id}") + @Operation( + operationId = "deleteRolePrefixHard", + summary = "Hard-delete a role and all descendants by FQN prefix", + description = + "Bulk hard-delete this role and all descendants whose FQN starts with this " + + "entity's FQN. Significantly faster than recursive delete for large hierarchies.", + responses = { + @ApiResponse(responseCode = "202", description = "Deletion accepted and running"), + @ApiResponse(responseCode = "404", description = "Entity for instance {id} is not found") + }) + public Response deletePrefixHardById( + @Context UriInfo uriInfo, + @Context SecurityContext securityContext, + @Parameter(description = "Id of the role", schema = @Schema(type = "UUID")) @PathParam("id") + UUID id) { + return super.deletePrefixHardById(uriInfo, securityContext, id); + } + @DELETE @Path("/name/{name}") @Operation( diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/resources/teams/TeamResource.java b/openmetadata-service/src/main/java/org/openmetadata/service/resources/teams/TeamResource.java index c5143f25fa32..294bebaf1e61 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/resources/teams/TeamResource.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/resources/teams/TeamResource.java @@ -662,6 +662,26 @@ public Response deleteByIdAsync( return deleteByIdAsync(uriInfo, securityContext, id, recursive, hardDelete); } + @DELETE + @Path("/prefix/{id}") + @Operation( + operationId = "deleteTeamPrefixHard", + summary = "Hard-delete a team and all descendants by FQN prefix", + description = + "Bulk hard-delete this team and all descendants whose FQN starts with this " + + "entity's FQN. Significantly faster than recursive delete for large hierarchies.", + responses = { + @ApiResponse(responseCode = "202", description = "Deletion accepted and running"), + @ApiResponse(responseCode = "404", description = "Entity for instance {id} is not found") + }) + public Response deletePrefixHardById( + @Context UriInfo uriInfo, + @Context SecurityContext securityContext, + @Parameter(description = "Id of the team", schema = @Schema(type = "UUID")) @PathParam("id") + UUID id) { + return super.deletePrefixHardById(uriInfo, securityContext, id); + } + @DELETE @Path("/name/{name}") @Operation( diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/resources/teams/UserResource.java b/openmetadata-service/src/main/java/org/openmetadata/service/resources/teams/UserResource.java index c1610ad70230..30957c4e97fa 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/resources/teams/UserResource.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/resources/teams/UserResource.java @@ -1127,6 +1127,26 @@ public Response deleteByIdAsync( return deleteByIdAsync(uriInfo, securityContext, id, false, hardDelete); } + @DELETE + @Path("/prefix/{id}") + @Operation( + operationId = "deleteUserPrefixHard", + summary = "Hard-delete a user and all descendants by FQN prefix", + description = + "Bulk hard-delete this user and all descendants whose FQN starts with this " + + "entity's FQN. Significantly faster than recursive delete for large hierarchies.", + responses = { + @ApiResponse(responseCode = "202", description = "Deletion accepted and running"), + @ApiResponse(responseCode = "404", description = "Entity for instance {id} is not found") + }) + public Response deletePrefixHardById( + @Context UriInfo uriInfo, + @Context SecurityContext securityContext, + @Parameter(description = "Id of the user", schema = @Schema(type = "UUID")) @PathParam("id") + UUID id) { + return super.deletePrefixHardById(uriInfo, securityContext, id); + } + @DELETE @Path("/name/{name}") @Operation( diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/resources/topics/TopicResource.java b/openmetadata-service/src/main/java/org/openmetadata/service/resources/topics/TopicResource.java index 6b092a86f0c8..9b95788a7180 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/resources/topics/TopicResource.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/resources/topics/TopicResource.java @@ -631,6 +631,26 @@ public Response deleteByIdAsync( return deleteByIdAsync(uriInfo, securityContext, id, false, hardDelete); } + @DELETE + @Path("/prefix/{id}") + @Operation( + operationId = "deleteTopicPrefixHard", + summary = "Hard-delete a topic and all descendants by FQN prefix", + description = + "Bulk hard-delete this topic and all descendants whose FQN starts with this " + + "entity's FQN. Significantly faster than recursive delete for large hierarchies.", + responses = { + @ApiResponse(responseCode = "202", description = "Deletion accepted and running"), + @ApiResponse(responseCode = "404", description = "Entity for instance {id} is not found") + }) + public Response deletePrefixHardById( + @Context UriInfo uriInfo, + @Context SecurityContext securityContext, + @Parameter(description = "Id of the topic", schema = @Schema(type = "UUID")) @PathParam("id") + UUID id) { + return super.deletePrefixHardById(uriInfo, securityContext, id); + } + @DELETE @Path("/name/{fqn}") @Operation( diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/resources/types/TypeResource.java b/openmetadata-service/src/main/java/org/openmetadata/service/resources/types/TypeResource.java index 9546182a40b0..4059e284f1d9 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/resources/types/TypeResource.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/resources/types/TypeResource.java @@ -481,6 +481,26 @@ public Response deleteByIdAsync( return deleteByIdAsync(uriInfo, securityContext, id, false, true); } + @DELETE + @Path("/prefix/{id}") + @Operation( + operationId = "deleteTypePrefixHard", + summary = "Hard-delete a type and all descendants by FQN prefix", + description = + "Bulk hard-delete this type and all descendants whose FQN starts with this " + + "entity's FQN. Significantly faster than recursive delete for large hierarchies.", + responses = { + @ApiResponse(responseCode = "202", description = "Deletion accepted and running"), + @ApiResponse(responseCode = "404", description = "Entity for instance {id} is not found") + }) + public Response deletePrefixHardById( + @Context UriInfo uriInfo, + @Context SecurityContext securityContext, + @Parameter(description = "Id of the type", schema = @Schema(type = "UUID")) @PathParam("id") + UUID id) { + return super.deletePrefixHardById(uriInfo, securityContext, id); + } + @DELETE @Path("/name/{name}") @Operation( diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/search/SearchRepository.java b/openmetadata-service/src/main/java/org/openmetadata/service/search/SearchRepository.java index b480a07eaaa3..8c4b6693d5f0 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/search/SearchRepository.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/search/SearchRepository.java @@ -2178,6 +2178,28 @@ public void deleteEntityByFQNPrefix(EntityInterface entity) { } } + public void deleteByEntityTypeFqnPrefix(String entityType, String fqnPrefix) { + if (!checkIfIndexingIsSupported(entityType)) { + return; + } + if (!getSearchClient().isClientAvailable()) { + return; + } + IndexMapping indexMapping = entityIndexMap.get(entityType); + Timer.Sample searchSample = RequestLatencyContext.startSearchOperation(); + try { + searchClient.deleteEntityByFQNPrefix(indexMapping.getIndexName(clusterAlias), fqnPrefix); + } catch (Exception ie) { + LOG.error( + "Issue deleting search documents for entityType [{}] with FQN prefix [{}]", + entityType, + fqnPrefix, + ie); + } finally { + RequestLatencyContext.endSearchOperation(searchSample); + } + } + public void deleteTimeSeriesEntityById(EntityTimeSeriesInterface entity) { if (entity != null) { String entityId = entity.getId().toString(); diff --git a/openmetadata-service/src/test/java/org/openmetadata/service/migration/mysql/v1130/MigrationTest.java b/openmetadata-service/src/test/java/org/openmetadata/service/migration/mysql/v1130/MigrationTest.java new file mode 100644 index 000000000000..af0b0c89a89b --- /dev/null +++ b/openmetadata-service/src/test/java/org/openmetadata/service/migration/mysql/v1130/MigrationTest.java @@ -0,0 +1,55 @@ +package org.openmetadata.service.migration.mysql.v1130; + +import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; +import static org.mockito.Answers.RETURNS_DEEP_STUBS; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.mockStatic; + +import java.lang.reflect.Field; +import org.jdbi.v3.core.Handle; +import org.junit.jupiter.api.Test; +import org.mockito.MockedStatic; +import org.openmetadata.service.migration.api.MigrationProcessImpl; +import org.openmetadata.service.migration.utils.MigrationFile; +import org.openmetadata.service.migration.utils.v1130.MigrationUtil; + +class MigrationTest { + + private Migration createMigrationWithHandle(Handle handle) throws Exception { + MigrationFile migrationFile = mock(MigrationFile.class); + Migration migration = new Migration(migrationFile); + Field handleField = MigrationProcessImpl.class.getDeclaredField("handle"); + handleField.setAccessible(true); + handleField.set(migration, handle); + return migration; + } + + @Test + void runDataMigrationCallsBackfillRelationshipFqnHashes() throws Exception { + Handle handle = mock(Handle.class, RETURNS_DEEP_STUBS); + Migration migration = createMigrationWithHandle(handle); + + try (MockedStatic util = mockStatic(MigrationUtil.class)) { + util.when(MigrationUtil::updateOwnerChartFormulas).then(inv -> null); + util.when(() -> MigrationUtil.backfillRelationshipFqnHashes(handle)).then(inv -> null); + + migration.runDataMigration(); + + util.verify(() -> MigrationUtil.backfillRelationshipFqnHashes(handle)); + } + } + + @Test + void runDataMigrationDoesNotThrowWhenBackfillFails() throws Exception { + Handle handle = mock(Handle.class, RETURNS_DEEP_STUBS); + Migration migration = createMigrationWithHandle(handle); + + try (MockedStatic util = mockStatic(MigrationUtil.class)) { + util.when(MigrationUtil::updateOwnerChartFormulas).then(inv -> null); + util.when(() -> MigrationUtil.backfillRelationshipFqnHashes(handle)) + .thenThrow(new RuntimeException("DB error")); + + assertDoesNotThrow(migration::runDataMigration); + } + } +} diff --git a/openmetadata-service/src/test/java/org/openmetadata/service/migration/postgres/v1130/MigrationTest.java b/openmetadata-service/src/test/java/org/openmetadata/service/migration/postgres/v1130/MigrationTest.java new file mode 100644 index 000000000000..04207aec3361 --- /dev/null +++ b/openmetadata-service/src/test/java/org/openmetadata/service/migration/postgres/v1130/MigrationTest.java @@ -0,0 +1,55 @@ +package org.openmetadata.service.migration.postgres.v1130; + +import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; +import static org.mockito.Answers.RETURNS_DEEP_STUBS; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.mockStatic; + +import java.lang.reflect.Field; +import org.jdbi.v3.core.Handle; +import org.junit.jupiter.api.Test; +import org.mockito.MockedStatic; +import org.openmetadata.service.migration.api.MigrationProcessImpl; +import org.openmetadata.service.migration.utils.MigrationFile; +import org.openmetadata.service.migration.utils.v1130.MigrationUtil; + +class MigrationTest { + + private Migration createMigrationWithHandle(Handle handle) throws Exception { + MigrationFile migrationFile = mock(MigrationFile.class); + Migration migration = new Migration(migrationFile); + Field handleField = MigrationProcessImpl.class.getDeclaredField("handle"); + handleField.setAccessible(true); + handleField.set(migration, handle); + return migration; + } + + @Test + void runDataMigrationCallsBackfillRelationshipFqnHashes() throws Exception { + Handle handle = mock(Handle.class, RETURNS_DEEP_STUBS); + Migration migration = createMigrationWithHandle(handle); + + try (MockedStatic util = mockStatic(MigrationUtil.class)) { + util.when(MigrationUtil::updateOwnerChartFormulas).then(inv -> null); + util.when(() -> MigrationUtil.backfillRelationshipFqnHashes(handle)).then(inv -> null); + + migration.runDataMigration(); + + util.verify(() -> MigrationUtil.backfillRelationshipFqnHashes(handle)); + } + } + + @Test + void runDataMigrationDoesNotThrowWhenBackfillFails() throws Exception { + Handle handle = mock(Handle.class, RETURNS_DEEP_STUBS); + Migration migration = createMigrationWithHandle(handle); + + try (MockedStatic util = mockStatic(MigrationUtil.class)) { + util.when(MigrationUtil::updateOwnerChartFormulas).then(inv -> null); + util.when(() -> MigrationUtil.backfillRelationshipFqnHashes(handle)) + .thenThrow(new RuntimeException("DB error")); + + assertDoesNotThrow(migration::runDataMigration); + } + } +} diff --git a/openmetadata-service/src/test/java/org/openmetadata/service/migration/utils/v1130/MigrationUtilTest.java b/openmetadata-service/src/test/java/org/openmetadata/service/migration/utils/v1130/MigrationUtilTest.java new file mode 100644 index 000000000000..06aee6ae7281 --- /dev/null +++ b/openmetadata-service/src/test/java/org/openmetadata/service/migration/utils/v1130/MigrationUtilTest.java @@ -0,0 +1,150 @@ +package org.openmetadata.service.migration.utils.v1130; + +import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; +import static org.mockito.Answers.RETURNS_DEEP_STUBS; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.ArgumentMatchers.contains; +import static org.mockito.Mockito.doReturn; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.mockStatic; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +import java.util.Set; +import org.jdbi.v3.core.Handle; +import org.junit.jupiter.api.Test; +import org.mockito.MockedStatic; +import org.openmetadata.service.Entity; +import org.openmetadata.service.jdbi3.EntityDAO; +import org.openmetadata.service.jdbi3.EntityRepository; + +class MigrationUtilTest { + + private Handle handleWithUpdateReturning(int rowCount) { + Handle handle = mock(Handle.class, RETURNS_DEEP_STUBS); + when(handle.createUpdate(any()).bind(anyString(), anyString()).execute()).thenReturn(rowCount); + return handle; + } + + @SuppressWarnings({"rawtypes", "unchecked"}) + private EntityRepository repoWithHashCol(String tableName, String hashCol) { + EntityDAO dao = mock(EntityDAO.class); + when(dao.getTableName()).thenReturn(tableName); + when(dao.getNameHashColumn()).thenReturn(hashCol); + EntityRepository repo = mock(EntityRepository.class); + doReturn(dao).when(repo).getDao(); + return repo; + } + + @Test + void backfillDoesNothingWhenEntityListIsEmpty() { + Handle handle = mock(Handle.class, RETURNS_DEEP_STUBS); + + try (MockedStatic entity = mockStatic(Entity.class)) { + entity.when(Entity::getEntityList).thenReturn(Set.of()); + + assertDoesNotThrow(() -> MigrationUtil.backfillRelationshipFqnHashes(handle)); + verify(handle, never()).createUpdate(any()); + } + } + + @Test + void backfillIssuesFromAndToUpdatesForFqnHashEntity() { + Handle handle = mock(Handle.class, RETURNS_DEEP_STUBS); + EntityRepository repo = repoWithHashCol("table_entity", "fqnHash"); + + try (MockedStatic entity = mockStatic(Entity.class)) { + entity.when(Entity::getEntityList).thenReturn(Set.of("table")); + entity.when(() -> Entity.getEntityRepository("table")).thenReturn(repo); + + assertDoesNotThrow(() -> MigrationUtil.backfillRelationshipFqnHashes(handle)); + + verify(handle).createUpdate(contains("fromFQNHash")); + verify(handle).createUpdate(contains("toFQNHash")); + } + } + + @Test + void backfillIssuesFromAndToUpdatesForNameHashEntity() { + Handle handle = mock(Handle.class, RETURNS_DEEP_STUBS); + EntityRepository repo = repoWithHashCol("user_entity", "nameHash"); + + try (MockedStatic entity = mockStatic(Entity.class)) { + entity.when(Entity::getEntityList).thenReturn(Set.of("user")); + entity.when(() -> Entity.getEntityRepository("user")).thenReturn(repo); + + assertDoesNotThrow(() -> MigrationUtil.backfillRelationshipFqnHashes(handle)); + + verify(handle).createUpdate(contains("fromFQNHash")); + verify(handle).createUpdate(contains("toFQNHash")); + } + } + + @Test + void backfillSkipsEntityTypeWithUnrecognisedHashColumn() { + Handle handle = mock(Handle.class, RETURNS_DEEP_STUBS); + EntityRepository repo = repoWithHashCol("some_entity", "otherHash"); + + try (MockedStatic entity = mockStatic(Entity.class)) { + entity.when(Entity::getEntityList).thenReturn(Set.of("someEntity")); + entity.when(() -> Entity.getEntityRepository("someEntity")).thenReturn(repo); + + assertDoesNotThrow(() -> MigrationUtil.backfillRelationshipFqnHashes(handle)); + + verify(handle, never()).createUpdate(any()); + } + } + + @Test + void backfillSkipsTimeSeriesEntityWhenRepositoryNotFound() { + Handle handle = mock(Handle.class, RETURNS_DEEP_STUBS); + + try (MockedStatic entity = mockStatic(Entity.class)) { + entity.when(Entity::getEntityList).thenReturn(Set.of("testCaseResolutionStatus")); + entity + .when(() -> Entity.getEntityRepository("testCaseResolutionStatus")) + .thenThrow(new RuntimeException("not a regular entity")); + + assertDoesNotThrow(() -> MigrationUtil.backfillRelationshipFqnHashes(handle)); + + verify(handle, never()).createUpdate(any()); + } + } + + @Test + void backfillContinuesToNextEntityWhenOneEntityUpdateFails() { + Handle handle = mock(Handle.class, RETURNS_DEEP_STUBS); + when(handle.createUpdate(any()).bind(anyString(), anyString()).execute()) + .thenThrow(new RuntimeException("DB error")); + + EntityRepository tableRepo = repoWithHashCol("table_entity", "fqnHash"); + EntityRepository userRepo = repoWithHashCol("user_entity", "nameHash"); + + try (MockedStatic entity = mockStatic(Entity.class)) { + entity.when(Entity::getEntityList).thenReturn(Set.of("table", "user")); + entity.when(() -> Entity.getEntityRepository("table")).thenReturn(tableRepo); + entity.when(() -> Entity.getEntityRepository("user")).thenReturn(userRepo); + + assertDoesNotThrow(() -> MigrationUtil.backfillRelationshipFqnHashes(handle)); + } + } + + @Test + void backfillSqlContainsCorrelatedSubqueryWithCast() { + Handle handle = mock(Handle.class, RETURNS_DEEP_STUBS); + EntityRepository repo = repoWithHashCol("table_entity", "fqnHash"); + + try (MockedStatic entity = mockStatic(Entity.class)) { + entity.when(Entity::getEntityList).thenReturn(Set.of("table")); + entity.when(() -> Entity.getEntityRepository("table")).thenReturn(repo); + + MigrationUtil.backfillRelationshipFqnHashes(handle); + + verify(handle, times(2)).createUpdate(contains("CAST(t.id AS CHAR(36))")); + verify(handle, times(2)).createUpdate(contains("CAST(t.fqnHash AS CHAR(768))")); + } + } +}