diff --git a/openmetadata-integration-tests/src/test/java/org/openmetadata/it/tests/TestCaseSoftDeleteSearchIT.java b/openmetadata-integration-tests/src/test/java/org/openmetadata/it/tests/TestCaseSoftDeleteSearchIT.java new file mode 100644 index 000000000000..2674f2c2bc8c --- /dev/null +++ b/openmetadata-integration-tests/src/test/java/org/openmetadata/it/tests/TestCaseSoftDeleteSearchIT.java @@ -0,0 +1,231 @@ +/* + * Copyright 2026 Collate + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.openmetadata.it.tests; + +import static org.awaitility.Awaitility.await; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import java.time.Duration; +import java.util.List; +import java.util.Map; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.TestInstance; +import org.junit.jupiter.api.parallel.Execution; +import org.junit.jupiter.api.parallel.ExecutionMode; +import org.junit.jupiter.api.parallel.Isolated; +import org.openmetadata.it.bootstrap.SharedEntities; +import org.openmetadata.it.util.SdkClients; +import org.openmetadata.schema.api.data.CreateDatabase; +import org.openmetadata.schema.api.data.CreateDatabaseSchema; +import org.openmetadata.schema.api.data.CreateTable; +import org.openmetadata.schema.api.tests.CreateTestCase; +import org.openmetadata.schema.api.tests.CreateTestCaseResolutionStatus; +import org.openmetadata.schema.entity.data.Database; +import org.openmetadata.schema.entity.data.DatabaseSchema; +import org.openmetadata.schema.entity.data.Table; +import org.openmetadata.schema.tests.TestCase; +import org.openmetadata.schema.tests.type.Severity; +import org.openmetadata.schema.tests.type.TestCaseResolutionStatus; +import org.openmetadata.schema.tests.type.TestCaseResolutionStatusTypes; +import org.openmetadata.schema.type.Column; +import org.openmetadata.schema.type.ColumnDataType; +import org.openmetadata.sdk.client.OpenMetadataClient; +import org.openmetadata.sdk.models.ListParams; +import org.openmetadata.sdk.models.ListResponse; + +/** + * Regression for the live-indexing soft-delete propagation bug. {@code SOFT_DELETE_RESTORE_SCRIPT} + * was stamping a top-level {@code deleted} field onto child docs of every alias listed in the + * parent's {@code indexMapping}. For {@code testCase}, two of those children + * ({@code testCaseResolutionStatus}, {@code testCaseResult}) are time-series indexes whose + * Java schemas declare no {@code deleted} field. The poisoned doc broke Jackson on read and + * the Incident Manager UI surfaced an "Unrecognized field 'deleted'" toast. + * + *

This test exercises the end-to-end path: create a TC + result + incident, soft-delete the + * TC, and confirm that (a) the resolution-status listing API still parses cleanly and (b) the + * underlying ES doc carries no top-level {@code deleted} field. + */ +@Execution(ExecutionMode.SAME_THREAD) +@Isolated +@TestInstance(TestInstance.Lifecycle.PER_CLASS) +public class TestCaseSoftDeleteSearchIT { + + private static final ObjectMapper MAPPER = new ObjectMapper(); + + @Test + void softDeletingTestCaseDoesNotPollutePropagatedTimeSeriesDocs() throws Exception { + OpenMetadataClient client = SdkClients.adminClient(); + + long ts = System.currentTimeMillis(); + Database database = null; + DatabaseSchema schema = null; + Table table = null; + TestCase testCase = null; + try { + database = + client + .databases() + .create( + new CreateDatabase() + .withName("soft_delete_db_" + ts) + .withService(SharedEntities.get().MYSQL_SERVICE.getFullyQualifiedName())); + schema = + client + .databaseSchemas() + .create( + new CreateDatabaseSchema() + .withName("soft_delete_schema_" + ts) + .withDatabase(database.getFullyQualifiedName())); + table = + client + .tables() + .create( + new CreateTable() + .withName("soft_delete_table_" + ts) + .withDatabaseSchema(schema.getFullyQualifiedName()) + .withColumns( + List.of( + new Column().withName("id").withDataType(ColumnDataType.BIGINT)))); + + String testDefFqn = + client + .testDefinitions() + .list(new ListParams().withLimit(1)) + .getData() + .get(0) + .getFullyQualifiedName(); + + testCase = + client + .testCases() + .create( + new CreateTestCase() + .withName("soft_delete_tc_" + ts) + .withEntityLink( + "<#E::table::" + table.getFullyQualifiedName() + "::columns::id>") + .withTestDefinition(testDefFqn)); + + client + .testCaseResolutionStatuses() + .create( + new CreateTestCaseResolutionStatus() + .withTestCaseResolutionStatusType(TestCaseResolutionStatusTypes.New) + .withTestCaseReference(testCase.getFullyQualifiedName()) + .withSeverity(Severity.Severity2)); + + awaitIncidentIndexed(client, testCase.getFullyQualifiedName()); + + client + .testCases() + .delete(testCase.getId().toString(), Map.of("hardDelete", "false", "recursive", "true")); + + assertListingApiReturnsCleanlyAfterSoftDelete(client, testCase.getFullyQualifiedName()); + assertNoTopLevelDeletedFieldOnIncidentDoc(client, testCase.getFullyQualifiedName()); + } finally { + // Hard-delete the entire database tree so the test leaves no artefacts behind. The + // testCase + resolution statuses are recursively cascaded with the parent table. + // Best-effort cleanup — assertion failures take precedence over cleanup exceptions. + if (database != null) { + try { + client + .databases() + .delete( + database.getId().toString(), Map.of("hardDelete", "true", "recursive", "true")); + } catch (Exception ignored) { + // intentionally swallowed + } + } + } + } + + private void awaitIncidentIndexed(OpenMetadataClient client, String testCaseFqn) { + await("Wait for resolution status to be searchable") + .atMost(Duration.ofMinutes(2)) + .pollInterval(Duration.ofSeconds(2)) + .ignoreExceptions() + .untilAsserted( + () -> { + ListResponse resp = + client + .testCaseResolutionStatuses() + .searchList( + new ListParams() + .withLimit(1) + .withLatest(true) + .addFilter("testCaseFQN", testCaseFqn)); + assertNotNull(resp); + assertEquals( + 1, + resp.getData().size(), + "Incident for the test case should be indexed before we soft-delete"); + }); + } + + private void assertListingApiReturnsCleanlyAfterSoftDelete( + OpenMetadataClient client, String testCaseFqn) { + await("API returns parseable body after soft-delete propagation") + .atMost(Duration.ofMinutes(1)) + .pollInterval(Duration.ofSeconds(2)) + .ignoreExceptions() + .untilAsserted( + () -> { + ListResponse resp = + client + .testCaseResolutionStatuses() + .searchList( + new ListParams() + .withLimit(10) + .withLatest(true) + .addFilter("testCaseFQN", testCaseFqn)); + assertNotNull( + resp, "list endpoint must return a body; null implies a deserialization failure"); + }); + } + + /** + * The fix in {@link org.openmetadata.service.search.SearchRepository#softDeleteOrRestoredChildren} + * filters out time-series child aliases before invoking the soft-delete script. Confirm by + * querying ES directly for any TCRS doc that has a top-level {@code deleted} field — there + * must be none for our test case. + */ + private void assertNoTopLevelDeletedFieldOnIncidentDoc( + OpenMetadataClient client, String testCaseFqn) throws Exception { + String rawJson = + client + .search() + .query( + "testCaseReference.fullyQualifiedName.keyword:\"" + + testCaseFqn + + "\" AND _exists_:deleted") + .index("test_case_resolution_status_search_index") + .size(5) + .execute(); + JsonNode root = MAPPER.readTree(rawJson); + JsonNode hits = root.path("hits").path("hits"); + assertTrue( + hits.isArray(), () -> "ES response missing hits.hits array; raw response was: " + rawJson); + assertFalse( + hits.elements().hasNext(), + () -> + "No `deleted` field should exist on testCaseResolutionStatus docs after a parent" + + " soft-delete; found " + + hits.size() + + " polluted docs. Raw response: " + + rawJson); + } +} diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/Entity.java b/openmetadata-service/src/main/java/org/openmetadata/service/Entity.java index fbdc582784a7..fbfdaae9202e 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/Entity.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/Entity.java @@ -79,6 +79,8 @@ import org.openmetadata.service.jobs.JobDAO; import org.openmetadata.service.resources.feeds.MessageParser.EntityLink; import org.openmetadata.service.search.SearchRepository; +import org.openmetadata.service.search.capability.EntityIndexCapability; +import org.openmetadata.service.search.capability.EntityIndexCapabilityRegistry; import org.openmetadata.service.search.indexes.SearchIndex; import org.openmetadata.service.util.EntityUtil.Fields; import org.openmetadata.service.util.FullyQualifiedName; @@ -406,10 +408,19 @@ public static void initializeRepositories(OpenMetadataApplicationConfig config, } } registerDomainSyncHandler(); + validateIndexMappingsAgainstCapabilities(); initializedRepositories = true; } } + private static void validateIndexMappingsAgainstCapabilities() { + if (searchRepository == null || searchRepository.getEntityIndexMap() == null) { + return; + } + org.openmetadata.service.search.validation.IndexMappingValidator.validate( + searchRepository.getEntityIndexMap()); + } + private static void registerDomainSyncHandler() { try { DomainSyncHandler domainSyncHandler = new DomainSyncHandler(); @@ -427,6 +438,7 @@ public static void cleanup() { searchRepository = null; entityRelationshipRepository = null; ENTITY_REPOSITORY_MAP.clear(); + EntityIndexCapabilityRegistry.clear(); } public static void registerEntity( @@ -435,6 +447,7 @@ public static void registerEntity( EntityInterface.CANONICAL_ENTITY_NAME_MAP.put(entity.toLowerCase(Locale.ROOT), entity); EntityInterface.ENTITY_TYPE_TO_CLASS_MAP.put(entity.toLowerCase(Locale.ROOT), clazz); ENTITY_LIST.add(entity); + EntityIndexCapabilityRegistry.register(EntityIndexCapability.forEntity(entity)); LOG.debug("Registering entity {} {}", clazz, entity); } @@ -446,6 +459,7 @@ public static void registerEntity( entity.toLowerCase(Locale.ROOT), entity); EntityTimeSeriesInterface.ENTITY_TYPE_TO_CLASS_MAP.put(entity.toLowerCase(Locale.ROOT), clazz); ENTITY_LIST.add(entity); + EntityIndexCapabilityRegistry.register(EntityIndexCapability.forTimeSeries(entity)); LOG.debug("Registering entity time series {} {}", clazz, entity); } @@ -720,6 +734,21 @@ public static boolean hasEntityRepository(@NonNull String entityType) { || ENTITY_TS_REPOSITORY_MAP.containsKey(entityType); } + /** + * Returns true when {@code entityTypeOrAlias} maps to an {@link EntityTimeSeriesInterface} + * (append-only, no top-level {@code deleted} field). Backed by + * {@link EntityIndexCapabilityRegistry}; the legacy {@code ENTITY_TS_REPOSITORY_MAP} fallback + * keeps the helper usable in tests that register repositories directly without going through + * the standard capability registration path. + */ + public static boolean isTimeSeriesEntity(@NonNull String entityTypeOrAlias) { + EntityIndexCapability capability = EntityIndexCapabilityRegistry.get(entityTypeOrAlias); + if (capability != null) { + return capability.isTimeSeries(); + } + return ENTITY_TS_REPOSITORY_MAP.containsKey(entityTypeOrAlias); + } + public static EntityTimeSeriesRepository getEntityTimeSeriesRepository(@NonNull String entityType) { EntityTimeSeriesRepository entityTimeSeriesRepository = diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/apps/bundles/searchIndex/DistributedIndexingStrategy.java b/openmetadata-service/src/main/java/org/openmetadata/service/apps/bundles/searchIndex/DistributedIndexingStrategy.java index 83c3d4b8ee04..e1cb349abbed 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/apps/bundles/searchIndex/DistributedIndexingStrategy.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/apps/bundles/searchIndex/DistributedIndexingStrategy.java @@ -22,6 +22,7 @@ import org.openmetadata.service.apps.bundles.searchIndex.distributed.DistributedSearchIndexExecutor; import org.openmetadata.service.apps.bundles.searchIndex.distributed.IndexJobStatus; import org.openmetadata.service.apps.bundles.searchIndex.distributed.SearchIndexJob; +import org.openmetadata.service.apps.bundles.searchIndex.promotion.RatioPromotionPolicy; import org.openmetadata.service.jdbi3.CollectionDAO; import org.openmetadata.service.jdbi3.EntityTimeSeriesRepository; import org.openmetadata.service.jdbi3.ListFilter; @@ -302,7 +303,10 @@ private boolean finalizeAllEntityReindex( return finalSuccess; } - return new DistributedReindexFinalizer(indexPromotionHandler, stagedIndexContext) + double minRatio = + config != null ? config.minSuccessRatio() : RatioPromotionPolicy.DEFAULT_MIN_SUCCESS_RATIO; + return new DistributedReindexFinalizer( + indexPromotionHandler, stagedIndexContext, new RatioPromotionPolicy(minRatio)) .finalizeRemainingEntities(getPromotedEntities(), getFinalEntityStats(), finalSuccess); } diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/apps/bundles/searchIndex/DistributedReindexFinalizer.java b/openmetadata-service/src/main/java/org/openmetadata/service/apps/bundles/searchIndex/DistributedReindexFinalizer.java index d4aaadd06527..a8c8e19a929e 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/apps/bundles/searchIndex/DistributedReindexFinalizer.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/apps/bundles/searchIndex/DistributedReindexFinalizer.java @@ -19,6 +19,8 @@ import lombok.extern.slf4j.Slf4j; import org.openmetadata.service.Entity; import org.openmetadata.service.apps.bundles.searchIndex.distributed.SearchIndexJob; +import org.openmetadata.service.apps.bundles.searchIndex.promotion.EntityPromotionContext; +import org.openmetadata.service.apps.bundles.searchIndex.promotion.PromotionPolicy; import org.openmetadata.service.search.RecreateIndexHandler; import org.openmetadata.service.search.ReindexContext; @@ -26,11 +28,15 @@ class DistributedReindexFinalizer { private final RecreateIndexHandler indexPromotionHandler; private final ReindexContext stagedIndexContext; + private final PromotionPolicy promotionPolicy; DistributedReindexFinalizer( - RecreateIndexHandler indexPromotionHandler, ReindexContext stagedIndexContext) { + RecreateIndexHandler indexPromotionHandler, + ReindexContext stagedIndexContext, + PromotionPolicy promotionPolicy) { this.indexPromotionHandler = indexPromotionHandler; this.stagedIndexContext = stagedIndexContext; + this.promotionPolicy = promotionPolicy; } boolean finalizeRemainingEntities( @@ -132,8 +138,23 @@ private boolean computeEntitySuccess( if (stats == null) { return false; } - return stats.getFailedRecords() == 0 - && stats.getSuccessRecords() + stats.getFailedRecords() >= stats.getTotalRecords(); + EntityPromotionContext promotionContext = + new EntityPromotionContext( + entityType, + stats.getTotalRecords(), + stats.getSuccessRecords(), + stats.getFailedRecords(), + stats.getProcessedRecords()); + PromotionPolicy.Decision decision = promotionPolicy.evaluate(promotionContext); + LOG.debug( + "Promotion decision for entity '{}': fullySuccessful={} reason={} (stats: total={}, success={}, failed={})", + entityType, + decision.fullySuccessful(), + decision.reason(), + stats.getTotalRecords(), + stats.getSuccessRecords(), + stats.getFailedRecords()); + return decision.fullySuccessful(); } private void finalizeEntityReindex(String entityType, boolean success) { diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/apps/bundles/searchIndex/ReindexingConfiguration.java b/openmetadata-service/src/main/java/org/openmetadata/service/apps/bundles/searchIndex/ReindexingConfiguration.java index 7d7a68357156..d3596c7a878f 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/apps/bundles/searchIndex/ReindexingConfiguration.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/apps/bundles/searchIndex/ReindexingConfiguration.java @@ -5,6 +5,7 @@ import java.util.Set; import org.openmetadata.schema.system.EventPublisherJob; import org.openmetadata.schema.type.IndexMappingLanguage; +import org.openmetadata.service.apps.bundles.searchIndex.promotion.RatioPromotionPolicy; import org.openmetadata.service.search.SearchClusterMetrics; import org.openmetadata.service.search.SearchRepository; import org.slf4j.Logger; @@ -35,7 +36,8 @@ public record ReindexingConfiguration( String slackBotToken, String slackChannel, int timeSeriesMaxDays, - Map timeSeriesEntityDays) { + Map timeSeriesEntityDays, + double minSuccessRatio) { private static final Logger LOG = LoggerFactory.getLogger(ReindexingConfiguration.class); @@ -53,6 +55,8 @@ public record ReindexingConfiguration( private static final int DEFAULT_INITIAL_BACKOFF = 1000; private static final int DEFAULT_MAX_BACKOFF = 10000; private static final int DEFAULT_TIME_SERIES_MAX_DAYS = 0; + private static final double DEFAULT_MIN_SUCCESS_RATIO = + RatioPromotionPolicy.DEFAULT_MIN_SUCCESS_RATIO; public static ReindexingConfiguration applyAutoTuning( ReindexingConfiguration config, SearchRepository searchRepository, long totalEntities) { @@ -86,6 +90,7 @@ public static ReindexingConfiguration applyAutoTuning( .slackChannel(config.slackChannel()) .timeSeriesMaxDays(config.timeSeriesMaxDays()) .timeSeriesEntityDays(config.timeSeriesEntityDays()) + .minSuccessRatio(config.minSuccessRatio()) .build(); } @@ -138,7 +143,10 @@ public static ReindexingConfiguration from(EventPublisherJob jobData) { : DEFAULT_TIME_SERIES_MAX_DAYS, jobData.getTimeSeriesEntityDays() != null ? jobData.getTimeSeriesEntityDays() - : Collections.emptyMap()); + : Collections.emptyMap(), + jobData.getMinSuccessRatio() != null + ? jobData.getMinSuccessRatio() + : DEFAULT_MIN_SUCCESS_RATIO); } /** @@ -213,6 +221,7 @@ public static class Builder { private String slackChannel; private int timeSeriesMaxDays = DEFAULT_TIME_SERIES_MAX_DAYS; private Map timeSeriesEntityDays = Collections.emptyMap(); + private double minSuccessRatio = DEFAULT_MIN_SUCCESS_RATIO; public Builder entities(Set entities) { this.entities = entities; @@ -319,6 +328,11 @@ public Builder timeSeriesEntityDays(Map timeSeriesEntityDays) { return this; } + public Builder minSuccessRatio(double minSuccessRatio) { + this.minSuccessRatio = minSuccessRatio; + return this; + } + public ReindexingConfiguration build() { return new ReindexingConfiguration( entities, @@ -341,7 +355,8 @@ public ReindexingConfiguration build() { slackBotToken, slackChannel, timeSeriesMaxDays, - timeSeriesEntityDays); + timeSeriesEntityDays, + minSuccessRatio); } } } diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/apps/bundles/searchIndex/promotion/EntityPromotionContext.java b/openmetadata-service/src/main/java/org/openmetadata/service/apps/bundles/searchIndex/promotion/EntityPromotionContext.java new file mode 100644 index 000000000000..6e49684606d8 --- /dev/null +++ b/openmetadata-service/src/main/java/org/openmetadata/service/apps/bundles/searchIndex/promotion/EntityPromotionContext.java @@ -0,0 +1,51 @@ +/* + * Copyright 2026 Collate. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.openmetadata.service.apps.bundles.searchIndex.promotion; + +/** + * Per-entity record-level counts used to decide whether to promote a staged index. Built from + * {@code SearchIndexJob.EntityTypeStats} at finalize time. Kept deliberately small — the policy + * should not need to reach back into the executor for additional signals. + */ +public record EntityPromotionContext( + String entityType, + long totalRecords, + long successRecords, + long failedRecords, + long processedRecords) { + + /** + * Fraction of records that landed in the staged index. Defaults to {@code 1.0} when nothing + * was scheduled (empty entity types are not failures). + */ + public double successRatio() { + if (totalRecords <= 0) { + return 1.0; + } + return (double) successRecords / totalRecords; + } + + /** + * Returns true if every scheduled record was accounted for (either succeeded or failed). A + * job that stopped early — e.g. operator stop, partition reclaimer, host crash — leaves + * {@code processedRecords < totalRecords} and must NOT be flagged fully successful even if + * the success ratio over the processed subset clears the threshold. + */ + public boolean allRecordsAccountedFor() { + if (totalRecords <= 0) { + return true; + } + long accounted = Math.max(processedRecords, successRecords + failedRecords); + return accounted >= totalRecords; + } +} diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/apps/bundles/searchIndex/promotion/PromotionPolicy.java b/openmetadata-service/src/main/java/org/openmetadata/service/apps/bundles/searchIndex/promotion/PromotionPolicy.java new file mode 100644 index 000000000000..4f7aba423c5d --- /dev/null +++ b/openmetadata-service/src/main/java/org/openmetadata/service/apps/bundles/searchIndex/promotion/PromotionPolicy.java @@ -0,0 +1,41 @@ +/* + * Copyright 2026 Collate. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.openmetadata.service.apps.bundles.searchIndex.promotion; + +/** + * Decides whether a per-entity reindex was "fully successful". The default implementation, + * {@link RatioPromotionPolicy}, declares success when the per-record success ratio clears a + * configurable threshold. Promotion itself remains unconditional — {@code DefaultRecreateHandler} + * always promotes a non-empty staged index via the existing doc-count rescue when this flag is + * false. The flag drives the operator-visible "did this entity run cleanly?" signal. + * + *

Prior to this abstraction the strict rule was binary ("zero failures") and the rescue lived + * unannounced inside the handler. Centralizing the success threshold here makes it tunable and + * makes the rescue's existence explicit in the contract. + */ +public interface PromotionPolicy { + + Decision evaluate(EntityPromotionContext context); + + /** + * Outcome of {@link #evaluate(EntityPromotionContext)}. + * + * @param fullySuccessful true if the entity reindex met the policy's strict success bar; + * false if the rescue path (doc-count fallback in + * {@code DefaultRecreateHandler.promoteEntityIndex}) must decide whether the staged index + * is salvageable. Promotion is always attempted regardless of this flag — the flag + * controls how the run is logged / reported. + * @param reason human-readable rationale for the audit log + */ + record Decision(boolean fullySuccessful, String reason) {} +} diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/apps/bundles/searchIndex/promotion/RatioPromotionPolicy.java b/openmetadata-service/src/main/java/org/openmetadata/service/apps/bundles/searchIndex/promotion/RatioPromotionPolicy.java new file mode 100644 index 000000000000..138475cf90c3 --- /dev/null +++ b/openmetadata-service/src/main/java/org/openmetadata/service/apps/bundles/searchIndex/promotion/RatioPromotionPolicy.java @@ -0,0 +1,75 @@ +/* + * Copyright 2026 Collate. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.openmetadata.service.apps.bundles.searchIndex.promotion; + +/** + * Per-entity reindex is "fully successful" when the per-record success ratio meets + * {@code minSuccessRatio}. The policy's {@link Decision#fullySuccessful()} captures that + * strict outcome; the caller hands it to {@code DefaultRecreateHandler.finalizeReindex} which + * already has a doc-count rescue path for cases where strict success was missed but the staged + * index still has data. Promotion is therefore unconditional at the policy level (the rescue + * decides whether to keep or drop the staged index), and the success flag carries the operator- + * visible "did this entity run cleanly" signal that {@code minSuccessRatio} controls. + * + *

The previous binary rule ({@code failedRecords == 0}) blocked the success signal on a + * single failed record. The ratio gives operators a tunable strict bar; below it, the + * downstream rescue still salvages a non-empty staged index. + */ +public class RatioPromotionPolicy implements PromotionPolicy { + + public static final double DEFAULT_MIN_SUCCESS_RATIO = 0.95d; + + private final double minSuccessRatio; + + public RatioPromotionPolicy(double minSuccessRatio) { + if (minSuccessRatio < 0.0d || minSuccessRatio > 1.0d) { + throw new IllegalArgumentException( + "minSuccessRatio must be in [0.0, 1.0]; got " + minSuccessRatio); + } + this.minSuccessRatio = minSuccessRatio; + } + + public static RatioPromotionPolicy withDefaultThreshold() { + return new RatioPromotionPolicy(DEFAULT_MIN_SUCCESS_RATIO); + } + + public double minSuccessRatio() { + return minSuccessRatio; + } + + @Override + public Decision evaluate(EntityPromotionContext context) { + if (context.totalRecords() <= 0L) { + return new Decision(true, "no records scheduled; nothing to evaluate"); + } + if (!context.allRecordsAccountedFor()) { + return new Decision( + false, + "incomplete run: only %d of %d records processed; not fully successful" + .formatted( + Math.max( + context.processedRecords(), + context.successRecords() + context.failedRecords()), + context.totalRecords())); + } + double ratio = context.successRatio(); + if (ratio >= minSuccessRatio) { + return new Decision( + true, "successRatio %.4f >= minSuccessRatio %.4f".formatted(ratio, minSuccessRatio)); + } + return new Decision( + false, + "successRatio %.4f below threshold %.4f; DefaultRecreateHandler will rescue via doc-count" + .formatted(ratio, minSuccessRatio)); + } +} diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/EntityTimeSeriesRepository.java b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/EntityTimeSeriesRepository.java index 57caa9870c07..99a0536644e6 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/EntityTimeSeriesRepository.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/EntityTimeSeriesRepository.java @@ -463,7 +463,7 @@ public ResultList listFromSearchWithOffset( searchListFilter, limit, offset, entityType, searchSortFilter, q, queryString); total = results.getTotal(); for (Map json : results.getResults()) { - T entity = setFieldsInternal(JsonUtils.readOrConvertValue(json, entityClass), fields); + T entity = setFieldsInternal(readTimeSeriesSource(json), fields); try { setInheritedFields(entity); } catch (RuntimeException e) { @@ -524,9 +524,7 @@ public ResultList listLatestFromSearch( hitList -> { for (Map hit : (List>) hitList) { Map source = extractAndFilterSource(hit); - T entity = - setFieldsInternal( - JsonUtils.readOrConvertValue(source, entityClass), fields); + T entity = setFieldsInternal(readTimeSeriesSource(source), fields); if (entity != null) { try { setInheritedFields(entity); @@ -684,7 +682,7 @@ public T latestFromSearch(EntityUtil.Fields fields, SearchListFilter searchListF SearchResultListMapper results = searchRepository.listWithOffset(searchListFilter, 1, 0, entityType, searchSortFilter, q); for (Map json : results.getResults()) { - T entity = setFieldsInternal(JsonUtils.readOrConvertValue(json, entityClass), fields); + T entity = setFieldsInternal(readTimeSeriesSource(json), fields); setInheritedFields(entity); clearFieldsInternal(entity, fields); return entity; @@ -696,6 +694,24 @@ protected void setIncludeSearchFields(SearchListFilter searchListFilter) { // Nothing to do in the default implementation } + /** + * Deserializes a search hit source into the time-series entity type. Strict by default so any + * genuine schema drift fails loudly. Targeted scrub of the legacy {@code deleted} field — the + * one known-pollution field stamped onto time-series docs by the soft-delete script before + * the Phase 1 fix — keeps that specific case from breaking reads, without the blanket + * unknown-field tolerance that {@link JsonUtils#readOrConvertValueLenient} would impose. + * Once a recreate-style reindex has cleaned the index, the scrub is a no-op. + */ + @SuppressWarnings("unchecked") + private T readTimeSeriesSource(Object source) { + if (source instanceof Map mapSource && mapSource.containsKey(Entity.FIELD_DELETED)) { + Map scrubbed = new HashMap<>((Map) mapSource); + scrubbed.remove(Entity.FIELD_DELETED); + return JsonUtils.readOrConvertValue(scrubbed, entityClass); + } + return JsonUtils.readOrConvertValue(source, entityClass); + } + protected void setExcludeSearchFields(SearchListFilter searchListFilter) { // Nothing to do in the default implementation } diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/TestCaseRepository.java b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/TestCaseRepository.java index 813ce837ea31..187b9da9ee56 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/TestCaseRepository.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/TestCaseRepository.java @@ -112,8 +112,9 @@ @Slf4j public class TestCaseRepository extends EntityRepository { - private static final String TEST_SUITE_FIELD = "testSuite"; - private static final String INCIDENTS_FIELD = "incidentId"; + public static final String TEST_SUITE_FIELD = "testSuite"; + public static final String TEST_DEFINITION_FIELD = "testDefinition"; + public static final String INCIDENTS_FIELD = "incidentId"; private static final String UPDATE_FIELDS = "owners,entityLink,testSuite,testSuites,testDefinition,dimensionColumns,topDimensions"; private static final String PATCH_FIELDS = diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/TestSuiteRepository.java b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/TestSuiteRepository.java index fd4afde0b90c..e92e165bdedc 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/TestSuiteRepository.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/TestSuiteRepository.java @@ -84,6 +84,7 @@ @Slf4j public class TestSuiteRepository extends EntityRepository { + public static final String SUMMARY_FIELD = "summary"; private static final String UPDATE_FIELDS = "tests"; private static final String PATCH_FIELDS = "tests"; @@ -137,7 +138,7 @@ public TestSuiteRepository() { supportsSearch = true; EntityLifecycleEventDispatcher.getInstance() .registerHandler(new TestSuitePipelineStatusHandler()); - fieldFetchers.put("summary", this::fetchAndSetTestCaseResultSummary); + fieldFetchers.put(SUMMARY_FIELD, this::fetchAndSetTestCaseResultSummary); fieldFetchers.put("pipelines", this::fetchAndSetIngestionPipelines); } @@ -179,11 +180,11 @@ public void setFields( fields.contains("pipelines") ? getIngestionPipelines(entity) : entity.getPipelines()); entity.setTests(fields.contains("tests") ? getTestCases(entity) : entity.getTests()); entity.setTestCaseResultSummary( - fields.contains("summary") + fields.contains(SUMMARY_FIELD) ? getResultSummary(entity.getId()) : entity.getTestCaseResultSummary()); entity.setSummary( - fields.contains("summary") + fields.contains(SUMMARY_FIELD) ? getTestSummary(entity.getTestCaseResultSummary()) : entity.getSummary()); @@ -252,9 +253,9 @@ private Map> batchFetchTestCases(List tes @Override public void clearFields(TestSuite entity, EntityUtil.Fields fields) { entity.setPipelines(fields.contains("pipelines") ? entity.getPipelines() : null); - entity.setSummary(fields.contains("summary") ? entity.getSummary() : null); + entity.setSummary(fields.contains(SUMMARY_FIELD) ? entity.getSummary() : null); entity.setTestCaseResultSummary( - fields.contains("summary") ? entity.getTestCaseResultSummary() : null); + fields.contains(SUMMARY_FIELD) ? entity.getTestCaseResultSummary() : null); entity.withTests(fields.contains(UPDATE_FIELDS) ? entity.getTests() : null); } @@ -531,7 +532,7 @@ protected void postCreate(TestSuite entity) { private void fetchAndSetTestCaseResultSummary( List testSuites, EntityUtil.Fields fields) { - if (!fields.contains("summary") || testSuites == null || testSuites.isEmpty()) { + if (!fields.contains(SUMMARY_FIELD) || testSuites == null || testSuites.isEmpty()) { return; } diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/search/SearchClient.java b/openmetadata-service/src/main/java/org/openmetadata/service/search/SearchClient.java index 2e185c03290e..3b094b646c9c 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/search/SearchClient.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/search/SearchClient.java @@ -83,8 +83,57 @@ public interface SearchClient ctx._source.put('%s', newObject); } """; - String SOFT_DELETE_RESTORE_SCRIPT = "ctx._source.put('deleted', '%s')"; - String REMOVE_TAGS_CHILDREN_SCRIPT = "ctx._source.tags.removeIf(tag -> tag.tagFQN == params.fqn)"; + + /** + * Painless snippet that re-derives {@code tier} / {@code classificationTags} / + * {@code glossaryTags} from the current state of {@code ctx._source.tags}. Append this to every + * script that mutates {@code tags[]} so live-indexing updates produce the same separation that + * {@code TaggableIndex.applyTagFields} (the reindex path) produces. Without this, a propagation + * or glossary-rename script can leave the lifted fields stale or land a Tier.* TagLabel inside + * {@code tags[]}. + * + *

The shape mirrors {@code ParseTags}: Tier.* is lifted out of {@code tags[]} into + * {@code tier}, but its FQN is still included in {@code classificationTags} since it's + * sourced from a Classification — {@code ParseTags} iterates the original list to populate + * {@code classificationTags}, so the painless equivalent must do the same. + * + *

Important: {@code ctx._source.tier} is only overwritten when a Tier.* entry is actually + * found in {@code tags[]}. {@code TaggableIndex.applyTagFields} already strips Tier out of + * {@code tags[]} into the dedicated {@code tier} field at index time, so docs touched by a + * tag-mutating painless almost never carry Tier in {@code tags[]}. Unconditionally assigning + * {@code tier = null} when no Tier was seen would wipe the live-indexed dedicated field — + * caught by {@code GlossaryRenameCascade.spec.ts}. + */ + String TAG_RESEPARATION_SCRIPT = + """ + def newTags = new ArrayList(); + def tier = null; + def classTags = new ArrayList(); + def glossTags = new ArrayList(); + if (ctx._source.containsKey('tags') && ctx._source.tags != null) { + for (def t : ctx._source.tags) { + if (t == null || !t.containsKey('tagFQN') || t.tagFQN == null) { continue; } + if (t.tagFQN.startsWith('Tier.')) { + tier = t; + } else { + newTags.add(t); + } + if (t.containsKey('source')) { + if (t.source == 'Classification') { classTags.add(t.tagFQN); } + else if (t.source == 'Glossary') { glossTags.add(t.tagFQN); } + } + } + ctx._source.tags = newTags; + if (tier != null) { + ctx._source.tier = tier; + } + ctx._source.classificationTags = classTags; + ctx._source.glossaryTags = glossTags; + } + """; + + String REMOVE_TAGS_CHILDREN_SCRIPT = + "ctx._source.tags.removeIf(tag -> tag.tagFQN == params.fqn);" + TAG_RESEPARATION_SCRIPT; String REMOVE_DATA_PRODUCTS_CHILDREN_SCRIPT = "ctx._source.dataProducts.removeIf(product -> product.fullyQualifiedName == params.fqn)"; @@ -191,7 +240,8 @@ public interface SearchClient } } } - """; + """ + + TAG_RESEPARATION_SCRIPT; String UPDATE_CLASSIFICATION_TAG_FQN_BY_PREFIX_SCRIPT = """ @@ -205,7 +255,8 @@ public interface SearchClient } } } - """; + """ + + TAG_RESEPARATION_SCRIPT; String UPDATE_FQN_PREFIX_SCRIPT = """ @@ -234,7 +285,8 @@ public interface SearchClient } } } - """; + """ + + TAG_RESEPARATION_SCRIPT; String REMOVE_LINEAGE_SCRIPT = """ @@ -382,7 +434,8 @@ public interface SearchClient Collections.sort(uniqueTags, (o1, o2) -> o1.tagFQN.compareTo(o2.tagFQN)); ctx._source.tags = uniqueTags; - """; + """ + + TAG_RESEPARATION_SCRIPT; String REMOVE_TEST_SUITE_CHILDREN_SCRIPT = "ctx._source.testSuites.removeIf(suite -> suite.id == params.suiteId)"; diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/search/SearchRepository.java b/openmetadata-service/src/main/java/org/openmetadata/service/search/SearchRepository.java index 083cddf6e6a3..33280f5693a8 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/search/SearchRepository.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/search/SearchRepository.java @@ -32,7 +32,6 @@ import static org.openmetadata.service.search.SearchClient.REMOVE_PROPAGATED_FIELD_SCRIPT; import static org.openmetadata.service.search.SearchClient.REMOVE_TAGS_CHILDREN_SCRIPT; import static org.openmetadata.service.search.SearchClient.REMOVE_TEST_SUITE_CHILDREN_SCRIPT; -import static org.openmetadata.service.search.SearchClient.SOFT_DELETE_RESTORE_SCRIPT; import static org.openmetadata.service.search.SearchClient.UPDATE_ADDED_DELETE_GLOSSARY_TAGS; import static org.openmetadata.service.search.SearchClient.UPDATE_CERTIFICATION_SCRIPT; import static org.openmetadata.service.search.SearchClient.UPDATE_PROPAGATED_ENTITY_REFERENCE_FIELD_SCRIPT; @@ -129,6 +128,7 @@ import org.openmetadata.service.jdbi3.EntityRepository; import org.openmetadata.service.monitoring.RequestLatencyContext; import org.openmetadata.service.resources.settings.SettingsCache; +import org.openmetadata.service.search.capability.EntityIndexCapabilityRegistry; import org.openmetadata.service.search.elasticsearch.ElasticSearchClient; import org.openmetadata.service.search.indexes.ColumnSearchIndex; import org.openmetadata.service.search.indexes.PipelineExecutionIndex; @@ -136,6 +136,7 @@ import org.openmetadata.service.search.nlq.NLQService; import org.openmetadata.service.search.nlq.NLQServiceFactory; import org.openmetadata.service.search.opensearch.OpenSearchClient; +import org.openmetadata.service.search.scripts.SoftDeleteScript; import org.openmetadata.service.search.vector.OpenSearchVectorService; import org.openmetadata.service.search.vector.VectorEmbeddingHandler; import org.openmetadata.service.search.vector.VectorIndexService; @@ -2200,7 +2201,8 @@ private String generateAddTagLabelListScript() { } } Collections.sort(ctx._source.tags, (o1, o2) -> o1.tagFQN.compareTo(o2.tagFQN)); - """; + """ + + SearchClient.TAG_RESEPARATION_SCRIPT; } private String generateDeleteTagLabelListScript() { @@ -2215,7 +2217,8 @@ private String generateDeleteTagLabelListScript() { } } } - """; + """ + + SearchClient.TAG_RESEPARATION_SCRIPT; } private String generateUpdateTagLabelListScript() { @@ -2248,7 +2251,8 @@ private String generateUpdateTagLabelListScript() { } } Collections.sort(ctx._source.tags, (o1, o2) -> o1.tagFQN.compareTo(o2.tagFQN)); - """; + """ + + SearchClient.TAG_RESEPARATION_SCRIPT; } public void deleteByScript(String entityType, String scriptTxt, Map params) { @@ -2389,10 +2393,11 @@ public void softDeleteOrRestoreEntityIndex(EntityInterface entity, boolean delet return; } IndexMapping indexMapping = entityIndexMap.get(entityType); - String scriptTxt = String.format(SOFT_DELETE_RESTORE_SCRIPT, delete); + SoftDeleteScript script = new SoftDeleteScript(delete); Timer.Sample searchSample = RequestLatencyContext.startSearchOperation(); try { - searchClient.softDeleteOrRestoreEntity(getWriteIndexName(indexMapping), entityId, scriptTxt); + searchClient.softDeleteOrRestoreEntity( + getWriteIndexName(indexMapping), entityId, script.painless()); softDeleteOrRestoredChildren(entity.getEntityReference(), indexMapping, delete); if (Entity.TABLE.equals(entityType)) { @@ -2419,12 +2424,12 @@ private void softDeleteOrRestoreTableColumns(Table table, boolean delete) { return; } - String scriptTxt = String.format(SOFT_DELETE_RESTORE_SCRIPT, delete); + SoftDeleteScript script = new SoftDeleteScript(delete); try { searchClient.updateChildren( List.of(columnIndexMapping.getIndexName(clusterAlias)), new ImmutablePair<>("table.id", table.getId().toString()), - new ImmutablePair<>(scriptTxt, null)); + new ImmutablePair<>(script.painless(), null)); } catch (Exception e) { LOG.error( "Issue soft deleting/restoring columns for table [{}]: {}", @@ -2511,32 +2516,28 @@ public void deleteOrUpdateChildren(EntityInterface entity, IndexMapping indexMap public void softDeleteOrRestoredChildren( EntityReference entityReference, IndexMapping indexMapping, boolean delete) throws IOException { - String docId = entityReference.getId().toString(); - String entityType = entityReference.getType(); - String scriptTxt = String.format(SOFT_DELETE_RESTORE_SCRIPT, delete); - switch (entityType) { - case Entity.DASHBOARD_SERVICE, - Entity.DATABASE_SERVICE, - Entity.MESSAGING_SERVICE, - Entity.PIPELINE_SERVICE, - Entity.MLMODEL_SERVICE, - Entity.STORAGE_SERVICE, - Entity.SEARCH_SERVICE, - Entity.SECURITY_SERVICE, - Entity.DRIVE_SERVICE -> searchClient.softDeleteOrRestoreChildren( - indexMapping.getChildAliases(clusterAlias), - scriptTxt, - List.of(new ImmutablePair<>("service.id", docId))); - default -> { - List indexNames = indexMapping.getChildAliases(clusterAlias); - if (!indexNames.isEmpty()) { - searchClient.softDeleteOrRestoreChildren( - indexMapping.getChildAliases(clusterAlias), - scriptTxt, - List.of(new ImmutablePair<>(entityType + ".id", docId))); - } - } + // Each childAlias is an entity-type name (per indexMapping.json). Use the typed script's + // capability check so we never apply soft-delete to an index whose schema lacks `deleted`. + SoftDeleteScript script = new SoftDeleteScript(delete); + boolean hasClusterAlias = clusterAlias != null && !clusterAlias.isEmpty(); + List targets = + indexMapping.getChildAliases().stream() + .filter(a -> script.compatibleWith(EntityIndexCapabilityRegistry.get(a))) + .map(a -> hasClusterAlias ? clusterAlias + IndexMapping.INDEX_NAME_SEPARATOR + a : a) + .toList(); + if (targets.isEmpty()) { + return; } + String entityType = entityReference.getType(); + // Service entities propagate child deletions through a shared service.id field; everything + // else uses the entity-type-specific .id. Reuses the canonical SERVICE_ENTITY_SET that + // updateChildrenForSearchPropagation also relies on, so the contract stays in one place. + String parentIdField = + SERVICE_ENTITY_SET.contains(entityType) ? "service.id" : entityType + ".id"; + searchClient.softDeleteOrRestoreChildren( + targets, + script.painless(), + List.of(new ImmutablePair<>(parentIdField, entityReference.getId().toString()))); } public String getScriptWithParams( diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/search/capability/EntityIndexCapability.java b/openmetadata-service/src/main/java/org/openmetadata/service/search/capability/EntityIndexCapability.java new file mode 100644 index 000000000000..f0f9ec375d8d --- /dev/null +++ b/openmetadata-service/src/main/java/org/openmetadata/service/search/capability/EntityIndexCapability.java @@ -0,0 +1,35 @@ +/* + * Copyright 2026 Collate. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.openmetadata.service.search.capability; + +/** + * Per-entity-type flags that drive what the search-indexing layer can safely do. Today the only + * consumer is {@code IndexUpdateScript.compatibleWith(...)} — soft-delete propagation must NOT + * target an entity whose docs do not carry a top-level {@code deleted} field. The record is built + * once per entity at registration time (see {@code Entity.registerEntity}) so new entity types + * gain a correct capability record by default and can never silently drift. + * + *

The field set is intentionally minimal. New flags should be added only when a script or + * validator actually consults them; otherwise we accumulate dead metadata. + */ +public record EntityIndexCapability( + String entityType, boolean isTimeSeries, boolean hasFieldDeleted) { + + public static EntityIndexCapability forEntity(String entityType) { + return new EntityIndexCapability(entityType, false, true); + } + + public static EntityIndexCapability forTimeSeries(String entityType) { + return new EntityIndexCapability(entityType, true, false); + } +} diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/search/capability/EntityIndexCapabilityRegistry.java b/openmetadata-service/src/main/java/org/openmetadata/service/search/capability/EntityIndexCapabilityRegistry.java new file mode 100644 index 000000000000..fd3df9d1e508 --- /dev/null +++ b/openmetadata-service/src/main/java/org/openmetadata/service/search/capability/EntityIndexCapabilityRegistry.java @@ -0,0 +1,50 @@ +/* + * Copyright 2026 Collate. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.openmetadata.service.search.capability; + +import java.util.Collection; +import java.util.Collections; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; + +/** + * Process-global registry of {@link EntityIndexCapability} keyed by entity type. Populated by + * {@code Entity.registerEntity(...)} at startup; consumers (typed scripts, validators) read it + * thereafter. Returns {@code null} for unknown types so callers can decide whether to fail-soft + * or fail-hard. + */ +public final class EntityIndexCapabilityRegistry { + + private static final Map CAPABILITIES = new ConcurrentHashMap<>(); + + private EntityIndexCapabilityRegistry() {} + + public static void register(EntityIndexCapability capability) { + CAPABILITIES.put(capability.entityType(), capability); + } + + public static EntityIndexCapability get(String entityType) { + if (entityType == null) { + return null; + } + return CAPABILITIES.get(entityType); + } + + public static Collection all() { + return Collections.unmodifiableCollection(CAPABILITIES.values()); + } + + public static void clear() { + CAPABILITIES.clear(); + } +} diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/search/elasticsearch/ElasticSearchColumnAggregator.java b/openmetadata-service/src/main/java/org/openmetadata/service/search/elasticsearch/ElasticSearchColumnAggregator.java index 92160ba67456..675cd38e468b 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/search/elasticsearch/ElasticSearchColumnAggregator.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/search/elasticsearch/ElasticSearchColumnAggregator.java @@ -699,23 +699,18 @@ private Query notExistsQuery(String field) { return Query.of(q -> q.bool(b -> b.mustNot(existsQuery(field)))); } + // `wildcard(field, "?*")` matches any doc whose indexed terms include at least one token of + // at least one character — the analyzer-friendly equivalent of "field has non-empty value". + // We can't use `term(field, "")` against analyzed text fields like `columns.description`: the + // field's analyzer produces no tokens for the empty string and ES 7.17 rejects the term query + // with `search_phase_execution_exception ... all shards failed`. Caught by + // ColumnGridResourceIT#test_getColumnGrid_withMetadataStatusIncomplete. private Query hasNonEmptyField(String field) { - return Query.of( - q -> - q.bool( - b -> - b.must(existsQuery(field)) - .mustNot(Query.of(qn -> qn.term(t -> t.field(field).value("")))))); + return Query.of(q -> q.wildcard(w -> w.field(field).value("?*"))); } private Query hasEmptyOrMissingField(String field) { - return Query.of( - q -> - q.bool( - b -> - b.should(notExistsQuery(field)) - .should(Query.of(qs -> qs.term(t -> t.field(field).value("")))) - .minimumShouldMatch("1"))); + return Query.of(q -> q.bool(b -> b.mustNot(hasNonEmptyField(field)))); } /** Phase 1: Get all matching column names using terms agg with include regex (no top_hits). */ diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/TaggableIndex.java b/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/TaggableIndex.java index af46ad8f9b1e..7639db3f7a26 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/TaggableIndex.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/TaggableIndex.java @@ -26,11 +26,18 @@ public interface TaggableIndex extends SearchIndex { /** * Applies tag-related fields to the search index document. Called automatically by {@link - * SearchIndex#buildSearchIndexDoc()}. + * SearchIndex#buildSearchIndexDoc()} and shared by both the live-indexing path + * ({@link org.openmetadata.service.search.SearchRepository#updateEntityIndex}) and the + * SearchIndexApp reindex path ({@code BulkSink.addEntity}) — both converge on this method. * - *

Sets: tags, tier, classificationTags, glossaryTags from entity-level tags. Child tags - * (columns, schema fields) are merged later via {@link #mergeChildTags(Map, Set)} from within - * {@code buildSearchIndexDocInternal}, so that child structure flattening only happens once. + *

The doc has a deliberate separation: {@code tags[]} carries only classification and + * glossary tags; {@code tier} is the lifted Tier TagLabel; {@code certification} (set by + * {@code populateCommonFields}) is the structured {@code AssetCertification} object. Consumers + * filter through dedicated fields — UI queries should use {@code tier.tagFQN}, + * {@code certification.tagLabel.tagFQN}, {@code classificationTags}, {@code glossaryTags} — + * rather than treating {@code tags[]} as an all-encompassing bag. Child tags (columns, schema + * fields) are merged later via {@link #mergeChildTags(Map, Set)} from within + * {@code buildSearchIndexDocInternal}, so child structure flattening only happens once. */ default void applyTagFields(Map doc) { Object entity = getEntity(); diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/TestCaseIndex.java b/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/TestCaseIndex.java index e19a891024a0..128ca3a5b890 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/TestCaseIndex.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/TestCaseIndex.java @@ -14,6 +14,7 @@ import org.openmetadata.schema.type.Include; import org.openmetadata.service.Entity; import org.openmetadata.service.exception.EntityNotFoundException; +import org.openmetadata.service.jdbi3.TestCaseRepository; import org.openmetadata.service.resources.feeds.MessageParser; import org.openmetadata.service.search.SearchIndexUtils; @@ -34,16 +35,19 @@ public String getEntityTypeName() { @Override public Set getRequiredReindexFields() { Set fields = new java.util.HashSet<>(TaggableIndex.super.getRequiredReindexFields()); - fields.add("testSuite"); - fields.add("testSuites"); - fields.add("testDefinition"); + fields.add(TestCaseRepository.TEST_SUITE_FIELD); + fields.add(Entity.FIELD_TEST_SUITES); + fields.add(TestCaseRepository.TEST_DEFINITION_FIELD); + fields.add(Entity.TEST_CASE_RESULT); + fields.add(TestCaseRepository.INCIDENTS_FIELD); return java.util.Collections.unmodifiableSet(fields); } @Override public void removeNonIndexableFields(Map esDoc) { TaggableIndex.super.removeNonIndexableFields(esDoc); - List> testSuites = (List>) esDoc.get("testSuites"); + List> testSuites = + (List>) esDoc.get(Entity.FIELD_TEST_SUITES); if (testSuites != null) { for (Map testSuite : testSuites) { SearchIndexUtils.removeNonIndexableFields(testSuite, excludeFields); diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/TestSuiteIndex.java b/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/TestSuiteIndex.java index 6cbfa4109c7f..9f14099a5c37 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/TestSuiteIndex.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/TestSuiteIndex.java @@ -1,5 +1,7 @@ package org.openmetadata.service.search.indexes; +import java.util.Collections; +import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Set; @@ -10,9 +12,11 @@ import org.openmetadata.schema.type.Include; import org.openmetadata.service.Entity; import org.openmetadata.service.exception.EntityNotFoundException; +import org.openmetadata.service.jdbi3.TestSuiteRepository; public record TestSuiteIndex(TestSuite testSuite) implements TaggableIndex { - private static final Set excludeFields = Set.of("summary", "testCaseResultSummary"); + private static final Set excludeFields = + Set.of(TestSuiteRepository.SUMMARY_FIELD, "testCaseResultSummary"); @Override public Object getEntity() { @@ -29,6 +33,13 @@ public Set getExcludedFields() { return excludeFields; } + @Override + public Set getRequiredReindexFields() { + Set fields = new HashSet<>(TaggableIndex.super.getRequiredReindexFields()); + fields.add(TestSuiteRepository.SUMMARY_FIELD); + return Collections.unmodifiableSet(fields); + } + public Map buildSearchIndexDocInternal(Map doc) { setParentRelationships(doc, testSuite); diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/search/opensearch/OpenSearchColumnAggregator.java b/openmetadata-service/src/main/java/org/openmetadata/service/search/opensearch/OpenSearchColumnAggregator.java index 28b5f3be543e..269d486428f9 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/search/opensearch/OpenSearchColumnAggregator.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/search/opensearch/OpenSearchColumnAggregator.java @@ -577,26 +577,18 @@ private Query notExistsQuery(String field) { return Query.of(q -> q.bool(b -> b.mustNot(existsQuery(field)))); } + // `wildcard(field, "?*")` matches any doc whose indexed terms include at least one token of + // at least one character — the analyzer-friendly equivalent of "field has non-empty value". + // We can't use `term(field, "")` against analyzed text fields like `columns.description`: the + // field's analyzer produces no tokens for the empty string and OS rejects the term query with + // `search_phase_execution_exception ... all shards failed`. Caught by + // ColumnGridResourceIT#test_getColumnGrid_withMetadataStatusIncomplete. private Query hasNonEmptyField(String field) { - return Query.of( - q -> - q.bool( - b -> - b.must(existsQuery(field)) - .mustNot( - Query.of( - qn -> qn.term(t -> t.field(field).value(FieldValue.of(""))))))); + return Query.of(q -> q.wildcard(w -> w.field(field).value("?*"))); } private Query hasEmptyOrMissingField(String field) { - return Query.of( - q -> - q.bool( - b -> - b.should(notExistsQuery(field)) - .should( - Query.of(qs -> qs.term(t -> t.field(field).value(FieldValue.of(""))))) - .minimumShouldMatch("1"))); + return Query.of(q -> q.bool(b -> b.mustNot(hasNonEmptyField(field)))); } /** Phase 1: Get all matching column names using terms agg with include regex (no top_hits). */ diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/search/scripts/IndexUpdateScript.java b/openmetadata-service/src/main/java/org/openmetadata/service/search/scripts/IndexUpdateScript.java new file mode 100644 index 000000000000..43805d73fe1c --- /dev/null +++ b/openmetadata-service/src/main/java/org/openmetadata/service/search/scripts/IndexUpdateScript.java @@ -0,0 +1,38 @@ +/* + * Copyright 2026 Collate. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.openmetadata.service.search.scripts; + +import java.util.Map; +import org.openmetadata.service.search.capability.EntityIndexCapability; + +/** + * A painless script targeted at an OpenSearch / Elasticsearch index, paired with the entity-type + * capabilities it requires. Sealed so the script catalogue is closed and discoverable; each + * implementation declares both its rendered painless source and the {@code compatibleWith} check + * that prevents misapplication. + * + *

Prior to this abstraction the soft-delete script was a {@code String.format} template in + * {@code SearchClient} with no notion of which indexes it was safe to run against — that + * directly caused the Incident Manager Jackson error when {@code deleted} got stamped onto + * {@code testCaseResolutionStatus} docs whose schema declares no such field. Adding a new + * script type now requires answering "which capabilities does the target index need" before + * it can compile. + */ +public sealed interface IndexUpdateScript permits SoftDeleteScript { + + String painless(); + + Map params(); + + boolean compatibleWith(EntityIndexCapability capability); +} diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/search/scripts/SoftDeleteScript.java b/openmetadata-service/src/main/java/org/openmetadata/service/search/scripts/SoftDeleteScript.java new file mode 100644 index 000000000000..04342e701c8e --- /dev/null +++ b/openmetadata-service/src/main/java/org/openmetadata/service/search/scripts/SoftDeleteScript.java @@ -0,0 +1,46 @@ +/* + * Copyright 2026 Collate. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.openmetadata.service.search.scripts; + +import java.util.Collections; +import java.util.Map; +import org.openmetadata.service.search.capability.EntityIndexCapability; + +/** + * Sets the top-level {@code deleted} field on docs in indexes whose schema declares it. Refuses + * to run against time-series indexes (no {@code deleted} field) — the previous string-template + * version had no such guard and so polluted child docs of a soft-deleted parent. + * + *

Also fixes a latent quoting bug: the legacy template was + * {@code "ctx._source.put('deleted', '%s')"}, which wraps a boolean in single quotes — the + * resulting field is a string {@code "true"} / {@code "false"} rather than a JSON boolean. + * Consumers that read {@code _source.deleted} as a boolean (the UI does) accept both forms today + * but a stricter parser would not. + */ +public record SoftDeleteScript(boolean deleted) implements IndexUpdateScript { + + @Override + public String painless() { + return "ctx._source.put('deleted', " + deleted + ")"; + } + + @Override + public Map params() { + return Collections.emptyMap(); + } + + @Override + public boolean compatibleWith(EntityIndexCapability capability) { + return capability != null && capability.hasFieldDeleted(); + } +} diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/search/validation/IndexMappingValidator.java b/openmetadata-service/src/main/java/org/openmetadata/service/search/validation/IndexMappingValidator.java new file mode 100644 index 000000000000..9126ca2c9069 --- /dev/null +++ b/openmetadata-service/src/main/java/org/openmetadata/service/search/validation/IndexMappingValidator.java @@ -0,0 +1,71 @@ +/* + * Copyright 2026 Collate. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.openmetadata.service.search.validation; + +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import lombok.extern.slf4j.Slf4j; +import org.openmetadata.search.IndexMapping; +import org.openmetadata.service.search.capability.EntityIndexCapability; +import org.openmetadata.service.search.capability.EntityIndexCapabilityRegistry; +import org.openmetadata.service.search.scripts.SoftDeleteScript; + +/** + * Boot-time sanity check over the loaded {@code indexMapping.json}. For every parent → child + * pairing this validator asks the registered scripts whether they can safely target the child; + * any incompatibility is logged at WARN. The original incident — soft-delete propagation onto + * {@code testCaseResolutionStatus} / {@code testCaseResult} — would have surfaced here at app + * startup instead of producing a Jackson exception in the Incident Manager UI. + * + *

WARN-level (rather than fail-boot) for now: existing deployments may have mappings the + * platform team has not yet audited against the capability model. Flip to fail-fast once the + * production mappings have been cleaned up. + */ +@Slf4j +public final class IndexMappingValidator { + + private IndexMappingValidator() {} + + public static List validate(Map indexMappings) { + List warnings = new ArrayList<>(); + if (indexMappings == null || indexMappings.isEmpty()) { + return warnings; + } + SoftDeleteScript softDelete = new SoftDeleteScript(true); + for (Map.Entry entry : indexMappings.entrySet()) { + String parentType = entry.getKey(); + IndexMapping mapping = entry.getValue(); + List children = mapping.getChildAliases(); + if (children == null || children.isEmpty()) { + continue; + } + for (String childAlias : children) { + EntityIndexCapability childCapability = EntityIndexCapabilityRegistry.get(childAlias); + if (childCapability == null) { + warnings.add( + "Parent '%s' declares child alias '%s' with no registered capability; soft-delete" + + " propagation will skip it".formatted(parentType, childAlias)); + continue; + } + if (!softDelete.compatibleWith(childCapability)) { + warnings.add( + "Parent '%s' declares child alias '%s' which does not support SoftDelete (isTimeSeries=%s)" + .formatted(parentType, childAlias, childCapability.isTimeSeries())); + } + } + } + warnings.forEach(LOG::warn); + return warnings; + } +} diff --git a/openmetadata-service/src/test/java/org/openmetadata/service/apps/bundles/searchIndex/DistributedIndexingStrategyTest.java b/openmetadata-service/src/test/java/org/openmetadata/service/apps/bundles/searchIndex/DistributedIndexingStrategyTest.java index 042f19abf42e..789b3af6ce4b 100644 --- a/openmetadata-service/src/test/java/org/openmetadata/service/apps/bundles/searchIndex/DistributedIndexingStrategyTest.java +++ b/openmetadata-service/src/test/java/org/openmetadata/service/apps/bundles/searchIndex/DistributedIndexingStrategyTest.java @@ -478,8 +478,15 @@ void finalizeAllEntityReindexSkipsPromotedEntitiesAndFailsMissingEntityStats() t contextCaptor.getAllValues().get(i).getEntityType(), successCaptor.getAllValues().get(i)); } - assertEquals(Boolean.FALSE, outcomes.get("user")); - assertEquals(Boolean.FALSE, outcomes.get("dashboard")); + assertEquals( + Boolean.FALSE, + outcomes.get("user"), + "user has no entityStats entry — finalizer can't evaluate; default to not fully successful"); + assertEquals( + Boolean.FALSE, + outcomes.get("dashboard"), + "dashboard 4/5 (ratio 0.80) is below 0.95 — finalizer reports NOT fully successful;" + + " DefaultRecreateHandler's doc-count rescue then decides whether to promote"); } @Test diff --git a/openmetadata-service/src/test/java/org/openmetadata/service/apps/bundles/searchIndex/DistributedReindexFinalizerTest.java b/openmetadata-service/src/test/java/org/openmetadata/service/apps/bundles/searchIndex/DistributedReindexFinalizerTest.java index 62100526b05d..1cbc7f59f335 100644 --- a/openmetadata-service/src/test/java/org/openmetadata/service/apps/bundles/searchIndex/DistributedReindexFinalizerTest.java +++ b/openmetadata-service/src/test/java/org/openmetadata/service/apps/bundles/searchIndex/DistributedReindexFinalizerTest.java @@ -1,6 +1,7 @@ package org.openmetadata.service.apps.bundles.searchIndex; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.mockito.ArgumentMatchers.any; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; @@ -12,19 +13,23 @@ import org.mockito.ArgumentCaptor; import org.openmetadata.service.Entity; import org.openmetadata.service.apps.bundles.searchIndex.distributed.SearchIndexJob; +import org.openmetadata.service.apps.bundles.searchIndex.promotion.RatioPromotionPolicy; import org.openmetadata.service.search.EntityReindexContext; import org.openmetadata.service.search.RecreateIndexHandler; import org.openmetadata.service.search.ReindexContext; class DistributedReindexFinalizerTest { + private static final RatioPromotionPolicy DEFAULT_POLICY = + RatioPromotionPolicy.withDefaultThreshold(); + @Test void finalizeRemainingEntitiesPromotesColumnOnceWhenTableAndColumnRemain() { RecreateIndexHandler indexPromotionHandler = mock(RecreateIndexHandler.class); ReindexContext stagedIndexContext = stagedContext(Entity.TABLE, Entity.TABLE_COLUMN); DistributedReindexFinalizer finalizer = - new DistributedReindexFinalizer(indexPromotionHandler, stagedIndexContext); + new DistributedReindexFinalizer(indexPromotionHandler, stagedIndexContext, DEFAULT_POLICY); finalizer.finalizeRemainingEntities(Set.of(), Map.of(Entity.TABLE, successfulStats()), true); ArgumentCaptor contextCaptor = @@ -45,7 +50,7 @@ void finalizeRemainingEntitiesDoesNotRepromoteAlreadyPromotedColumnWhenTableRema ReindexContext stagedIndexContext = stagedContext(Entity.TABLE, Entity.TABLE_COLUMN); DistributedReindexFinalizer finalizer = - new DistributedReindexFinalizer(indexPromotionHandler, stagedIndexContext); + new DistributedReindexFinalizer(indexPromotionHandler, stagedIndexContext, DEFAULT_POLICY); finalizer.finalizeRemainingEntities( Set.of(Entity.TABLE_COLUMN), Map.of(Entity.TABLE, successfulStats()), true); @@ -59,6 +64,86 @@ void finalizeRemainingEntitiesDoesNotRepromoteAlreadyPromotedColumnWhenTableRema assertEquals(Boolean.TRUE, successCaptor.getValue()); } + @Test + void finalizeRemainingEntitiesPromotesPartialSuccessAboveThreshold() { + RecreateIndexHandler indexPromotionHandler = mock(RecreateIndexHandler.class); + ReindexContext stagedIndexContext = stagedContext(Entity.TABLE); + + SearchIndexJob.EntityTypeStats partial = + SearchIndexJob.EntityTypeStats.builder() + .entityType(Entity.TABLE) + .totalRecords(100) + .successRecords(99) + .failedRecords(1) + .build(); + + DistributedReindexFinalizer finalizer = + new DistributedReindexFinalizer( + indexPromotionHandler, stagedIndexContext, new RatioPromotionPolicy(0.95)); + finalizer.finalizeRemainingEntities(Set.of(), Map.of(Entity.TABLE, partial), false); + + ArgumentCaptor successCaptor = ArgumentCaptor.forClass(Boolean.class); + verify(indexPromotionHandler, times(1)).finalizeReindex(any(), successCaptor.capture()); + assertEquals( + Boolean.TRUE, + successCaptor.getValue(), + "99/100 records succeeded — above 0.95 threshold — must still promote"); + } + + @Test + void finalizeRemainingEntitiesFlagsBelowThresholdAsNotFullySuccessful() { + RecreateIndexHandler indexPromotionHandler = mock(RecreateIndexHandler.class); + ReindexContext stagedIndexContext = stagedContext(Entity.TABLE); + + SearchIndexJob.EntityTypeStats lowSuccess = + SearchIndexJob.EntityTypeStats.builder() + .entityType(Entity.TABLE) + .totalRecords(100) + .successRecords(40) + .failedRecords(60) + .build(); + + DistributedReindexFinalizer finalizer = + new DistributedReindexFinalizer( + indexPromotionHandler, stagedIndexContext, new RatioPromotionPolicy(0.95)); + finalizer.finalizeRemainingEntities(Set.of(), Map.of(Entity.TABLE, lowSuccess), false); + + ArgumentCaptor successCaptor = ArgumentCaptor.forClass(Boolean.class); + verify(indexPromotionHandler, times(1)).finalizeReindex(any(), successCaptor.capture()); + assertEquals( + Boolean.FALSE, + successCaptor.getValue(), + "40/100 records succeeded — below 0.95 threshold — finalizer reports NOT fully" + + " successful; DefaultRecreateHandler will rescue via doc-count when this is false."); + } + + @Test + void finalizeRemainingEntitiesFlagsZeroSuccessAsNotFullySuccessful() { + RecreateIndexHandler indexPromotionHandler = mock(RecreateIndexHandler.class); + ReindexContext stagedIndexContext = stagedContext(Entity.TABLE); + + SearchIndexJob.EntityTypeStats zeroSuccess = + SearchIndexJob.EntityTypeStats.builder() + .entityType(Entity.TABLE) + .totalRecords(100) + .successRecords(0) + .failedRecords(100) + .build(); + + DistributedReindexFinalizer finalizer = + new DistributedReindexFinalizer( + indexPromotionHandler, stagedIndexContext, new RatioPromotionPolicy(0.95)); + finalizer.finalizeRemainingEntities(Set.of(), Map.of(Entity.TABLE, zeroSuccess), false); + + ArgumentCaptor successCaptor = ArgumentCaptor.forClass(Boolean.class); + verify(indexPromotionHandler, times(1)).finalizeReindex(any(), successCaptor.capture()); + assertEquals( + Boolean.FALSE, + successCaptor.getValue(), + "zero successful records — handler's docCount rescue will then drop the empty staged" + + " index."); + } + private Map finalizations( ArgumentCaptor contextCaptor, ArgumentCaptor successCaptor) { List contexts = contextCaptor.getAllValues(); diff --git a/openmetadata-service/src/test/java/org/openmetadata/service/apps/bundles/searchIndex/promotion/RatioPromotionPolicyTest.java b/openmetadata-service/src/test/java/org/openmetadata/service/apps/bundles/searchIndex/promotion/RatioPromotionPolicyTest.java new file mode 100644 index 000000000000..efaa927d433c --- /dev/null +++ b/openmetadata-service/src/test/java/org/openmetadata/service/apps/bundles/searchIndex/promotion/RatioPromotionPolicyTest.java @@ -0,0 +1,109 @@ +/* + * Copyright 2026 Collate. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.openmetadata.service.apps.bundles.searchIndex.promotion; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import org.junit.jupiter.api.Test; + +class RatioPromotionPolicyTest { + + private static EntityPromotionContext ctx(long total, long success, long failed, long processed) { + return new EntityPromotionContext("table", total, success, failed, processed); + } + + private static EntityPromotionContext completeCtx(long total, long success, long failed) { + return ctx(total, success, failed, success + failed); + } + + @Test + void fullySuccessfulAtOrAboveThreshold() { + RatioPromotionPolicy policy = new RatioPromotionPolicy(0.95); + + assertTrue( + policy.evaluate(completeCtx(100, 95, 5)).fullySuccessful(), + "exactly at threshold must report fully successful"); + assertTrue( + policy.evaluate(completeCtx(100, 100, 0)).fullySuccessful(), + "100% must report fully successful"); + } + + @Test + void notFullySuccessfulBelowThreshold() { + RatioPromotionPolicy policy = new RatioPromotionPolicy(0.95); + + PromotionPolicy.Decision decision = policy.evaluate(completeCtx(100, 40, 60)); + + assertFalse( + decision.fullySuccessful(), + "below threshold must NOT be fully successful — handler's doc-count rescue decides" + + " whether the staged index is promoted"); + assertTrue( + decision.reason().contains("rescue"), + () -> "reason should mention the downstream rescue; got: " + decision.reason()); + } + + @Test + void zeroSuccessRecordsNotFullySuccessful() { + RatioPromotionPolicy policy = new RatioPromotionPolicy(0.95); + + assertFalse(policy.evaluate(completeCtx(100, 0, 100)).fullySuccessful()); + } + + @Test + void noRecordsScheduledIsFullySuccessful() { + RatioPromotionPolicy policy = new RatioPromotionPolicy(0.95); + + assertTrue( + policy.evaluate(ctx(0, 0, 0, 0)).fullySuccessful(), "empty entity types are not failures"); + } + + @Test + void incompleteRunIsNotFullySuccessfulEvenAtHighRatio() { + RatioPromotionPolicy policy = new RatioPromotionPolicy(0.95); + + PromotionPolicy.Decision decision = policy.evaluate(ctx(100, 96, 0, 96)); + + assertFalse( + decision.fullySuccessful(), + "only 96 of 100 records were processed — job stopped early; must NOT be fully" + + " successful regardless of ratio over the processed subset"); + assertTrue( + decision.reason().contains("incomplete run"), + () -> "reason should call out the incomplete run explicitly; got: " + decision.reason()); + } + + @Test + void defaultFactoryUsesNinetyFivePercentThreshold() { + assertEquals( + 0.95d, + RatioPromotionPolicy.withDefaultThreshold().minSuccessRatio(), + "default threshold should be 0.95 — change in lockstep with eventPublisherJob.json"); + } + + @Test + void rejectsConstructionOutsideUnitInterval() { + assertThrows(IllegalArgumentException.class, () -> new RatioPromotionPolicy(-0.01)); + assertThrows(IllegalArgumentException.class, () -> new RatioPromotionPolicy(1.5)); + } + + @Test + void successRatioComputedCorrectlyOnContext() { + assertEquals(1.0d, ctx(0, 0, 0, 0).successRatio()); + assertEquals(0.5d, completeCtx(10, 5, 5).successRatio()); + assertEquals(0.95d, completeCtx(100, 95, 5).successRatio()); + } +} diff --git a/openmetadata-service/src/test/java/org/openmetadata/service/search/SearchClientTagScriptSeparationTest.java b/openmetadata-service/src/test/java/org/openmetadata/service/search/SearchClientTagScriptSeparationTest.java new file mode 100644 index 000000000000..57b23ec96a12 --- /dev/null +++ b/openmetadata-service/src/test/java/org/openmetadata/service/search/SearchClientTagScriptSeparationTest.java @@ -0,0 +1,146 @@ +/* + * Copyright 2026 Collate. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.openmetadata.service.search; + +import static org.junit.jupiter.api.Assertions.assertTrue; + +import org.junit.jupiter.api.Test; + +/** + * Locks in the contract that every painless script which mutates {@code ctx._source.tags} also + * ends with the {@link SearchClient#TAG_RESEPARATION_SCRIPT} re-derivation snippet. Live-indexing + * updates use these scripts; the SearchIndexApp reindex path uses + * {@link org.openmetadata.service.search.indexes.TaggableIndex#applyTagFields} (which calls + * {@link ParseTags}). Both paths must produce the same separation — Tier lifted to + * {@code tier}, classification FQNs on {@code classificationTags}, glossary FQNs on + * {@code glossaryTags} — or queries that filter via the dedicated fields diverge between the + * two paths. + */ +class SearchClientTagScriptSeparationTest { + + @Test + void removeTagsChildrenScriptReseparatesAfterMutation() { + assertEndsWithReseparation(SearchClient.REMOVE_TAGS_CHILDREN_SCRIPT, "REMOVE_TAGS_CHILDREN"); + } + + @Test + void updateGlossaryTermTagFqnByPrefixScriptReseparatesAfterMutation() { + assertEndsWithReseparation( + SearchClient.UPDATE_GLOSSARY_TERM_TAG_FQN_BY_PREFIX_SCRIPT, + "UPDATE_GLOSSARY_TERM_TAG_FQN_BY_PREFIX"); + } + + @Test + void updateClassificationTagFqnByPrefixScriptReseparatesAfterMutation() { + assertEndsWithReseparation( + SearchClient.UPDATE_CLASSIFICATION_TAG_FQN_BY_PREFIX_SCRIPT, + "UPDATE_CLASSIFICATION_TAG_FQN_BY_PREFIX"); + } + + @Test + void updateFqnPrefixScriptReseparatesAfterMutation() { + assertEndsWithReseparation(SearchClient.UPDATE_FQN_PREFIX_SCRIPT, "UPDATE_FQN_PREFIX"); + } + + @Test + void updateAddedDeleteGlossaryTagsReseparatesAfterMutation() { + assertEndsWithReseparation( + SearchClient.UPDATE_ADDED_DELETE_GLOSSARY_TAGS, "UPDATE_ADDED_DELETE_GLOSSARY_TAGS"); + } + + @Test + void tagReseparationScriptSkipsDocsWithoutTagsField() { + // UPDATE_FQN_PREFIX_SCRIPT is invoked against GLOBAL_SEARCH_ALIAS, which includes + // tag_search_index. Tag docs have no `tags` field; if the four reseparation writes + // run unconditionally they pollute the doc with empty tags / null tier / + // empty classificationTags / empty glossaryTags. Guard the writes inside the + // containsKey('tags') block. + String snippet = SearchClient.TAG_RESEPARATION_SCRIPT; + int guardIndex = snippet.indexOf("if (ctx._source.containsKey('tags')"); + assertTrue(guardIndex >= 0, "snippet must guard on ctx._source.containsKey('tags')"); + String beforeGuard = snippet.substring(0, guardIndex); + for (String forbidden : + new String[] { + "ctx._source.tags =", + "ctx._source.tier =", + "ctx._source.classificationTags =", + "ctx._source.glossaryTags =" + }) { + assertTrue( + !beforeGuard.contains(forbidden), + () -> + "Reseparation write '" + + forbidden + + "' must live inside the containsKey('tags') guard so docs without a" + + " tags field (e.g., tag_search_index) are not polluted."); + } + } + + @Test + void tagReseparationScriptLiftsTierAndPopulatesDenormalizations() { + String snippet = SearchClient.TAG_RESEPARATION_SCRIPT; + assertTrue( + snippet.contains("ctx._source.tier"), + "snippet must assign ctx._source.tier (the lifted Tier TagLabel)"); + assertTrue( + snippet.contains("ctx._source.classificationTags"), + "snippet must assign ctx._source.classificationTags (denormalised FQN list)"); + assertTrue( + snippet.contains("ctx._source.glossaryTags"), + "snippet must assign ctx._source.glossaryTags (denormalised FQN list)"); + assertTrue( + snippet.contains("startsWith('Tier.')"), + "snippet must filter Tier.* tags out of tags[] so they don't leak into the bag"); + } + + @Test + void tagReseparationScriptOnlyOverwritesTierWhenFoundInTagsBag() { + // TaggableIndex.applyTagFields strips Tier out of tags[] into the dedicated tier field at + // index time, so a doc touched by any tag-mutating painless almost never carries Tier + // inside tags[]. If the snippet unconditionally executed `ctx._source.tier = tier` after a + // loop that didn't see any Tier.* entry, `tier` is null and the assignment wipes the + // live-indexed dedicated field — caught by GlossaryRenameCascade.spec.ts. The guard + // `if (tier != null)` around the assignment keeps the existing tier untouched in that + // case while still allowing the snippet to lift Tier back out of tags[] when a legacy / + // polluted doc has one stuck in there. + String snippet = SearchClient.TAG_RESEPARATION_SCRIPT; + int tierAssignIndex = snippet.indexOf("ctx._source.tier ="); + assertTrue( + tierAssignIndex >= 0, + "snippet must contain a `ctx._source.tier = ...` assignment to lift legacy Tier" + + " entries; if you removed it intentionally update this test."); + String upToAssignment = snippet.substring(0, tierAssignIndex); + int lastNullCheck = upToAssignment.lastIndexOf("if (tier != null)"); + assertTrue( + lastNullCheck >= 0, + "Reseparation write `ctx._source.tier = tier` must be guarded by `if (tier != null)`" + + " so docs whose Tier already lives on the dedicated field (the normal post-Phase 4a" + + " shape) are not wiped to null when no Tier.* is present in tags[]."); + } + + private static void assertEndsWithReseparation(String script, String label) { + // Suffix match — the snippet must be the LAST thing the script does so subsequent + // mutations can't re-break the separation. `contains` would let a future patch append + // additional tag-mutation logic after the reseparation and silently re-introduce drift. + String trimmedScript = script.trim(); + String trimmedSnippet = SearchClient.TAG_RESEPARATION_SCRIPT.trim(); + assertTrue( + trimmedScript.endsWith(trimmedSnippet), + () -> + "Painless script " + + label + + " must END WITH TAG_RESEPARATION_SCRIPT so no later mutation can re-introduce" + + " separation drift. Append TAG_RESEPARATION_SCRIPT at the very end of the" + + " script string."); + } +} diff --git a/openmetadata-service/src/test/java/org/openmetadata/service/search/SearchIndexFactoryTest.java b/openmetadata-service/src/test/java/org/openmetadata/service/search/SearchIndexFactoryTest.java index 7138bcc5ad32..b19c7e015f01 100644 --- a/openmetadata-service/src/test/java/org/openmetadata/service/search/SearchIndexFactoryTest.java +++ b/openmetadata-service/src/test/java/org/openmetadata/service/search/SearchIndexFactoryTest.java @@ -71,6 +71,7 @@ import org.openmetadata.schema.tests.type.TestCaseResolutionStatus; import org.openmetadata.schema.tests.type.TestCaseResult; import org.openmetadata.service.Entity; +import org.openmetadata.service.jdbi3.TestCaseRepository; import org.openmetadata.service.search.indexes.APICollectionIndex; import org.openmetadata.service.search.indexes.APIEndpointIndex; import org.openmetadata.service.search.indexes.APIServiceIndex; @@ -206,9 +207,20 @@ void reindexFieldsIncludeKnownOverrides() { assertTrue(userFields.contains("roles")); assertTrue(userFields.contains("inheritedRoles")); Set testCaseFields = factory.getReindexFieldsFor(Entity.TEST_CASE); - assertTrue(testCaseFields.contains("testSuite")); - assertTrue(testCaseFields.contains("testSuites")); - assertTrue(testCaseFields.contains("testDefinition")); + assertTrue(testCaseFields.contains(TestCaseRepository.TEST_SUITE_FIELD)); + assertTrue(testCaseFields.contains(Entity.FIELD_TEST_SUITES)); + assertTrue(testCaseFields.contains(TestCaseRepository.TEST_DEFINITION_FIELD)); + // Regression: testCaseResult/incidentId are stripped from storage JSON and + // only fetched by setFieldsInBulk when explicitly requested. Reindex without + // them produces docs missing testCaseStatus, blanking statuses in the UI. + assertTrue(testCaseFields.contains(Entity.TEST_CASE_RESULT)); + assertTrue(testCaseFields.contains(TestCaseRepository.INCIDENTS_FIELD)); + // TestSuiteRepository registers a fetcher for "summary" that populates + // testCaseResultSummary. The DQ TestSuites list page sorts by the + // top-level lastResultTimestamp field (computed in TestSuiteIndex from + // that summary) and renders a success-% column per row. Without + // "summary" the fetcher never runs and the ES doc has neither field. + assertTrue(factory.getReindexFieldsFor(Entity.TEST_SUITE).contains("summary")); } @Test diff --git a/openmetadata-service/src/test/java/org/openmetadata/service/search/SearchRepositoryBehaviorTest.java b/openmetadata-service/src/test/java/org/openmetadata/service/search/SearchRepositoryBehaviorTest.java index 0bc8e1bc61ba..8e4a4671ea61 100644 --- a/openmetadata-service/src/test/java/org/openmetadata/service/search/SearchRepositoryBehaviorTest.java +++ b/openmetadata-service/src/test/java/org/openmetadata/service/search/SearchRepositoryBehaviorTest.java @@ -89,7 +89,7 @@ class SearchRepositoryBehaviorTest { IndexMapping.builder() .indexName("table_search_index") .alias("table") - .childAliases(List.of("column_search_index")) + .childAliases(List.of(Entity.TABLE_COLUMN)) .indexMappingFile("/elasticsearch/%s/table_index_mapping.json") .build(); @@ -113,7 +113,7 @@ class SearchRepositoryBehaviorTest { IndexMapping.builder() .indexName("database_service_search_index") .alias("databaseService") - .childAliases(List.of("database_search_index")) + .childAliases(List.of(Entity.DATABASE)) .indexMappingFile("/elasticsearch/%s/database_service_index_mapping.json") .build(); @@ -133,14 +133,29 @@ class SearchRepositoryBehaviorTest { .indexMappingFile("/elasticsearch/%s/test_suite_index_mapping.json") .build(); + private static final IndexMapping TEST_CASE_MAPPING = + IndexMapping.builder() + .indexName("test_case_search_index") + .alias("testCase") + .childAliases( + List.of( + Entity.TEST_CASE_RESOLUTION_STATUS, Entity.TEST_CASE_RESULT, Entity.TABLE_COLUMN)) + .indexMappingFile("/elasticsearch/%s/test_case_index_mapping.json") + .build(); + + private static final List MOCK_TIME_SERIES_ENTITY_TYPES = + List.of(Entity.TEST_CASE_RESOLUTION_STATUS, Entity.TEST_CASE_RESULT); + private static final List MOCK_ENTITY_TYPES = List.of( Entity.TABLE, + Entity.TABLE_COLUMN, Entity.GLOSSARY_TERM, Entity.TAG, Entity.PAGE, Entity.DOMAIN, Entity.DATABASE_SERVICE, + Entity.DATABASE, Entity.TEST_SUITE, Entity.GLOSSARY, Entity.CLASSIFICATION, @@ -168,16 +183,19 @@ void setUp() { Map.entry(Entity.CLASSIFICATION, TABLE_MAPPING), Map.entry(Entity.PAGE, PAGE_MAPPING), Map.entry(Entity.TEST_SUITE, TEST_SUITE_MAPPING), + Map.entry(Entity.TEST_CASE, TEST_CASE_MAPPING), Map.entry(Entity.QUERY, TABLE_MAPPING)), "cluster"); Entity.setSearchRepository(repository); registerMockEntityRepositories(); + registerMockTimeSeriesRepositories(); } @AfterEach void tearDown() { Entity.setSearchRepository(null); clearMockEntityRepositories(); + clearMockTimeSeriesRepositories(); } @SuppressWarnings("unchecked") @@ -192,6 +210,8 @@ private void registerMockEntityRepositories() { EntityRepository mockRepo = mock(EntityRepository.class); doReturn(descriptors).when(mockRepo).getSearchPropagationDescriptors(); repoMap.put(entityType, mockRepo); + org.openmetadata.service.search.capability.EntityIndexCapabilityRegistry.register( + org.openmetadata.service.search.capability.EntityIndexCapability.forEntity(entityType)); } } catch (Exception e) { throw new RuntimeException("Failed to register mock entity repositories", e); @@ -205,11 +225,41 @@ private void clearMockEntityRepositories() { repoMapField.setAccessible(true); Map repoMap = (Map) repoMapField.get(null); MOCK_ENTITY_TYPES.forEach(repoMap::remove); + org.openmetadata.service.search.capability.EntityIndexCapabilityRegistry.clear(); } catch (Exception e) { throw new RuntimeException("Failed to clear mock entity repositories", e); } } + @SuppressWarnings({"unchecked", "rawtypes"}) + private void registerMockTimeSeriesRepositories() { + try { + Field tsMap = Entity.class.getDeclaredField("ENTITY_TS_REPOSITORY_MAP"); + tsMap.setAccessible(true); + Map map = (Map) tsMap.get(null); + for (String entityType : MOCK_TIME_SERIES_ENTITY_TYPES) { + map.put(entityType, mock(org.openmetadata.service.jdbi3.EntityTimeSeriesRepository.class)); + org.openmetadata.service.search.capability.EntityIndexCapabilityRegistry.register( + org.openmetadata.service.search.capability.EntityIndexCapability.forTimeSeries( + entityType)); + } + } catch (Exception e) { + throw new RuntimeException("Failed to register mock time-series repositories", e); + } + } + + @SuppressWarnings("unchecked") + private void clearMockTimeSeriesRepositories() { + try { + Field tsMap = Entity.class.getDeclaredField("ENTITY_TS_REPOSITORY_MAP"); + tsMap.setAccessible(true); + Map map = (Map) tsMap.get(null); + MOCK_TIME_SERIES_ENTITY_TYPES.forEach(map::remove); + } catch (Exception e) { + throw new RuntimeException("Failed to clear mock time-series repositories", e); + } + } + private List buildDescriptorsFor(String entityType) { String displayNameNestPath = Entity.DATABASE_SERVICE.equals(entityType) @@ -623,9 +673,7 @@ void propagateInheritedFieldsToChildrenUsesServiceParentFieldForServiceDisplayNa ArgumentCaptor.forClass(Pair.class); verify(searchClient) .updateChildren( - eq(List.of("cluster_database_search_index")), - fieldCaptor.capture(), - updateCaptor.capture()); + eq(List.of("cluster_database")), fieldCaptor.capture(), updateCaptor.capture()); assertEquals("service.id", fieldCaptor.getValue().getLeft()); assertEquals("service-id", fieldCaptor.getValue().getRight()); assertEquals("New Service", updateCaptor.getValue().getRight().get(Entity.FIELD_DISPLAY_NAME)); @@ -889,7 +937,7 @@ void deleteAndSoftDeleteOperationsSkipUnsupportedTypesButHandleMappedEntities() .softDeleteOrRestoreEntity( "cluster_table_search_index", entity.getId().toString(), - String.format(SearchClient.SOFT_DELETE_RESTORE_SCRIPT, true)); + new org.openmetadata.service.search.scripts.SoftDeleteScript(true).painless()); EntityInterface unsupported = mockEntity("unsupported", UUID.randomUUID(), "skip-me"); spyRepository.deleteEntityIndex(unsupported); @@ -920,7 +968,7 @@ void deleteEntityIndexDeletesServiceChildrenByServiceId() throws Exception { verify(searchClient) .deleteEntityByFields( - List.of("cluster_database_search_index"), + List.of("cluster_database"), List.of( new org.apache.commons.lang3.tuple.ImmutablePair<>( "service.id", service.getId().toString()))); @@ -934,7 +982,7 @@ void deleteEntityIndexDeletesGenericChildrenByEntityTypeId() throws Exception { verify(searchClient) .deleteEntityByFields( - List.of("cluster_column_search_index"), + List.of("cluster_tableColumn"), List.of( new org.apache.commons.lang3.tuple.ImmutablePair<>( "table.id", table.getId().toString()))); @@ -2019,7 +2067,8 @@ void deleteEntityIndexHandlesNonBasicTestSuitesByUpdatingChildren() throws Excep @Test void softDeleteOrRestoreEntityIndexPropagatesServiceDeletionToChildren() throws Exception { EntityInterface service = mockEntity(Entity.DATABASE_SERVICE, UUID.randomUUID(), "service"); - String scriptTxt = String.format(SearchClient.SOFT_DELETE_RESTORE_SCRIPT, true); + String scriptTxt = + new org.openmetadata.service.search.scripts.SoftDeleteScript(true).painless(); repository.softDeleteOrRestoreEntityIndex(service, true); @@ -2028,7 +2077,7 @@ void softDeleteOrRestoreEntityIndexPropagatesServiceDeletionToChildren() throws "cluster_database_service_search_index", service.getId().toString(), scriptTxt); verify(searchClient) .softDeleteOrRestoreChildren( - List.of("cluster_database_search_index"), + List.of("cluster_database"), scriptTxt, List.of( new org.apache.commons.lang3.tuple.ImmutablePair<>( @@ -2038,19 +2087,71 @@ void softDeleteOrRestoreEntityIndexPropagatesServiceDeletionToChildren() throws @Test void softDeleteOrRestoredChildrenUsesEntityTypeFieldForGenericEntities() throws IOException { EntityReference table = new EntityReference().withId(UUID.randomUUID()).withType(Entity.TABLE); - String scriptTxt = String.format(SearchClient.SOFT_DELETE_RESTORE_SCRIPT, false); + String scriptTxt = + new org.openmetadata.service.search.scripts.SoftDeleteScript(false).painless(); repository.softDeleteOrRestoredChildren(table, TABLE_MAPPING, false); verify(searchClient) .softDeleteOrRestoreChildren( - List.of("cluster_column_search_index"), + List.of("cluster_tableColumn"), scriptTxt, List.of( new org.apache.commons.lang3.tuple.ImmutablePair<>( "table.id", table.getId().toString()))); } + /** + * Regression for the Incident Manager Jackson error. The soft-delete script must NOT target + * {@code testCaseResolutionStatus} / {@code testCaseResult} — those are time-series indexes + * whose entity class declares no top-level {@code deleted} field. Non-time-series children on + * the same parent (here {@code tableColumn}) are still propagated. + */ + @Test + @SuppressWarnings("unchecked") + void softDeleteOrRestoredChildrenSkipsTimeSeriesAliases() throws IOException { + EntityReference testCase = + new EntityReference().withId(UUID.randomUUID()).withType(Entity.TEST_CASE); + + repository.softDeleteOrRestoredChildren(testCase, TEST_CASE_MAPPING, true); + + ArgumentCaptor> aliasCaptor = ArgumentCaptor.forClass(List.class); + verify(searchClient) + .softDeleteOrRestoreChildren(aliasCaptor.capture(), any(String.class), any(List.class)); + List aliases = aliasCaptor.getValue(); + assertFalse( + aliases.contains("cluster_" + Entity.TEST_CASE_RESOLUTION_STATUS), + "testCaseResolutionStatus has no `deleted` field; the soft-delete script must not target it"); + assertFalse( + aliases.contains("cluster_" + Entity.TEST_CASE_RESULT), + "testCaseResult has no `deleted` field; the soft-delete script must not target it"); + assertTrue( + aliases.contains("cluster_tableColumn"), + "non-time-series children must still receive the propagation script"); + } + + /** + * When every declared child alias is a time-series entity, propagation is a no-op — the + * search client must not be invoked at all rather than be invoked with an empty list. + */ + @Test + void softDeleteOrRestoredChildrenIsNoOpWhenEveryChildIsTimeSeries() throws IOException { + IndexMapping timeSeriesOnly = + IndexMapping.builder() + .indexName("test_case_search_index") + .alias("testCase") + .childAliases(List.of(Entity.TEST_CASE_RESOLUTION_STATUS, Entity.TEST_CASE_RESULT)) + .indexMappingFile("/elasticsearch/%s/test_case_index_mapping.json") + .build(); + EntityReference testCase = + new EntityReference().withId(UUID.randomUUID()).withType(Entity.TEST_CASE); + + repository.softDeleteOrRestoredChildren(testCase, timeSeriesOnly, false); + + verify(searchClient, never()) + .softDeleteOrRestoreChildren(any(List.class), any(String.class), any(List.class)); + } + @Test void getScriptWithParamsBuildsExtensionAndDescriptionUpdates() { EntityInterface entity = mockEntity(Entity.TABLE, UUID.randomUUID(), "orders"); @@ -2555,6 +2656,7 @@ void repositoryMetadataHelpersExposeUnderlyingState() throws Exception { Entity.CLASSIFICATION, Entity.PAGE, Entity.TEST_SUITE, + Entity.TEST_CASE, Entity.QUERY), repository.getSearchEntities()); assertSame(highLevelClient, repository.getHighLevelClient()); diff --git a/openmetadata-service/src/test/java/org/openmetadata/service/search/capability/EntityIndexCapabilityRegistryTest.java b/openmetadata-service/src/test/java/org/openmetadata/service/search/capability/EntityIndexCapabilityRegistryTest.java new file mode 100644 index 000000000000..b1bb7f8be14e --- /dev/null +++ b/openmetadata-service/src/test/java/org/openmetadata/service/search/capability/EntityIndexCapabilityRegistryTest.java @@ -0,0 +1,78 @@ +/* + * Copyright 2026 Collate. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.openmetadata.service.search.capability; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +class EntityIndexCapabilityRegistryTest { + + @BeforeEach + @AfterEach + void resetRegistry() { + EntityIndexCapabilityRegistry.clear(); + } + + @Test + void registeredEntityHasFieldDeletedAndIsNotTimeSeries() { + EntityIndexCapabilityRegistry.register(EntityIndexCapability.forEntity("table")); + + EntityIndexCapability capability = EntityIndexCapabilityRegistry.get("table"); + assertTrue(capability.hasFieldDeleted()); + assertFalse(capability.isTimeSeries()); + assertEquals("table", capability.entityType()); + } + + @Test + void registeredTimeSeriesEntityLacksFieldDeleted() { + EntityIndexCapabilityRegistry.register( + EntityIndexCapability.forTimeSeries("testCaseResolutionStatus")); + + EntityIndexCapability capability = + EntityIndexCapabilityRegistry.get("testCaseResolutionStatus"); + assertFalse( + capability.hasFieldDeleted(), + "time-series entities never carry a top-level `deleted` field; scripts must opt out"); + assertTrue(capability.isTimeSeries()); + } + + @Test + void getReturnsNullForUnknownEntityType() { + assertNull(EntityIndexCapabilityRegistry.get("does-not-exist")); + assertNull(EntityIndexCapabilityRegistry.get(null)); + } + + @Test + void registrationOverwritesPriorCapability() { + EntityIndexCapabilityRegistry.register(EntityIndexCapability.forTimeSeries("test")); + EntityIndexCapabilityRegistry.register(EntityIndexCapability.forEntity("test")); + + assertTrue(EntityIndexCapabilityRegistry.get("test").hasFieldDeleted()); + } + + @Test + void clearEmptiesTheRegistry() { + EntityIndexCapabilityRegistry.register(EntityIndexCapability.forEntity("a")); + EntityIndexCapabilityRegistry.register(EntityIndexCapability.forTimeSeries("b")); + + EntityIndexCapabilityRegistry.clear(); + + assertEquals(0, EntityIndexCapabilityRegistry.all().size()); + } +} diff --git a/openmetadata-service/src/test/java/org/openmetadata/service/search/indexes/TaggableIndexTest.java b/openmetadata-service/src/test/java/org/openmetadata/service/search/indexes/TaggableIndexTest.java index d186d8de2fe4..3af46a271a52 100644 --- a/openmetadata-service/src/test/java/org/openmetadata/service/search/indexes/TaggableIndexTest.java +++ b/openmetadata-service/src/test/java/org/openmetadata/service/search/indexes/TaggableIndexTest.java @@ -70,6 +70,66 @@ void testApplyTagFieldsSetsAllFourTagFields() { assertNotNull(doc.get("glossaryTags")); } + /** + * Locks in the doc-shape separation that both the live-indexing path + * ({@code SearchRepository.updateEntityIndex}) and the SearchIndexApp reindex path + * ({@code BulkSink.addEntity}) produce — they converge on this same {@code applyTagFields}. + * + *

Tier is lifted out of {@code tags[]} onto the {@code tier} field; classification + + * glossary tags stay in {@code tags[]}. Consumers (UI, DQ filters, RBAC) must filter via the + * dedicated fields ({@code tier.tagFQN}, {@code certification.tagLabel.tagFQN}) — treating + * {@code tags[]} as an all-encompassing bag was the wrong contract. + */ + @Test + @SuppressWarnings("unchecked") + void tierIsLiftedOutOfTagsArrayOntoDedicatedField() { + TagLabel pii = + new TagLabel().withTagFQN("PII.Sensitive").withSource(TagLabel.TagSource.CLASSIFICATION); + TagLabel glossary = + new TagLabel() + .withTagFQN("BusinessGlossary.Revenue") + .withSource(TagLabel.TagSource.GLOSSARY); + TagLabel tier = + new TagLabel().withTagFQN("Tier.Tier1").withSource(TagLabel.TagSource.CLASSIFICATION); + + entityStaticMock + .when(() -> Entity.getEntityTags(anyString(), any(Dashboard.class))) + .thenReturn(new java.util.ArrayList<>(List.of(pii, glossary, tier))); + + Dashboard dashboard = + new Dashboard() + .withId(UUID.randomUUID()) + .withName("tier-separated") + .withFullyQualifiedName("svc.tier-separated"); + + DashboardIndex index = new DashboardIndex(dashboard); + Map doc = new HashMap<>(); + + index.applyTagFields(doc); + + List tags = (List) doc.get("tags"); + assertEquals( + 2, + tags.size(), + "Tier must NOT be in tags[]; only classification and glossary tags belong there"); + assertTrue( + tags.stream().noneMatch(t -> t.getTagFQN().startsWith("Tier.")), + "no Tier.* TagLabel may leak into tags[]; consumers must filter via tier.tagFQN"); + + TagLabel tierField = (TagLabel) doc.get("tier"); + assertNotNull(tierField, "tier field must carry the lifted Tier TagLabel"); + assertEquals("Tier.Tier1", tierField.getTagFQN()); + + List classificationTags = (List) doc.get("classificationTags"); + assertTrue( + classificationTags.contains("PII.Sensitive"), + "non-Tier classification FQNs go on classificationTags"); + + List glossaryTags = (List) doc.get("glossaryTags"); + assertTrue( + glossaryTags.contains("BusinessGlossary.Revenue"), "glossary FQNs go on glossaryTags"); + } + @Test void testApplyTagFieldsWithEmptyTags() { entityStaticMock diff --git a/openmetadata-service/src/test/java/org/openmetadata/service/search/indexes/TestCaseIndexTest.java b/openmetadata-service/src/test/java/org/openmetadata/service/search/indexes/TestCaseIndexTest.java index 54cf08ef2f2b..4b2f8c74eed1 100644 --- a/openmetadata-service/src/test/java/org/openmetadata/service/search/indexes/TestCaseIndexTest.java +++ b/openmetadata-service/src/test/java/org/openmetadata/service/search/indexes/TestCaseIndexTest.java @@ -15,6 +15,7 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Set; import java.util.UUID; import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.BeforeAll; @@ -30,6 +31,7 @@ import org.openmetadata.service.Entity; import org.openmetadata.service.exception.EntityNotFoundException; import org.openmetadata.service.jdbi3.CollectionDAO; +import org.openmetadata.service.jdbi3.TestCaseRepository; import org.openmetadata.service.search.SearchClient; import org.openmetadata.service.search.SearchRepository; @@ -144,6 +146,28 @@ void testBuildSearchIndexDoc_endToEnd_hasCommonAndTagFields() { assertNotNull(result.get("originEntityFQN")); } + @Test + void testRequiredReindexFields_includesTestCaseResultAndIncidentId() { + // Regression test for the 1.12.7 reindex bug: testCaseResult and incidentId + // are stripped from the storage JSON and only loaded by + // TestCaseRepository.setFieldsInBulk when present in the requested field + // set. If they are not in getRequiredReindexFields(), the reindexer writes + // a doc with no testCaseStatus and the UI/search shows test cases with no + // status until a per-case write re-populates them. + TestCase tc = new TestCase().withId(UUID.randomUUID()).withName("tc"); + Set required = new TestCaseIndex(tc).getRequiredReindexFields(); + + assertTrue( + required.contains(Entity.TEST_CASE_RESULT), + "TestCaseIndex.getRequiredReindexFields() must include 'testCaseResult'"); + assertTrue( + required.contains(TestCaseRepository.INCIDENTS_FIELD), + "TestCaseIndex.getRequiredReindexFields() must include 'incidentId'"); + assertTrue(required.contains(TestCaseRepository.TEST_SUITE_FIELD)); + assertTrue(required.contains(Entity.FIELD_TEST_SUITES)); + assertTrue(required.contains(TestCaseRepository.TEST_DEFINITION_FIELD)); + } + @Test void testBuildSearchIndexDocInternal_testDefinitionNotFound() { UUID testDefId = UUID.randomUUID(); diff --git a/openmetadata-service/src/test/java/org/openmetadata/service/search/scripts/SoftDeleteScriptTest.java b/openmetadata-service/src/test/java/org/openmetadata/service/search/scripts/SoftDeleteScriptTest.java new file mode 100644 index 000000000000..10582903f939 --- /dev/null +++ b/openmetadata-service/src/test/java/org/openmetadata/service/search/scripts/SoftDeleteScriptTest.java @@ -0,0 +1,53 @@ +/* + * Copyright 2026 Collate. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.openmetadata.service.search.scripts; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import org.junit.jupiter.api.Test; +import org.openmetadata.service.search.capability.EntityIndexCapability; + +class SoftDeleteScriptTest { + + @Test + void rendersBooleanWithoutQuotes() { + assertEquals( + "ctx._source.put('deleted', true)", + new SoftDeleteScript(true).painless(), + "the latent quoting bug — '%s' wrapping the boolean — was the reason the field landed" + + " as a string rather than a JSON boolean. The typed script must emit a JSON" + + " boolean."); + assertEquals("ctx._source.put('deleted', false)", new SoftDeleteScript(false).painless()); + } + + @Test + void compatibleWithEntitiesThatCarryTheDeletedField() { + SoftDeleteScript script = new SoftDeleteScript(true); + + assertTrue(script.compatibleWith(EntityIndexCapability.forEntity("table"))); + assertFalse( + script.compatibleWith(EntityIndexCapability.forTimeSeries("testCaseResolutionStatus")), + "time-series entities have no `deleted` field — the regression that broke Incident " + + "Manager"); + assertFalse( + script.compatibleWith(null), + "unregistered entity types are treated as incompatible — fail-safe"); + } + + @Test + void paramsAreEmpty() { + assertEquals(0, new SoftDeleteScript(true).params().size()); + } +} diff --git a/openmetadata-service/src/test/java/org/openmetadata/service/search/validation/IndexMappingValidatorTest.java b/openmetadata-service/src/test/java/org/openmetadata/service/search/validation/IndexMappingValidatorTest.java new file mode 100644 index 000000000000..eda01ff96e1e --- /dev/null +++ b/openmetadata-service/src/test/java/org/openmetadata/service/search/validation/IndexMappingValidatorTest.java @@ -0,0 +1,101 @@ +/* + * Copyright 2026 Collate. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.openmetadata.service.search.validation; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.util.List; +import java.util.Map; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.openmetadata.search.IndexMapping; +import org.openmetadata.service.search.capability.EntityIndexCapability; +import org.openmetadata.service.search.capability.EntityIndexCapabilityRegistry; + +class IndexMappingValidatorTest { + + @BeforeEach + @AfterEach + void resetRegistry() { + EntityIndexCapabilityRegistry.clear(); + } + + @Test + void flagsParentTargetingTimeSeriesChild() { + EntityIndexCapabilityRegistry.register(EntityIndexCapability.forEntity("testCase")); + EntityIndexCapabilityRegistry.register( + EntityIndexCapability.forTimeSeries("testCaseResolutionStatus")); + + IndexMapping testCaseMapping = + IndexMapping.builder() + .indexName("test_case_search_index") + .alias("testCase") + .childAliases(List.of("testCaseResolutionStatus")) + .indexMappingFile("/elasticsearch/%s/test_case_index_mapping.json") + .build(); + + List warnings = IndexMappingValidator.validate(Map.of("testCase", testCaseMapping)); + + assertEquals(1, warnings.size()); + assertTrue( + warnings.get(0).contains("testCase"), + () -> "warning should name the parent; got: " + warnings.get(0)); + assertTrue( + warnings.get(0).contains("testCaseResolutionStatus"), + () -> "warning should name the child; got: " + warnings.get(0)); + } + + @Test + void silentWhenAllChildrenAreCompatible() { + EntityIndexCapabilityRegistry.register(EntityIndexCapability.forEntity("table")); + EntityIndexCapabilityRegistry.register(EntityIndexCapability.forEntity("tableColumn")); + + IndexMapping tableMapping = + IndexMapping.builder() + .indexName("table_search_index") + .alias("table") + .childAliases(List.of("tableColumn")) + .indexMappingFile("/elasticsearch/%s/table_index_mapping.json") + .build(); + + assertEquals(0, IndexMappingValidator.validate(Map.of("table", tableMapping)).size()); + } + + @Test + void flagsUnregisteredChildAlias() { + EntityIndexCapabilityRegistry.register(EntityIndexCapability.forEntity("table")); + + IndexMapping tableMapping = + IndexMapping.builder() + .indexName("table_search_index") + .alias("table") + .childAliases(List.of("ghost")) + .indexMappingFile("/elasticsearch/%s/table_index_mapping.json") + .build(); + + List warnings = IndexMappingValidator.validate(Map.of("table", tableMapping)); + + assertEquals(1, warnings.size()); + assertTrue( + warnings.get(0).contains("no registered capability"), + () -> "warning should mention missing capability; got: " + warnings.get(0)); + } + + @Test + void emptyInputProducesNoWarnings() { + assertEquals(0, IndexMappingValidator.validate(Map.of()).size()); + assertEquals(0, IndexMappingValidator.validate(null).size()); + } +} diff --git a/openmetadata-spec/src/main/resources/json/schema/system/eventPublisherJob.json b/openmetadata-spec/src/main/resources/json/schema/system/eventPublisherJob.json index dafbdff87b6c..c455ad64e88f 100644 --- a/openmetadata-spec/src/main/resources/json/schema/system/eventPublisherJob.json +++ b/openmetadata-spec/src/main/resources/json/schema/system/eventPublisherJob.json @@ -236,6 +236,13 @@ "description": "This schema publisher run modes.", "type": "boolean" }, + "minSuccessRatio": { + "description": "Minimum per-entity success ratio (successRecords / totalRecords) required to mark a per-entity reindex as fully successful. Below this threshold the per-entity run is flagged as not fully successful; promotion still proceeds when the staged index contains at least one document, through the existing doc-count rescue in DefaultRecreateHandler. Default 0.95.", + "type": "number", + "default": 0.95, + "minimum": 0, + "maximum": 1 + }, "batchSize": { "description": "Maximum number of events sent in a batch (Default 10).", "type": "integer", diff --git a/openmetadata-ui/src/main/resources/ui/playwright/e2e/Features/DataQuality/IncidentManagerAfterSoftDelete.spec.ts b/openmetadata-ui/src/main/resources/ui/playwright/e2e/Features/DataQuality/IncidentManagerAfterSoftDelete.spec.ts new file mode 100644 index 000000000000..dda94572e660 --- /dev/null +++ b/openmetadata-ui/src/main/resources/ui/playwright/e2e/Features/DataQuality/IncidentManagerAfterSoftDelete.spec.ts @@ -0,0 +1,118 @@ +/* + * Copyright 2026 Collate. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * Regression for the soft-delete propagation bug. SearchRepository's + * SOFT_DELETE_RESTORE_SCRIPT was stamping `deleted` onto child docs of every + * alias in the parent's IndexMapping. For testCase that included + * `testCaseResolutionStatus` and `testCaseResult` — both time-series indexes + * whose Java schemas declare no `deleted` field. The poisoned doc broke + * Jackson on read and the Incident Manager page surfaced an + * "Unrecognized field 'deleted'" toast on load. + * + * This test reproduces the failing user path: soft-delete a test case that + * has an incident, then navigate to Incident Manager and confirm the page + * renders without an error toast. + */ + +import test, { expect } from '@playwright/test'; +import { SidebarItem } from '../../../constant/sidebar'; +import { TableClass } from '../../../support/entity/TableClass'; +import { createNewPage, redirectToHomePage } from '../../../utils/common'; +import { sidebarClick } from '../../../utils/sidebar'; + +test.use({ storageState: 'playwright/.auth/admin.json' }); + +test('Incident Manager renders without Jackson error after a test case is soft-deleted', async ({ + browser, + page, +}) => { + const { apiContext, afterAction } = await createNewPage(browser); + + const table = new TableClass(); + + try { + await table.create(apiContext); + const testCase = await table.createTestCase(apiContext); + const testCaseFqn = testCase.fullyQualifiedName as string; + const testCaseId = testCase.id as string; + + await table.addTestCaseResult(apiContext, testCaseFqn, { + result: 'soft-delete propagation regression', + testCaseStatus: 'Failed', + timestamp: Date.now(), + }); + + // Wait until the incident is actually indexed before we soft-delete — + // otherwise the script propagation race is meaningless. CI runners can be slow on the + // first-time test-result + resolution-status indexing pipeline, so allow up to 2 min. + test.setTimeout(180_000); + // Match the production UI's call shape — the search endpoint expects offset + latest + // (matches `getListTestCaseIncidentStatusFromSearch` in + // openmetadata-ui/src/main/resources/ui/src/rest/incidentManagerAPI.ts). Without them the + // server rejects with 400. + await expect + .poll( + async () => { + const res = await apiContext.get( + `/api/v1/dataQuality/testCases/testCaseIncidentStatus/search/list?testCaseFQN=${encodeURIComponent( + testCaseFqn + )}&limit=5&offset=0&latest=true` + ); + + return res.status(); + }, + { + message: + 'incident status endpoint must serve the test case before soft-delete', + timeout: 120_000, + } + ) + .toBe(200); + + // Soft-delete the test case via the API. This is the path that, before + // the fix, would stamp `deleted` onto the TCRS docs and break the next read. + const deleteRes = await apiContext.delete( + `/api/v1/dataQuality/testCases/${testCaseId}?recursive=true&hardDelete=false` + ); + + expect(deleteRes.status()).toBeLessThan(400); + + // The page-load API call that broke before the fix. + const incidentListResponse = page.waitForResponse((response) => + response + .url() + .includes('/api/v1/dataQuality/testCases/testCaseIncidentStatus') + ); + + await redirectToHomePage(page); + await sidebarClick(page, SidebarItem.INCIDENT_MANAGER); + + const response = await incidentListResponse; + + // The API must return 200 — before the fix this returned 500 with + // `Unrecognized field "deleted"` in the body. + expect(response.status()).toBe(200); + + // And the toast bar must not surface a Jackson "Unrecognized field"/"deleted" error. + // Scope to the toast container so we don't false-positive on legitimate page text + // (e.g. a table cell that happens to contain the word "deleted"). + const errorToast = page + .locator('[data-testid="alert-bar"]') + .filter({ hasText: /Unrecognized field|deleted/i }); + await expect(errorToast).toHaveCount(0); + } finally { + await table.delete(apiContext); + await afterAction(); + } +}); diff --git a/openmetadata-ui/src/main/resources/ui/playwright/e2e/Features/DataQuality/TestCaseStatusAfterReindex.spec.ts b/openmetadata-ui/src/main/resources/ui/playwright/e2e/Features/DataQuality/TestCaseStatusAfterReindex.spec.ts new file mode 100644 index 000000000000..d4d130d0a493 --- /dev/null +++ b/openmetadata-ui/src/main/resources/ui/playwright/e2e/Features/DataQuality/TestCaseStatusAfterReindex.spec.ts @@ -0,0 +1,122 @@ +/* + * Copyright 2026 Collate. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * Regression for the 1.12.7 selective-reindex bug + * (https://github.com/open-metadata/OpenMetadata/pull/27723): + * TestCaseIndex.getRequiredReindexFields() omitted `testCaseResult` and + * `incidentId`, both of which are stripped from the storage JSON. On reindex, + * TestCaseRepository.setFieldsInBulk skipped fetching them and the resulting + * ES doc had no `testCaseStatus` — wiping status from search/UI until a + * per-case write re-populated it. + * + * This test creates a test case, writes a result, forces an entity reindex + * with `recreate=true` (delete + re-add), and asserts the status survives. + */ + +import test, { expect } from '@playwright/test'; +import { TableClass } from '../../../support/entity/TableClass'; +import { createNewPage } from '../../../utils/common'; + +test.use({ storageState: 'playwright/.auth/admin.json' }); + +const TEST_CASE_STATUS = 'Failed' as const; + +test('Test case status survives a full entity reindex', async ({ browser }) => { + const { apiContext, afterAction } = await createNewPage(browser); + + const table = new TableClass(); + + try { + await table.create(apiContext); + + const testCase = await table.createTestCase(apiContext); + const testCaseFqn = testCase.fullyQualifiedName as string; + const testCaseId = testCase.id as string; + + await table.addTestCaseResult(apiContext, testCaseFqn, { + result: 'Reindex regression check', + testCaseStatus: TEST_CASE_STATUS, + timestamp: Date.now(), + }); + + // Wait for the search doc to settle and assert the status is indexed. + await expect + .poll( + async () => { + const res = await apiContext.get( + `/api/v1/search/query?q=fullyQualifiedName:%22${encodeURIComponent( + testCaseFqn + )}%22&index=test_case_search_index` + ); + if (res.status() !== 200) { + return undefined; + } + const body = await res.json(); + const hits = body?.hits?.hits ?? []; + return hits[0]?._source?.testCaseResult?.testCaseStatus; + }, + { + message: + 'pre-reindex: test case search doc must include testCaseResult.testCaseStatus', + timeout: 30_000, + } + ) + .toBe(TEST_CASE_STATUS); + + // Force a recreate-style reindex of the test case — this is the exact + // path that drops the status before the fix. + const reindexRes = await apiContext.post( + '/api/v1/search/reindexEntities?recreate=true', + { + data: [ + { + id: testCaseId, + type: 'testCase', + fullyQualifiedName: testCaseFqn, + }, + ], + } + ); + + expect(reindexRes.status()).toBeLessThan(400); + + // Assert the status is still there after reindex. Before the fix, the + // recreated doc had no testCaseResult and this poll would time out. + await expect + .poll( + async () => { + const res = await apiContext.get( + `/api/v1/search/query?q=fullyQualifiedName:%22${encodeURIComponent( + testCaseFqn + )}%22&index=test_case_search_index` + ); + if (res.status() !== 200) { + return undefined; + } + const body = await res.json(); + const hits = body?.hits?.hits ?? []; + return hits[0]?._source?.testCaseResult?.testCaseStatus; + }, + { + message: + 'post-reindex: test case search doc must still include testCaseResult.testCaseStatus', + timeout: 30_000, + } + ) + .toBe(TEST_CASE_STATUS); + } finally { + await table.delete(apiContext); + await afterAction(); + } +}); diff --git a/openmetadata-ui/src/main/resources/ui/playwright/e2e/Features/DataQuality/TestSuiteSummaryAfterReindex.spec.ts b/openmetadata-ui/src/main/resources/ui/playwright/e2e/Features/DataQuality/TestSuiteSummaryAfterReindex.spec.ts new file mode 100644 index 000000000000..66463ed51469 --- /dev/null +++ b/openmetadata-ui/src/main/resources/ui/playwright/e2e/Features/DataQuality/TestSuiteSummaryAfterReindex.spec.ts @@ -0,0 +1,106 @@ +/* + * Copyright 2026 Collate. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * Regression for the selective-reindex refactor (PR 27723): + * + * TestSuiteIndex.buildSearchIndexDocInternal computes `lastResultTimestamp` + * from `testSuite.getTestCaseResultSummary()`. TestSuiteRepository registers + * a fetcher for that under the field name `"summary"`. The reindex path only + * runs fetchers whose field is in `getRequiredReindexFields()`. Without + * `"summary"` declared, the fetcher does not run, the Index falls through to + * `doc.put("lastResultTimestamp", 0L)` (TestSuiteIndex.java:41), and the DQ + * `/data-quality/test-suites` list page — which sorts by that exact field + * (`TestSuites.component.tsx:175`) — collapses every reindexed suite to the + * 1970 epoch, breaking "most recently run first" ordering. + * + * The test creates a basic suite, writes a result, asserts the field is + * non-zero on the live-write path, forces a `recreate=true` reindex of the + * testSuite, and asserts the field is still non-zero. Before the fix this + * dropped back to 0 — provably the data the UI sort depends on. + */ + +import test, { expect } from '@playwright/test'; +import { TableClass } from '../../../support/entity/TableClass'; +import { createNewPage } from '../../../utils/common'; + +test.use({ storageState: 'playwright/.auth/admin.json' }); + +test('Test suite lastResultTimestamp survives a full entity reindex', async ({ + browser, +}) => { + const { apiContext, afterAction } = await createNewPage(browser); + + const table = new TableClass(); + + try { + await table.create(apiContext); + const testCase = await table.createTestCase(apiContext); + + const resultTimestamp = Date.now(); + await table.addTestCaseResult(apiContext, testCase.fullyQualifiedName, { + result: 'Reindex regression check', + testCaseStatus: 'Success', + timestamp: resultTimestamp, + }); + + const suite = testCase.testSuite as { + id: string; + fullyQualifiedName: string; + }; + + const reindexRes = await apiContext.post( + '/api/v1/search/reindexEntities?recreate=true', + { + data: [ + { + fullyQualifiedName: suite.fullyQualifiedName, + id: suite.id, + type: 'testSuite', + }, + ], + } + ); + + expect(reindexRes.status()).toBeLessThan(400); + + // Before the fix, the recreated doc has lastResultTimestamp=0 because the + // "summary" fetcher never ran and the Index hit its 0L fallback branch. + // After the fix it is the millisecond timestamp of the most recent result. + await expect + .poll( + async () => { + const res = await apiContext.get( + `/api/v1/search/query?q=fullyQualifiedName:%22${encodeURIComponent( + suite.fullyQualifiedName + )}%22&index=test_suite_search_index` + ); + if (res.status() !== 200) { + return 0; + } + const body = await res.json(); + + return body?.hits?.hits?.[0]?._source?.lastResultTimestamp ?? 0; + }, + { + message: + 'post-reindex: test suite doc must still include a non-zero lastResultTimestamp', + timeout: 30_000, + } + ) + .toBeGreaterThan(0); + } finally { + await table.delete(apiContext); + await afterAction(); + } +}); diff --git a/openmetadata-ui/src/main/resources/ui/playwright/e2e/Features/SearchSeparation/ApiEndpoint.spec.ts b/openmetadata-ui/src/main/resources/ui/playwright/e2e/Features/SearchSeparation/ApiEndpoint.spec.ts new file mode 100644 index 000000000000..712049f2bdd3 --- /dev/null +++ b/openmetadata-ui/src/main/resources/ui/playwright/e2e/Features/SearchSeparation/ApiEndpoint.spec.ts @@ -0,0 +1,24 @@ +/* + * Copyright 2026 Collate. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { test } from '@playwright/test'; +import { ApiEndpointClass } from '../../../support/entity/ApiEndpointClass'; +import { registerFilterSeparationSuite } from './searchSeparationSuite'; + +test.use({ storageState: 'playwright/.auth/admin.json' }); + +registerFilterSeparationSuite({ + suiteName: 'ApiEndpoint', + reindexEntityType: 'apiEndpoint', + entityFactory: () => new ApiEndpointClass(), +}); diff --git a/openmetadata-ui/src/main/resources/ui/playwright/e2e/Features/SearchSeparation/Container.spec.ts b/openmetadata-ui/src/main/resources/ui/playwright/e2e/Features/SearchSeparation/Container.spec.ts new file mode 100644 index 000000000000..58d9918d8103 --- /dev/null +++ b/openmetadata-ui/src/main/resources/ui/playwright/e2e/Features/SearchSeparation/Container.spec.ts @@ -0,0 +1,24 @@ +/* + * Copyright 2026 Collate. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { test } from '@playwright/test'; +import { ContainerClass } from '../../../support/entity/ContainerClass'; +import { registerFilterSeparationSuite } from './searchSeparationSuite'; + +test.use({ storageState: 'playwright/.auth/admin.json' }); + +registerFilterSeparationSuite({ + suiteName: 'Container', + reindexEntityType: 'container', + entityFactory: () => new ContainerClass(), +}); diff --git a/openmetadata-ui/src/main/resources/ui/playwright/e2e/Features/SearchSeparation/Dashboard.spec.ts b/openmetadata-ui/src/main/resources/ui/playwright/e2e/Features/SearchSeparation/Dashboard.spec.ts new file mode 100644 index 000000000000..e2222bf6edea --- /dev/null +++ b/openmetadata-ui/src/main/resources/ui/playwright/e2e/Features/SearchSeparation/Dashboard.spec.ts @@ -0,0 +1,24 @@ +/* + * Copyright 2026 Collate. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { test } from '@playwright/test'; +import { DashboardClass } from '../../../support/entity/DashboardClass'; +import { registerFilterSeparationSuite } from './searchSeparationSuite'; + +test.use({ storageState: 'playwright/.auth/admin.json' }); + +registerFilterSeparationSuite({ + suiteName: 'Dashboard', + reindexEntityType: 'dashboard', + entityFactory: () => new DashboardClass(), +}); diff --git a/openmetadata-ui/src/main/resources/ui/playwright/e2e/Features/SearchSeparation/Database.spec.ts b/openmetadata-ui/src/main/resources/ui/playwright/e2e/Features/SearchSeparation/Database.spec.ts new file mode 100644 index 000000000000..5067a3ab0be4 --- /dev/null +++ b/openmetadata-ui/src/main/resources/ui/playwright/e2e/Features/SearchSeparation/Database.spec.ts @@ -0,0 +1,24 @@ +/* + * Copyright 2026 Collate. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { test } from '@playwright/test'; +import { DatabaseClass } from '../../../support/entity/DatabaseClass'; +import { registerFilterSeparationSuite } from './searchSeparationSuite'; + +test.use({ storageState: 'playwright/.auth/admin.json' }); + +registerFilterSeparationSuite({ + suiteName: 'Database', + reindexEntityType: 'database', + entityFactory: () => new DatabaseClass(), +}); diff --git a/openmetadata-ui/src/main/resources/ui/playwright/e2e/Features/SearchSeparation/DatabaseSchema.spec.ts b/openmetadata-ui/src/main/resources/ui/playwright/e2e/Features/SearchSeparation/DatabaseSchema.spec.ts new file mode 100644 index 000000000000..365c9f9c4df6 --- /dev/null +++ b/openmetadata-ui/src/main/resources/ui/playwright/e2e/Features/SearchSeparation/DatabaseSchema.spec.ts @@ -0,0 +1,24 @@ +/* + * Copyright 2026 Collate. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { test } from '@playwright/test'; +import { DatabaseSchemaClass } from '../../../support/entity/DatabaseSchemaClass'; +import { registerFilterSeparationSuite } from './searchSeparationSuite'; + +test.use({ storageState: 'playwright/.auth/admin.json' }); + +registerFilterSeparationSuite({ + suiteName: 'DatabaseSchema', + reindexEntityType: 'databaseSchema', + entityFactory: () => new DatabaseSchemaClass(), +}); diff --git a/openmetadata-ui/src/main/resources/ui/playwright/e2e/Features/SearchSeparation/ExploreFilterSeparation.spec.ts b/openmetadata-ui/src/main/resources/ui/playwright/e2e/Features/SearchSeparation/ExploreFilterSeparation.spec.ts new file mode 100644 index 000000000000..26804786d5be --- /dev/null +++ b/openmetadata-ui/src/main/resources/ui/playwright/e2e/Features/SearchSeparation/ExploreFilterSeparation.spec.ts @@ -0,0 +1,31 @@ +/* + * Copyright 2026 Collate. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * Live-indexing + SearchIndexApp reindex parity for Table. Both paths must produce the same + * separation: Tier on tier.tagFQN, Certification on certification.tagLabel.tagFQN, classification + * and glossary tags in tags[]. See {@link registerFilterSeparationSuite} for the shared logic; + * sibling specs in this folder cover other entity types via the same factory. + */ + +import { test } from '@playwright/test'; +import { TableClass } from '../../../support/entity/TableClass'; +import { registerFilterSeparationSuite } from './searchSeparationSuite'; + +test.use({ storageState: 'playwright/.auth/admin.json' }); + +registerFilterSeparationSuite({ + suiteName: 'Table', + reindexEntityType: 'table', + entityFactory: () => new TableClass(), +}); diff --git a/openmetadata-ui/src/main/resources/ui/playwright/e2e/Features/SearchSeparation/GlossaryRenameCascade.spec.ts b/openmetadata-ui/src/main/resources/ui/playwright/e2e/Features/SearchSeparation/GlossaryRenameCascade.spec.ts new file mode 100644 index 000000000000..da1eaec22aaf --- /dev/null +++ b/openmetadata-ui/src/main/resources/ui/playwright/e2e/Features/SearchSeparation/GlossaryRenameCascade.spec.ts @@ -0,0 +1,184 @@ +/* + * Copyright 2026 Collate. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * Locks in the painless reseparation contract introduced when {@code TAG_RESEPARATION_SCRIPT} + * was appended to every script that mutates {@code ctx._source.tags}. This is the one path + * that the static {@code SearchClientTagScriptSeparationTest} unit test guards by name, but + * doesn't actually execute end-to-end. + * + *

Scenario: tag a Table with a glossary term, then rename the term. The server fires + * {@code UPDATE_GLOSSARY_TERM_TAG_FQN_BY_PREFIX_SCRIPT} against every doc tagged with the + * term — which mutates {@code tags[].tagFQN} in place. Without the appended + * {@code TAG_RESEPARATION_SCRIPT}, {@code glossaryTags[]} keeps the old FQN and queries + * against the dedicated field stop matching. This spec proves the appended snippet runs. + */ + +import test, { expect } from '@playwright/test'; +import { TableClass } from '../../../support/entity/TableClass'; +import { Glossary } from '../../../support/glossary/Glossary'; +import { GlossaryTerm } from '../../../support/glossary/GlossaryTerm'; +import { createNewPage } from '../../../utils/common'; + +test.use({ storageState: 'playwright/.auth/admin.json' }); + +const TIER_FQN = 'Tier.Tier1'; +const CERTIFICATION_FQN = 'Certification.Gold'; + +test('glossary-term rename cascade keeps tags[] + glossaryTags + tier + cert consistent', async ({ + browser, +}) => { + // Create + tag + rename + dual ES poll comfortably exceeds the 60s default. + test.setTimeout(180_000); + const { apiContext, afterAction } = await createNewPage(browser); + + const table = new TableClass(); + const glossary = new Glossary(); + const glossaryTerm = new GlossaryTerm(glossary); + + try { + await glossary.create(apiContext); + await glossaryTerm.create(apiContext); + await table.create(apiContext); + + const originalGlossaryTermFqn = glossaryTerm.responseData + .fullyQualifiedName as string; + + // Apply all four facets so a regression in any of them is observable. + await table.patch({ + apiContext, + patchData: [ + { + op: 'add', + path: '/tags/0', + value: { + name: 'Tier1', + tagFQN: TIER_FQN, + labelType: 'Manual', + state: 'Confirmed', + source: 'Classification', + }, + }, + { + op: 'add', + path: '/tags/1', + value: { + tagFQN: originalGlossaryTermFqn, + labelType: 'Manual', + state: 'Confirmed', + source: 'Glossary', + }, + }, + { + op: 'add', + path: '/certification', + value: { + tagLabel: { + tagFQN: CERTIFICATION_FQN, + labelType: 'Manual', + state: 'Confirmed', + source: 'Classification', + }, + }, + }, + ], + }); + + const tableFqn = table.entityResponseData.fullyQualifiedName as string; + + // Pre-rename: confirm live indexing produced the expected separation. + await assertDocShape({ + apiContext, + tableFqn, + expectedGlossaryFqn: originalGlossaryTermFqn, + label: 'pre-rename live shape', + }); + + // Rename the glossary term — triggers UPDATE_GLOSSARY_TERM_TAG_FQN_BY_PREFIX_SCRIPT on + // every doc tagged with this term. Without TAG_RESEPARATION_SCRIPT appended, tags[] is + // updated but glossaryTags[] keeps the old FQN, which the assertion below catches. + const renamedTermName = `${glossaryTerm.responseData.name}_renamed`; + await glossaryTerm.patch(apiContext, [ + { op: 'replace', path: '/name', value: renamedTermName }, + ]); + const newGlossaryTermFqn = glossaryTerm.responseData + .fullyQualifiedName as string; + expect(newGlossaryTermFqn).not.toBe(originalGlossaryTermFqn); + + // Post-rename: tags[].tagFQN AND glossaryTags[] must both reflect the new FQN, tier and + // certification stay on their dedicated fields, no Tier.* leakage into tags[]. + await assertDocShape({ + apiContext, + tableFqn, + expectedGlossaryFqn: newGlossaryTermFqn, + label: 'post-rename cascade shape', + }); + } finally { + await table.delete(apiContext); + await glossaryTerm.delete(apiContext); + await glossary.delete(apiContext); + await afterAction(); + } +}); + +async function assertDocShape(opts: { + apiContext: import('@playwright/test').APIRequestContext; + tableFqn: string; + expectedGlossaryFqn: string; + label: string; +}): Promise { + const { apiContext, tableFqn, expectedGlossaryFqn, label } = opts; + + await expect + .poll( + async () => { + const res = await apiContext.get( + `/api/v1/search/query?q=fullyQualifiedName:%22${encodeURIComponent( + tableFqn + )}%22&index=table_search_index` + ); + if (res.status() !== 200) { + return undefined; + } + const body = await res.json(); + const source = body?.hits?.hits?.[0]?._source; + if (!source) { + return undefined; + } + return { + tier: source.tier?.tagFQN, + certification: source.certification?.tagLabel?.tagFQN, + tagsHasGlossary: (source.tags ?? []).some( + (t: { tagFQN?: string }) => t.tagFQN === expectedGlossaryFqn + ), + glossaryTagsHasFqn: (source.glossaryTags ?? []).includes( + expectedGlossaryFqn + ), + tierNotInTagsBag: !(source.tags ?? []).some( + (t: { tagFQN?: string }) => t.tagFQN === TIER_FQN + ), + }; + }, + { + message: `${label}: tier must be on tier.tagFQN, certification on certification.tagLabel.tagFQN, glossary term in both tags[] and glossaryTags[], and Tier.* must NOT leak into tags[]`, + timeout: 60_000, + } + ) + .toEqual({ + tier: TIER_FQN, + certification: CERTIFICATION_FQN, + tagsHasGlossary: true, + glossaryTagsHasFqn: true, + tierNotInTagsBag: true, + }); +} diff --git a/openmetadata-ui/src/main/resources/ui/playwright/e2e/Features/SearchSeparation/Metric.spec.ts b/openmetadata-ui/src/main/resources/ui/playwright/e2e/Features/SearchSeparation/Metric.spec.ts new file mode 100644 index 000000000000..1e0fff6aa782 --- /dev/null +++ b/openmetadata-ui/src/main/resources/ui/playwright/e2e/Features/SearchSeparation/Metric.spec.ts @@ -0,0 +1,24 @@ +/* + * Copyright 2026 Collate. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { test } from '@playwright/test'; +import { MetricClass } from '../../../support/entity/MetricClass'; +import { registerFilterSeparationSuite } from './searchSeparationSuite'; + +test.use({ storageState: 'playwright/.auth/admin.json' }); + +registerFilterSeparationSuite({ + suiteName: 'Metric', + reindexEntityType: 'metric', + entityFactory: () => new MetricClass(), +}); diff --git a/openmetadata-ui/src/main/resources/ui/playwright/e2e/Features/SearchSeparation/MlModel.spec.ts b/openmetadata-ui/src/main/resources/ui/playwright/e2e/Features/SearchSeparation/MlModel.spec.ts new file mode 100644 index 000000000000..0af351a79bb0 --- /dev/null +++ b/openmetadata-ui/src/main/resources/ui/playwright/e2e/Features/SearchSeparation/MlModel.spec.ts @@ -0,0 +1,24 @@ +/* + * Copyright 2026 Collate. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { test } from '@playwright/test'; +import { MlModelClass } from '../../../support/entity/MlModelClass'; +import { registerFilterSeparationSuite } from './searchSeparationSuite'; + +test.use({ storageState: 'playwright/.auth/admin.json' }); + +registerFilterSeparationSuite({ + suiteName: 'MlModel', + reindexEntityType: 'mlmodel', + entityFactory: () => new MlModelClass(), +}); diff --git a/openmetadata-ui/src/main/resources/ui/playwright/e2e/Features/SearchSeparation/Pipeline.spec.ts b/openmetadata-ui/src/main/resources/ui/playwright/e2e/Features/SearchSeparation/Pipeline.spec.ts new file mode 100644 index 000000000000..532ede2d2e01 --- /dev/null +++ b/openmetadata-ui/src/main/resources/ui/playwright/e2e/Features/SearchSeparation/Pipeline.spec.ts @@ -0,0 +1,24 @@ +/* + * Copyright 2026 Collate. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { test } from '@playwright/test'; +import { PipelineClass } from '../../../support/entity/PipelineClass'; +import { registerFilterSeparationSuite } from './searchSeparationSuite'; + +test.use({ storageState: 'playwright/.auth/admin.json' }); + +registerFilterSeparationSuite({ + suiteName: 'Pipeline', + reindexEntityType: 'pipeline', + entityFactory: () => new PipelineClass(), +}); diff --git a/openmetadata-ui/src/main/resources/ui/playwright/e2e/Features/SearchSeparation/README.md b/openmetadata-ui/src/main/resources/ui/playwright/e2e/Features/SearchSeparation/README.md new file mode 100644 index 000000000000..17dd285b4738 --- /dev/null +++ b/openmetadata-ui/src/main/resources/ui/playwright/e2e/Features/SearchSeparation/README.md @@ -0,0 +1,77 @@ +# SearchSeparation suite + +End-to-end coverage for the doc-shape contract between live indexing and +SearchIndexApp reindex. Every spec runs the same two passes against a fresh +entity instance: + +1. **Live indexing**: PATCH the entity to add Tier, Certification, a + classification tag, and a glossary term. Open Explore and filter by each + dedicated field (`tier.tagFQN`, `certification.tagLabel.tagFQN`, + `tags.tagFQN`). The entity must appear under every filter. +2. **Recreate reindex**: POST `/api/v1/search/reindexEntities?recreate=true` + for the entity, poll the search doc until the separation is preserved + (Tier on `tier.tagFQN`, Cert on `certification.tagLabel.tagFQN`, no Tier + leakage into `tags[]`), then re-run all four Explore filters. + +If live and reindex paths diverge for any of the four facets, the second +pass fails and the offending facet is named in the assertion message. + +## Adding a new entity + +```ts +import { test } from '@playwright/test'; +import { MyEntityClass } from '../../../support/entity/MyEntityClass'; +import { registerFilterSeparationSuite } from './searchSeparationSuite'; + +test.use({ storageState: 'playwright/.auth/admin.json' }); + +registerFilterSeparationSuite({ + suiteName: 'MyEntity', + reindexEntityType: 'myEntity', // matches ENTITY_PATH value + entityFactory: () => new MyEntityClass(), +}); +``` + +The entity class must expose `create(apiContext)`, `delete(apiContext)`, +and `patch({ apiContext, patchData })`. If a new entity class still uses +the legacy positional `patch(apiContext, payload)` signature, normalize it +to the object-based one (as was done for `DatabaseClass` / +`DatabaseSchemaClass`) before slotting it in. + +## Current matrix + +| Entity | Spec | +|---|---| +| Table | `ExploreFilterSeparation.spec.ts` | +| Dashboard | `Dashboard.spec.ts` | +| Topic | `Topic.spec.ts` | +| Pipeline | `Pipeline.spec.ts` | +| MlModel | `MlModel.spec.ts` | +| Container | `Container.spec.ts` | +| ApiEndpoint | `ApiEndpoint.spec.ts` | +| StoredProcedure | `StoredProcedure.spec.ts` | +| Metric | `Metric.spec.ts` | +| Database | `Database.spec.ts` | +| DatabaseSchema | `DatabaseSchema.spec.ts` | + +## Painless-mutation cascade + +`GlossaryRenameCascade.spec.ts` covers the live-update path that uses a +painless script rather than a full-doc rebuild: renaming a glossary term +fires `UPDATE_GLOSSARY_TERM_TAG_FQN_BY_PREFIX_SCRIPT` against every doc +tagged with the term. The script mutates `tags[]` in place and, because +`TAG_RESEPARATION_SCRIPT` is appended, also re-derives `tier`, +`classificationTags`, `glossaryTags` from the updated `tags[]`. If the +reseparation snippet is dropped from any tag-mutating script, this spec +fails — `glossaryTags[]` will keep the old FQN while `tags[]` has the new +one. + +## Not yet covered + +- Service-level entities (`DatabaseService`, `DashboardService`, etc.) — + no entity-level Tier/Cert/Tag/Glossary surface separate from their + children; their docs are covered transitively when the child specs run. + Add an explicit spec here if a service-only filter regression surfaces. +- Time-series entities (`testCaseResolutionStatus`, `testCaseResult`) — do + not implement `TaggableIndex` and have no Tier/Cert/Tag/Glossary surface, + so the separation contract doesn't apply. diff --git a/openmetadata-ui/src/main/resources/ui/playwright/e2e/Features/SearchSeparation/StoredProcedure.spec.ts b/openmetadata-ui/src/main/resources/ui/playwright/e2e/Features/SearchSeparation/StoredProcedure.spec.ts new file mode 100644 index 000000000000..98e32705a218 --- /dev/null +++ b/openmetadata-ui/src/main/resources/ui/playwright/e2e/Features/SearchSeparation/StoredProcedure.spec.ts @@ -0,0 +1,24 @@ +/* + * Copyright 2026 Collate. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { test } from '@playwright/test'; +import { StoredProcedureClass } from '../../../support/entity/StoredProcedureClass'; +import { registerFilterSeparationSuite } from './searchSeparationSuite'; + +test.use({ storageState: 'playwright/.auth/admin.json' }); + +registerFilterSeparationSuite({ + suiteName: 'StoredProcedure', + reindexEntityType: 'storedProcedure', + entityFactory: () => new StoredProcedureClass(), +}); diff --git a/openmetadata-ui/src/main/resources/ui/playwright/e2e/Features/SearchSeparation/Topic.spec.ts b/openmetadata-ui/src/main/resources/ui/playwright/e2e/Features/SearchSeparation/Topic.spec.ts new file mode 100644 index 000000000000..cd0fc9d7e4cc --- /dev/null +++ b/openmetadata-ui/src/main/resources/ui/playwright/e2e/Features/SearchSeparation/Topic.spec.ts @@ -0,0 +1,24 @@ +/* + * Copyright 2026 Collate. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { test } from '@playwright/test'; +import { TopicClass } from '../../../support/entity/TopicClass'; +import { registerFilterSeparationSuite } from './searchSeparationSuite'; + +test.use({ storageState: 'playwright/.auth/admin.json' }); + +registerFilterSeparationSuite({ + suiteName: 'Topic', + reindexEntityType: 'topic', + entityFactory: () => new TopicClass(), +}); diff --git a/openmetadata-ui/src/main/resources/ui/playwright/e2e/Features/SearchSeparation/searchSeparationSuite.ts b/openmetadata-ui/src/main/resources/ui/playwright/e2e/Features/SearchSeparation/searchSeparationSuite.ts new file mode 100644 index 000000000000..97b0888e8b2c --- /dev/null +++ b/openmetadata-ui/src/main/resources/ui/playwright/e2e/Features/SearchSeparation/searchSeparationSuite.ts @@ -0,0 +1,314 @@ +/* + * Copyright 2026 Collate. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * Reusable factory for per-entity "live indexing + SearchIndexApp reindex parity" suites. + * + * Each suite runs two tests against the same entity instance: + * 1. After live PATCH (Tier + Certification + classification tag + glossary term), the four + * Explore filters that target dedicated fields (`tier.tagFQN`, + * `certification.tagLabel.tagFQN`, `tags.tagFQN` for classification + glossary) match the + * entity. + * 2. After a forced `recreate=true` reindex through `/api/v1/search/reindexEntities`, the + * same four filters still match the entity, AND the rebuilt ES doc shape preserves the + * separation (Tier NOT in tags[], certification on certification.tagLabel.tagFQN, etc.). + * + * If either path drifts — live or reindex — one of the two passes will fail. + * + * Adding a new entity type to the coverage matrix is a ~20-line spec that calls + * {@link registerFilterSeparationSuite} with an entity factory. + */ + +import { APIRequestContext, expect, Page, test } from '@playwright/test'; +import { Operation } from 'fast-json-patch'; +import { EntityClass } from '../../../support/entity/EntityClass'; +import { Glossary } from '../../../support/glossary/Glossary'; +import { GlossaryTerm } from '../../../support/glossary/GlossaryTerm'; +import { ClassificationClass } from '../../../support/tag/ClassificationClass'; +import { TagClass } from '../../../support/tag/TagClass'; +import { createAdminApiContext } from '../../../utils/admin'; +import { redirectToHomePage } from '../../../utils/common'; +import { checkExploreSearchFilter } from '../../../utils/entity'; + +const TIER_FQN = 'Tier.Tier1'; +const CERTIFICATION_FQN = 'Certification.Gold'; + +/** + * Structural shape every Playwright entity class implements. We don't extend {@link EntityClass} + * directly because {@code create} / {@code delete} / {@code patch} are declared on the concrete + * subclasses rather than on the abstract base. + */ +export type FilterSeparationEntity = EntityClass & { + entityResponseData: { id: string; fullyQualifiedName?: string }; + create(apiContext: APIRequestContext): Promise; + delete(apiContext: APIRequestContext): Promise; + patch(opts: { + apiContext: APIRequestContext; + patchData: Operation[]; + }): Promise; +}; + +export interface FilterSeparationOptions { + /** Suite label rendered in Playwright output — usually the entity-type display name. */ + suiteName: string; + /** Reindex payload `type` — must match ENTITY_PATH values, e.g. 'dashboard', 'topic'. */ + reindexEntityType: string; + /** Factory called once per suite to construct a fresh entity instance. */ + entityFactory: () => FilterSeparationEntity; +} + +/** + * Registers a `test.describe.serial` block that exercises both indexing paths for the given + * entity factory. Tag/glossary/classification fixtures are created once in `beforeAll` and + * torn down in `afterAll`. + */ +export function registerFilterSeparationSuite( + options: FilterSeparationOptions +): void { + const { suiteName, reindexEntityType, entityFactory } = options; + + test.describe + .serial(`${suiteName} | live + reindex filter separation`, () => { + // Each test does entity setup + reindex + multiple ES polls + Explore UI assertions, + // which can legitimately exceed Playwright's default 30s per-test timeout. Configure + // both the per-test timeout (covers the actual tests) and the per-hook timeout (covers + // beforeAll's entity create + PATCH and afterAll's teardown) explicitly rather than + // relying on test.slow() inside a hook, which is not a supported way to extend hook + // timeouts. + test.describe.configure({ timeout: 180_000 }); + + let entity: FilterSeparationEntity; + const classification = new ClassificationClass(); + const classificationTag = new TagClass({ + classification: classification.data.name, + }); + const glossary = new Glossary(); + const glossaryTerm = new GlossaryTerm(glossary); + + test.beforeAll(async () => { + test.setTimeout(180_000); + const { apiContext, afterAction } = await createAdminApiContext(); + await classification.create(apiContext); + await classificationTag.create(apiContext); + await glossary.create(apiContext); + await glossaryTerm.create(apiContext); + entity = entityFactory(); + await entity.create(apiContext); + await applyAllFacets(apiContext, entity, classificationTag, glossaryTerm); + await afterAction(); + }); + + test.afterAll(async () => { + test.setTimeout(180_000); + const { apiContext, afterAction } = await createAdminApiContext(); + await entity.delete(apiContext); + await glossaryTerm.delete(apiContext); + await glossary.delete(apiContext); + await classificationTag.delete(apiContext); + await classification.delete(apiContext); + await afterAction(); + }); + + test('live indexing produces searchable separation for all four facets', async ({ + page, + }) => { + await redirectToHomePage(page); + await assertAllFourFiltersWork( + page, + entity, + classificationTag, + glossaryTerm + ); + }); + + test('SearchIndexApp recreate reindex preserves searchable separation', async ({ + page, + }) => { + // POST /api/v1/search/reindexEntities requires admin scope. createAdminApiContext fetches + // a fresh admin JWT without opening a UI page; the test's `page` fixture handles the + // post-reindex Explore filter UI assertions. + const { apiContext, afterAction } = await createAdminApiContext(); + + const reindexRes = await apiContext.post( + '/api/v1/search/reindexEntities?recreate=true', + { + data: [ + { + id: entity.entityResponseData.id, + type: reindexEntityType, + fullyQualifiedName: entity.entityResponseData.fullyQualifiedName, + }, + ], + } + ); + + expect(reindexRes.status()).toBeLessThan(400); + + await assertReindexedDocPreservesSeparation( + apiContext, + entity, + classificationTag, + glossaryTerm + ); + + await afterAction(); + await redirectToHomePage(page); + await assertAllFourFiltersWork( + page, + entity, + classificationTag, + glossaryTerm + ); + }); + }); +} + +async function applyAllFacets( + apiContext: APIRequestContext, + entity: FilterSeparationEntity, + classificationTag: TagClass, + glossaryTerm: GlossaryTerm +): Promise { + await entity.patch({ + apiContext, + patchData: [ + { + op: 'add', + path: '/tags/0', + value: { + name: 'Tier1', + tagFQN: TIER_FQN, + labelType: 'Manual', + state: 'Confirmed', + source: 'Classification', + }, + }, + { + op: 'add', + path: '/tags/1', + value: { + tagFQN: classificationTag.responseData.fullyQualifiedName, + labelType: 'Manual', + state: 'Confirmed', + source: 'Classification', + }, + }, + { + op: 'add', + path: '/tags/2', + value: { + tagFQN: glossaryTerm.responseData.fullyQualifiedName, + labelType: 'Manual', + state: 'Confirmed', + source: 'Glossary', + }, + }, + { + op: 'add', + path: '/certification', + value: { + tagLabel: { + tagFQN: CERTIFICATION_FQN, + labelType: 'Manual', + state: 'Confirmed', + source: 'Classification', + }, + }, + }, + ], + }); +} + +async function assertReindexedDocPreservesSeparation( + apiContext: APIRequestContext, + entity: FilterSeparationEntity, + classificationTag: TagClass, + glossaryTerm: GlossaryTerm +): Promise { + await expect + .poll( + async () => { + const res = await apiContext.get( + `/api/v1/search/query?q=fullyQualifiedName:%22${encodeURIComponent( + entity.entityResponseData.fullyQualifiedName ?? '' + )}%22&index=dataAsset` + ); + if (res.status() !== 200) { + return undefined; + } + const body = await res.json(); + const source = body?.hits?.hits?.[0]?._source; + if (!source) { + return undefined; + } + + return { + tier: source.tier?.tagFQN, + certification: source.certification?.tagLabel?.tagFQN, + hasClassificationTag: (source.tags ?? []).some( + (t: { tagFQN?: string }) => + t.tagFQN === classificationTag.responseData.fullyQualifiedName + ), + hasGlossaryTag: (source.tags ?? []).some( + (t: { tagFQN?: string }) => + t.tagFQN === glossaryTerm.responseData.fullyQualifiedName + ), + tierNotInTagsBag: !(source.tags ?? []).some( + (t: { tagFQN?: string }) => t.tagFQN === TIER_FQN + ), + }; + }, + { + message: + 'post-reindex: tier must be on tier.tagFQN, certification on certification.tagLabel.tagFQN, classification + glossary tags in tags[], Tier.* must NOT leak into tags[]', + timeout: 60_000, + } + ) + .toEqual({ + tier: TIER_FQN, + certification: CERTIFICATION_FQN, + hasClassificationTag: true, + hasGlossaryTag: true, + tierNotInTagsBag: true, + }); +} + +async function assertAllFourFiltersWork( + page: Page, + entity: FilterSeparationEntity, + classificationTag: TagClass, + glossaryTerm: GlossaryTerm +): Promise { + await checkExploreSearchFilter(page, 'Tier', 'tier.tagFQN', TIER_FQN, entity); + await checkExploreSearchFilter( + page, + 'Certification', + 'certification.tagLabel.tagFQN', + CERTIFICATION_FQN, + entity + ); + await checkExploreSearchFilter( + page, + 'Tag', + 'tags.tagFQN', + classificationTag.responseData.fullyQualifiedName, + entity + ); + await checkExploreSearchFilter( + page, + 'Tag', + 'tags.tagFQN', + glossaryTerm.responseData.fullyQualifiedName, + entity + ); +} diff --git a/openmetadata-ui/src/main/resources/ui/playwright/e2e/VersionPages/ServiceEntityVersionPage.spec.ts b/openmetadata-ui/src/main/resources/ui/playwright/e2e/VersionPages/ServiceEntityVersionPage.spec.ts index 4bef42fe841f..833b4e4defa0 100644 --- a/openmetadata-ui/src/main/resources/ui/playwright/e2e/VersionPages/ServiceEntityVersionPage.spec.ts +++ b/openmetadata-ui/src/main/resources/ui/playwright/e2e/VersionPages/ServiceEntityVersionPage.spec.ts @@ -10,7 +10,13 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -import { expect, Page, test as base } from '@playwright/test'; +import { + APIRequestContext, + expect, + Page, + test as base, +} from '@playwright/test'; +import { Operation } from 'fast-json-patch'; import { BIG_ENTITY_DELETE_TIMEOUT } from '../../constant/delete'; import { ApiCollectionClass } from '../../support/entity/ApiCollectionClass'; import { DatabaseClass } from '../../support/entity/DatabaseClass'; @@ -34,6 +40,31 @@ import { } from '../../utils/common'; import { addMultiOwner, assignTier } from '../../utils/entity'; +/** + * Service entity classes here still use the legacy positional patch(apiContext, payload) + * signature; {@link DatabaseClass} and {@link DatabaseSchemaClass} were normalized to the + * object form ({apiContext, patchData}). This helper dispatches by concrete class so both + * shapes work without resorting to `any` casts. Once all entity classes are normalized this + * function can be deleted and replaced with a single object-form call. + */ +const applyServicePatch = async ( + entity: object, + apiContext: APIRequestContext, + patchData: Operation[] +): Promise => { + if ( + entity instanceof DatabaseClass || + entity instanceof DatabaseSchemaClass + ) { + await entity.patch({ apiContext, patchData }); + return; + } + const legacy = entity as { + patch: (ctx: APIRequestContext, payload: Operation[]) => Promise; + }; + await legacy.patch(apiContext, patchData); +}; + const entities = { 'Api Service': new ApiServiceClass(), 'Api Collection': new ApiCollectionClass(), @@ -71,7 +102,7 @@ test.describe('Service Version pages', () => { for (const entity of Object.values(entities)) { await entity.create(apiContext); const domain = EntityDataClass.domain1.responseData; - await entity.patch(apiContext, [ + const patchData: Operation[] = [ { op: 'add', path: '/tags/0', @@ -109,7 +140,8 @@ test.describe('Service Version pages', () => { }, ], }, - ]); + ]; + await applyServicePatch(entity, apiContext, patchData); } await afterAction(); diff --git a/openmetadata-ui/src/main/resources/ui/playwright/support/entity/DatabaseClass.ts b/openmetadata-ui/src/main/resources/ui/playwright/support/entity/DatabaseClass.ts index 7f80e682c745..a6793fb19139 100644 --- a/openmetadata-ui/src/main/resources/ui/playwright/support/entity/DatabaseClass.ts +++ b/openmetadata-ui/src/main/resources/ui/playwright/support/entity/DatabaseClass.ts @@ -169,11 +169,17 @@ export class DatabaseClass extends EntityClass { }; } - async patch(apiContext: APIRequestContext, payload: Operation[]) { + async patch({ + apiContext, + patchData, + }: { + apiContext: APIRequestContext; + patchData: Operation[]; + }) { const serviceResponse = await apiContext.patch( `/api/v1/databases/${this.entityResponseData?.['id']}`, { - data: payload, + data: patchData, headers: { 'Content-Type': 'application/json-patch+json', }, diff --git a/openmetadata-ui/src/main/resources/ui/playwright/support/entity/DatabaseSchemaClass.ts b/openmetadata-ui/src/main/resources/ui/playwright/support/entity/DatabaseSchemaClass.ts index 4d615ad45eac..1adda64a87cf 100644 --- a/openmetadata-ui/src/main/resources/ui/playwright/support/entity/DatabaseSchemaClass.ts +++ b/openmetadata-ui/src/main/resources/ui/playwright/support/entity/DatabaseSchemaClass.ts @@ -95,11 +95,17 @@ export class DatabaseSchemaClass extends EntityClass { }; } - async patch(apiContext: APIRequestContext, payload: Operation[]) { + async patch({ + apiContext, + patchData, + }: { + apiContext: APIRequestContext; + patchData: Operation[]; + }) { const serviceResponse = await apiContext.patch( `/api/v1/databaseSchemas/${this.entityResponseData?.['id']}`, { - data: payload, + data: patchData, headers: { 'Content-Type': 'application/json-patch+json', }, diff --git a/openmetadata-ui/src/main/resources/ui/src/generated/system/eventPublisherJob.ts b/openmetadata-ui/src/main/resources/ui/src/generated/system/eventPublisherJob.ts index 0c480f5354bb..4ef6f3ff720c 100644 --- a/openmetadata-ui/src/main/resources/ui/src/generated/system/eventPublisherJob.ts +++ b/openmetadata-ui/src/main/resources/ui/src/generated/system/eventPublisherJob.ts @@ -76,6 +76,14 @@ export interface EventPublisherJob { * Maximum number of retries for a failed request */ maxRetries?: number; + /** + * Minimum per-entity success ratio (successRecords / totalRecords) required to mark a + * per-entity reindex as fully successful. Below this threshold the per-entity run is + * flagged as not fully successful; promotion still proceeds when the staged index contains + * at least one document, through the existing doc-count rescue in DefaultRecreateHandler. + * Default 0.95. + */ + minSuccessRatio?: number; /** * Name of the result */